From 46b6b3f2e3e7bab187fb7a698b387ae40276e9b2 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 3 Jul 2024 23:06:02 +0000 Subject: [PATCH 01/79] feat: add ShufflingCache to EpochCache --- .../state-transition/src/cache/epochCache.ts | 196 ++++++++++++++---- .../src/util/epochShuffling.ts | 7 + 2 files changed, 158 insertions(+), 45 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 191aa7f3985c..53df533e829a 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -1,5 +1,5 @@ import {PublicKey} from "@chainsafe/blst"; -import {BLSSignature, CommitteeIndex, Epoch, Slot, ValidatorIndex, phase0, SyncPeriod} from "@lodestar/types"; +import {BLSSignature, CommitteeIndex, Epoch, Slot, ValidatorIndex, phase0, SyncPeriod, RootHex} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainConfig} from "@lodestar/config"; import { ATTESTATION_SUBNET_COUNT, @@ -25,7 +25,12 @@ import { computeProposers, getActivationChurnLimit, } from "../util/index.js"; -import {computeEpochShuffling, EpochShuffling, getShufflingDecisionBlock} from "../util/epochShuffling.js"; +import { + computeEpochShuffling, + EpochShuffling, + getShufflingDecisionBlock, + IShufflingCache, +} from "../util/epochShuffling.js"; import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../util/syncCommittee.js"; import {sumTargetUnslashedBalanceIncrements} from "../util/targetUnslashedBalance.js"; import {getTotalSlashingsByIncrement} from "../epoch/processSlashings.js"; @@ -46,12 +51,12 @@ export type EpochCacheImmutableData = { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; + shufflingCache?: IShufflingCache; }; export type EpochCacheOpts = { skipSyncCommitteeCache?: boolean; skipSyncPubkeys?: boolean; - shufflingGetter?: ShufflingGetter; }; /** Defers computing proposers by persisting only the seed, and dropping it once indexes are computed */ @@ -99,6 +104,11 @@ export class EpochCache { * $VALIDATOR_COUNT x BLST deserialized pubkey (Jacobian coordinates) */ index2pubkey: Index2PubkeyCache; + /** + * ShufflingCache is passed in from `beacon-node` so should be available at runtime but may not be + * present during testing. + */ + shufflingCache?: IShufflingCache; /** * Indexes of the block proposers for the current epoch. * @@ -116,6 +126,12 @@ export class EpochCache { */ proposersNextEpoch: ProposersDeferred; + /** + * Epoch decision roots to look up correct shuffling from the Shuffling Cache + */ + previousDecisionRoot: RootHex; + currentDecisionRoot: RootHex; + nextDecisionRoot: RootHex; /** * Shuffling of validator indexes. Immutable through the epoch, then it's replaced entirely. * Note: Per spec definition, shuffling will always be defined. They are never called before loadState() @@ -126,7 +142,12 @@ export class EpochCache { /** Same as previousShuffling */ currentShuffling: EpochShuffling; /** Same as previousShuffling */ - nextShuffling: EpochShuffling; + nextShuffling: EpochShuffling | null; + /** + * Cache nextActiveIndices so that in afterProcessEpoch the next shuffling can be build synchronously + * in case it is not built or the ShufflingCache is not available + */ + nextActiveIndices: ValidatorIndex[]; /** * Effective balances, for altair processAttestations() */ @@ -196,19 +217,29 @@ export class EpochCache { nextSyncCommitteeIndexed: SyncCommitteeCache; // TODO: Helper stats - epoch: Epoch; syncPeriod: SyncPeriod; + epoch: Epoch; + + get nextEpoch(): Epoch { + return this.epoch + 1; + } + constructor(data: { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; + shufflingCache?: IShufflingCache; proposers: number[]; proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; + previousDecisionRoot: RootHex; + currentDecisionRoot: RootHex; + nextDecisionRoot: RootHex; previousShuffling: EpochShuffling; currentShuffling: EpochShuffling; - nextShuffling: EpochShuffling; + nextShuffling: EpochShuffling | null; + nextActiveIndices: ValidatorIndex[]; effectiveBalanceIncrements: EffectiveBalanceIncrements; totalSlashingsByIncrement: number; syncParticipantReward: number; @@ -229,12 +260,17 @@ export class EpochCache { this.config = data.config; this.pubkey2index = data.pubkey2index; this.index2pubkey = data.index2pubkey; + this.shufflingCache = data.shufflingCache; this.proposers = data.proposers; this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; + this.previousDecisionRoot = data.previousDecisionRoot; + this.currentDecisionRoot = data.currentDecisionRoot; + this.nextDecisionRoot = data.nextDecisionRoot; this.previousShuffling = data.previousShuffling; this.currentShuffling = data.currentShuffling; this.nextShuffling = data.nextShuffling; + this.nextActiveIndices = data.nextActiveIndices; this.effectiveBalanceIncrements = data.effectiveBalanceIncrements; this.totalSlashingsByIncrement = data.totalSlashingsByIncrement; this.syncParticipantReward = data.syncParticipantReward; @@ -261,7 +297,7 @@ export class EpochCache { */ static createFromState( state: BeaconStateAllForks, - {config, pubkey2index, index2pubkey}: EpochCacheImmutableData, + {config, pubkey2index, index2pubkey, shufflingCache}: EpochCacheImmutableData, opts?: EpochCacheOpts ): EpochCache { const currentEpoch = computeEpochAtSlot(state.slot); @@ -290,12 +326,12 @@ export class EpochCache { // BeaconChain could provide a shuffling cache to avoid re-computing shuffling every epoch // in that case, we don't need to compute shufflings again - const previousShufflingDecisionBlock = getShufflingDecisionBlock(state, previousEpoch); - const cachedPreviousShuffling = opts?.shufflingGetter?.(previousEpoch, previousShufflingDecisionBlock); - const currentShufflingDecisionBlock = getShufflingDecisionBlock(state, currentEpoch); - const cachedCurrentShuffling = opts?.shufflingGetter?.(currentEpoch, currentShufflingDecisionBlock); - const nextShufflingDecisionBlock = getShufflingDecisionBlock(state, nextEpoch); - const cachedNextShuffling = opts?.shufflingGetter?.(nextEpoch, nextShufflingDecisionBlock); + const previousDecisionRoot = getShufflingDecisionBlock(state, previousEpoch); + const cachedPreviousShuffling = shufflingCache?.getSync(previousEpoch, previousDecisionRoot); + const currentDecisionRoot = getShufflingDecisionBlock(state, currentEpoch); + const cachedCurrentShuffling = shufflingCache?.getSync(currentEpoch, currentDecisionRoot); + const nextDecisionRoot = getShufflingDecisionBlock(state, nextEpoch); + const cachedNextShuffling = shufflingCache?.getSync(nextEpoch, nextDecisionRoot); for (let i = 0; i < validatorCount; i++) { const validator = validators[i]; @@ -338,16 +374,32 @@ export class EpochCache { throw Error("totalActiveBalanceIncrements >= Number.MAX_SAFE_INTEGER. MAX_EFFECTIVE_BALANCE is too low."); } - const currentShuffling = - cachedCurrentShuffling ?? - computeEpochShuffling(state, currentActiveIndices, currentActiveIndices.length, currentEpoch); - const previousShuffling = - cachedPreviousShuffling ?? - (isGenesis - ? currentShuffling - : computeEpochShuffling(state, previousActiveIndices, previousActiveIndices.length, previousEpoch)); - const nextShuffling = - cachedNextShuffling ?? computeEpochShuffling(state, nextActiveIndices, nextActiveIndices.length, nextEpoch); + let currentShuffling: EpochShuffling; + if (cachedCurrentShuffling) { + currentShuffling = cachedCurrentShuffling; + } else { + currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentEpoch); + shufflingCache?.set(currentShuffling, currentDecisionRoot); + } + + let previousShuffling: EpochShuffling; + if (cachedPreviousShuffling) { + previousShuffling = cachedPreviousShuffling; + } else if (isGenesis) { + // TODO: (@matthewkeil) does this need to be added to the cache at previousEpoch and previousDecisionRoot? + previousShuffling = currentShuffling; + } else { + previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); + shufflingCache?.set(previousShuffling, previousDecisionRoot); + } + + let nextShuffling: EpochShuffling; + if (cachedNextShuffling) { + nextShuffling = cachedNextShuffling; + } else { + nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); + shufflingCache?.set(nextShuffling, nextDecisionRoot); + } const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER); @@ -431,13 +483,18 @@ export class EpochCache { config, pubkey2index, index2pubkey, + shufflingCache, proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics proposersPrevEpoch: null, proposersNextEpoch, + previousDecisionRoot, + currentDecisionRoot, + nextDecisionRoot, previousShuffling, currentShuffling, nextShuffling, + nextActiveIndices, effectiveBalanceIncrements, totalSlashingsByIncrement, syncParticipantReward, @@ -469,13 +526,18 @@ export class EpochCache { // Common append-only structures shared with all states, no need to clone pubkey2index: this.pubkey2index, index2pubkey: this.index2pubkey, + shufflingCache: this.shufflingCache, // Immutable data proposers: this.proposers, proposersPrevEpoch: this.proposersPrevEpoch, proposersNextEpoch: this.proposersNextEpoch, + previousDecisionRoot: this.previousDecisionRoot, + currentDecisionRoot: this.currentDecisionRoot, + nextDecisionRoot: this.nextDecisionRoot, previousShuffling: this.previousShuffling, currentShuffling: this.currentShuffling, nextShuffling: this.nextShuffling, + nextActiveIndices: this.nextActiveIndices, // Uint8Array, requires cloning, but it is cloned only when necessary before an epoch transition // See EpochCache.beforeEpochTransition() effectiveBalanceIncrements: this.effectiveBalanceIncrements, @@ -510,17 +572,19 @@ export class EpochCache { nextEpochTotalActiveBalanceByIncrement: number; } ): void { + this.previousDecisionRoot = this.currentDecisionRoot; this.previousShuffling = this.currentShuffling; - this.currentShuffling = this.nextShuffling; - const currEpoch = this.currentShuffling.epoch; - const nextEpoch = currEpoch + 1; - - this.nextShuffling = computeEpochShuffling( - state, - epochTransitionCache.nextEpochShufflingActiveValidatorIndices, - epochTransitionCache.nextEpochShufflingActiveIndicesLength, - nextEpoch - ); + + this.currentDecisionRoot = this.nextDecisionRoot; + this.currentShuffling = this.shufflingCache + ? this.shufflingCache.getOrBuild(this.nextEpoch, this.nextDecisionRoot, this.nextActiveIndices) + : computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); + + const currentEpoch = this.nextEpoch; + this.nextShuffling = null; + this.nextDecisionRoot = getShufflingDecisionBlock(state, currentEpoch + 1); + this.nextActiveIndices = epochTransitionCache.nextEpochShufflingActiveValidatorIndices; + this.shufflingCache?.build(currentEpoch + 1, this.nextDecisionRoot, this.nextActiveIndices); // Roll current proposers into previous proposers for metrics this.proposersPrevEpoch = this.proposers; @@ -529,7 +593,7 @@ export class EpochCache { this.proposers = computeProposers(currentProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements); // Only pre-compute the seed since it's very cheap. Do the expensive computeProposers() call only on demand. - this.proposersNextEpoch = {computed: false, seed: getSeed(state, this.nextShuffling.epoch, DOMAIN_BEACON_PROPOSER)}; + this.proposersNextEpoch = {computed: false, seed: getSeed(state, currentEpoch + 1, DOMAIN_BEACON_PROPOSER)}; // TODO: DEDUPLICATE from createEpochCache // @@ -555,14 +619,14 @@ export class EpochCache { ); // Maybe advance exitQueueEpoch at the end of the epoch if there haven't been any exists for a while - const exitQueueEpoch = computeActivationExitEpoch(currEpoch); + const exitQueueEpoch = computeActivationExitEpoch(currentEpoch); if (exitQueueEpoch > this.exitQueueEpoch) { this.exitQueueEpoch = exitQueueEpoch; this.exitQueueChurn = 0; } this.totalActiveBalanceIncrements = epochTransitionCache.nextEpochTotalActiveBalanceByIncrement; - if (currEpoch >= this.config.ALTAIR_FORK_EPOCH) { + if (currentEpoch >= this.config.ALTAIR_FORK_EPOCH) { this.syncParticipantReward = computeSyncParticipantReward(this.totalActiveBalanceIncrements); this.syncProposerReward = Math.floor(this.syncParticipantReward * PROPOSER_WEIGHT_FACTOR); this.baseRewardPerIncrement = computeBaseRewardPerIncrement(this.totalActiveBalanceIncrements); @@ -673,7 +737,7 @@ export class EpochCache { if (!this.proposersNextEpoch.computed) { const indexes = computeProposers( this.proposersNextEpoch.seed, - this.nextShuffling, + this.getShufflingAtEpoch(this.nextEpoch), this.effectiveBalanceIncrements ); this.proposersNextEpoch = {computed: true, indexes}; @@ -784,6 +848,13 @@ export class EpochCache { getShufflingAtEpoch(epoch: Epoch): EpochShuffling { const shuffling = this.getShufflingAtEpochOrNull(epoch); if (shuffling === null) { + if (epoch === this.nextEpoch) { + throw new EpochCacheError({ + code: EpochCacheErrorCode.NEXT_SHUFFLING_NOT_AVAILABLE, + epoch: epoch, + decisionRoot: this.getDecisionRoot(this.nextEpoch), + }); + } throw new EpochCacheError({ code: EpochCacheErrorCode.COMMITTEE_EPOCH_OUT_OF_RANGE, currentEpoch: this.currentShuffling.epoch, @@ -794,15 +865,38 @@ export class EpochCache { return shuffling; } + getDecisionRoot(epoch: Epoch): RootHex { + switch (epoch) { + case this.epoch - 1: + return this.previousDecisionRoot; + case this.epoch: + return this.currentDecisionRoot; + case this.nextEpoch: + return this.nextDecisionRoot; + default: + throw new EpochCacheError({ + code: EpochCacheErrorCode.DECISION_ROOT_EPOCH_OUT_OF_RANGE, + currentEpoch: this.epoch, + requestedEpoch: epoch, + }); + } + } + getShufflingAtEpochOrNull(epoch: Epoch): EpochShuffling | null { - if (epoch === this.previousShuffling.epoch) { - return this.previousShuffling; - } else if (epoch === this.currentShuffling.epoch) { - return this.currentShuffling; - } else if (epoch === this.nextShuffling.epoch) { - return this.nextShuffling; - } else { - return null; + switch (epoch) { + case this.epoch - 1: + return this.previousShuffling; + case this.epoch: + return this.currentShuffling; + case this.nextEpoch: + if (!this.nextShuffling) { + this.nextShuffling = this.shufflingCache + ? this.shufflingCache.getSync(this.nextEpoch, this.getDecisionRoot(this.nextEpoch)) + : null; + } + return this.nextShuffling; + default: + return null; } } @@ -874,6 +968,8 @@ type AttesterDuty = { export enum EpochCacheErrorCode { COMMITTEE_INDEX_OUT_OF_RANGE = "EPOCH_CONTEXT_ERROR_COMMITTEE_INDEX_OUT_OF_RANGE", COMMITTEE_EPOCH_OUT_OF_RANGE = "EPOCH_CONTEXT_ERROR_COMMITTEE_EPOCH_OUT_OF_RANGE", + DECISION_ROOT_EPOCH_OUT_OF_RANGE = "EPOCH_CONTEXT_ERROR_DECISION_ROOT_EPOCH_OUT_OF_RANGE", + NEXT_SHUFFLING_NOT_AVAILABLE = "EPOCH_CONTEXT_ERROR_NEXT_SHUFFLING_NOT_AVAILABLE", NO_SYNC_COMMITTEE = "EPOCH_CONTEXT_ERROR_NO_SYNC_COMMITTEE", PROPOSER_EPOCH_MISMATCH = "EPOCH_CONTEXT_ERROR_PROPOSER_EPOCH_MISMATCH", } @@ -889,6 +985,16 @@ type EpochCacheErrorType = requestedEpoch: Epoch; currentEpoch: Epoch; } + | { + code: EpochCacheErrorCode.DECISION_ROOT_EPOCH_OUT_OF_RANGE; + requestedEpoch: Epoch; + currentEpoch: Epoch; + } + | { + code: EpochCacheErrorCode.NEXT_SHUFFLING_NOT_AVAILABLE; + epoch: Epoch; + decisionRoot: RootHex; + } | { code: EpochCacheErrorCode.NO_SYNC_COMMITTEE; epoch: Epoch; diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index e101da38f297..dd625deb549a 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -13,6 +13,13 @@ import {unshuffleList} from "./shuffle.js"; import {computeStartSlotAtEpoch} from "./epoch.js"; import {getBlockRootAtSlot} from "./blockRoot.js"; +export interface IShufflingCache { + getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null; + getOrBuild(epoch: Epoch, decisionRoot: RootHex, activeIndices: ValidatorIndex[]): EpochShuffling; + build(epoch: Epoch, decisionRoot: RootHex, activeIndices: ValidatorIndex[]): void; + set(shuffling: EpochShuffling, decisionRoot: RootHex): void; +} + /** * Readonly interface for EpochShuffling. */ From 99269ce10cc9fab8f1d0804d4c6928621e598ef6 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 3 Jul 2024 23:07:15 +0000 Subject: [PATCH 02/79] fix: implementation in state-transition for EpochCache with ShufflingCache --- .../state-transition/src/epoch/processSyncCommitteeUpdates.ts | 2 +- packages/state-transition/src/slot/upgradeStateToAltair.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts b/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts index b3fd9b45053c..062e4cf7ced2 100644 --- a/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts +++ b/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts @@ -14,7 +14,7 @@ export function processSyncCommitteeUpdates(state: CachedBeaconStateAltair): voi const nextEpoch = state.epochCtx.epoch + 1; if (nextEpoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD === 0) { - const activeValidatorIndices = state.epochCtx.nextShuffling.activeIndices; + const activeValidatorIndices = state.epochCtx.nextActiveIndices; const {effectiveBalanceIncrements} = state.epochCtx; const nextSyncCommitteeIndices = getNextSyncCommitteeIndices( diff --git a/packages/state-transition/src/slot/upgradeStateToAltair.ts b/packages/state-transition/src/slot/upgradeStateToAltair.ts index 0afa43930ef0..bdd3d9edc24c 100644 --- a/packages/state-transition/src/slot/upgradeStateToAltair.ts +++ b/packages/state-transition/src/slot/upgradeStateToAltair.ts @@ -71,7 +71,7 @@ export function upgradeStateToAltair(statePhase0: CachedBeaconStatePhase0): Cach const {syncCommittee, indices} = getNextSyncCommittee( stateAltair, - stateAltair.epochCtx.nextShuffling.activeIndices, + stateAltair.epochCtx.nextActiveIndices, stateAltair.epochCtx.effectiveBalanceIncrements ); const syncCommitteeView = ssz.altair.SyncCommittee.toViewDU(syncCommittee); From 52187c9199437ecb36505b07ca6c01fd131f5ba4 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 3 Jul 2024 23:12:28 +0000 Subject: [PATCH 03/79] feat: remove shufflingCache.processState --- .../src/chain/blocks/importBlock.ts | 6 --- packages/beacon-node/src/chain/chain.ts | 3 -- .../beacon-node/src/chain/shufflingCache.ts | 47 ------------------- 3 files changed, 56 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 9d46410a8638..3b363ebcf1e9 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -334,12 +334,6 @@ export async function importBlock( this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postState.slot}); } - if (parentEpoch < blockEpoch) { - // current epoch and previous epoch are likely cached in previous states - this.shufflingCache.processState(postState, postState.epochCtx.nextShuffling.epoch); - this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: blockSlot}); - } - if (blockSlot % SLOTS_PER_EPOCH === 0) { // Cache state to preserve epoch transition work const checkpointState = postState; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index a12ee4a21f64..16d61136057b 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -254,9 +254,6 @@ export class BeaconChain implements IBeaconChain { pubkey2index: new PubkeyIndexMap(), index2pubkey: [], }); - this.shufflingCache.processState(cachedState, cachedState.epochCtx.previousShuffling.epoch); - this.shufflingCache.processState(cachedState, cachedState.epochCtx.currentShuffling.epoch); - this.shufflingCache.processState(cachedState, cachedState.epochCtx.nextShuffling.epoch); // Persist single global instance of state caches this.pubkey2index = cachedState.epochCtx.pubkey2index; diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 23177142d846..2b198c361214 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -71,53 +71,6 @@ export class ShufflingCache { this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS; } - /** - * Extract shuffling from state and add to cache - */ - processState(state: CachedBeaconStateAllForks, shufflingEpoch: Epoch): EpochShuffling { - const decisionBlockHex = getDecisionBlock(state, shufflingEpoch); - let shuffling: EpochShuffling; - switch (shufflingEpoch) { - case state.epochCtx.nextShuffling.epoch: - shuffling = state.epochCtx.nextShuffling; - break; - case state.epochCtx.currentShuffling.epoch: - shuffling = state.epochCtx.currentShuffling; - break; - case state.epochCtx.previousShuffling.epoch: - shuffling = state.epochCtx.previousShuffling; - break; - default: - throw new Error(`Shuffling not found from state ${state.slot} for epoch ${shufflingEpoch}`); - } - - let cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionBlockHex); - if (cacheItem !== undefined) { - // update existing promise - if (isPromiseCacheItem(cacheItem)) { - // unblock consumers of this promise - cacheItem.resolveFn(shuffling); - // then update item type to shuffling - cacheItem = { - type: CacheItemType.shuffling, - shuffling, - }; - this.add(shufflingEpoch, decisionBlockHex, cacheItem); - // we updated type to CacheItemType.shuffling so the above fields are not used anyway - this.metrics?.shufflingCache.processStateUpdatePromise.inc(); - } else { - // ShufflingCacheItem, do nothing - this.metrics?.shufflingCache.processStateNoOp.inc(); - } - } else { - // not found, new shuffling - this.add(shufflingEpoch, decisionBlockHex, {type: CacheItemType.shuffling, shuffling}); - this.metrics?.shufflingCache.processStateInsertNew.inc(); - } - - return shuffling; - } - /** * Insert a promise to make sure we don't regen state for the same shuffling. * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. From b6b2f205588086bc826d0b3fafb2f382266bea7f Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 4 Jul 2024 00:01:51 +0000 Subject: [PATCH 04/79] feat: implement ShufflingCache changes in beacon-node --- .../src/chain/blocks/importBlock.ts | 1 - packages/beacon-node/src/chain/chain.ts | 3 +- .../beacon-node/src/chain/shufflingCache.ts | 96 +++++++++++++++---- .../src/chain/validation/attestation.ts | 2 +- .../state-transition/src/cache/epochCache.ts | 6 +- .../src/util/epochShuffling.ts | 9 +- 6 files changed, 91 insertions(+), 26 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 3b363ebcf1e9..4a4f2243712d 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -65,7 +65,6 @@ export async function importBlock( const blockRootHex = toHexString(blockRoot); const currentEpoch = computeEpochAtSlot(this.forkChoice.getTime()); const blockEpoch = computeEpochAtSlot(blockSlot); - const parentEpoch = computeEpochAtSlot(parentBlockSlot); const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; const blockDelaySec = (fullyVerifiedBlock.seenTimestampSec - postState.genesisTime) % this.config.SECONDS_PER_SLOT; const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000); diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 16d61136057b..5c1c090f6257 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -13,6 +13,7 @@ import { PubkeyIndexMap, EpochShuffling, computeEndSlotAtEpoch, + getShufflingDecisionBlock, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import { @@ -890,7 +891,7 @@ export class BeaconChain implements IBeaconChain { } // resolve the promise to unblock other calls of the same epoch and dependent root - return this.shufflingCache.processState(state, attEpoch); + return this.shufflingCache.get(attEpoch, getShufflingDecisionBlock(state, attEpoch)); } /** diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 2b198c361214..8753df94698a 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -1,10 +1,7 @@ -import {toHexString} from "@chainsafe/ssz"; -import {CachedBeaconStateAllForks, EpochShuffling, getShufflingDecisionBlock} from "@lodestar/state-transition"; -import {Epoch, RootHex, ssz} from "@lodestar/types"; -import {MapDef, pruneSetToMax} from "@lodestar/utils"; -import {GENESIS_SLOT} from "@lodestar/params"; +import {BeaconStateAllForks, EpochShuffling, IShufflingCache, computeEpochShuffling} from "@lodestar/state-transition"; +import {Epoch, RootHex} from "@lodestar/types"; +import {LodestarError, MapDef, pruneSetToMax} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; -import {computeAnchorCheckpoint} from "./initState.js"; /** * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case: @@ -48,7 +45,7 @@ export type ShufflingCacheOpts = { * - if a shuffling is not available (which does not happen with default chain option of maxSkipSlots = 32), track a promise to make sure we don't compute the same shuffling twice * - skip computing shuffling when loading state bytes from disk */ -export class ShufflingCache { +export class ShufflingCache implements IShufflingCache { /** LRU cache implemented as a map, pruned every time we add an item */ private readonly itemsByDecisionRootByEpoch: MapDef> = new MapDef( () => new Map() @@ -106,7 +103,7 @@ export class ShufflingCache { * If there's a promise, it means we are computing the same shuffling, so we wait for the promise to resolve. * Return null if we don't have a shuffling for this epoch and dependentRootHex. */ - async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { + async getShufflingOrNull(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); if (cacheItem === undefined) { return null; @@ -120,8 +117,20 @@ export class ShufflingCache { } } + async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { + const item = await this.getShufflingOrNull(shufflingEpoch, decisionRootHex); + if (!item) { + throw new ShufflingCacheError({ + code: ShufflingCacheErrorCode.NO_SHUFFLING_FOUND, + epoch: shufflingEpoch, + decisionRoot: decisionRootHex, + }); + } + return item; + } + /** - * Same to get() function but synchronous. + * Same to getShufflingOrNull() function but synchronous. */ getSync(shufflingEpoch: Epoch, decisionRootHex: RootHex): EpochShuffling | null { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); @@ -137,6 +146,56 @@ export class ShufflingCache { return null; } + getOrBuildSync( + epoch: number, + decisionRoot: string, + state: BeaconStateAllForks, + activeIndices: number[] + ): EpochShuffling { + const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); + if (cacheItem && isShufflingCacheItem(cacheItem)) { + // this.metrics?.shufflingCache.cacheHitInEpochTransition(); + return cacheItem.shuffling; + } + // if (cacheItem) { + // this.metrics?.shufflingCache.cacheMissInEpochTransition(); + // } else { + // this.metrics?.shufflingCache.shufflingPromiseNotResolvedInEpochTransition(); + // } + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + this.add(epoch, decisionRoot, { + type: CacheItemType.shuffling, + shuffling, + }); + return shuffling; + } + + build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): void { + let resolveFn: (shuffling: EpochShuffling) => void = () => {}; + this.add(epoch, decisionRoot, { + type: CacheItemType.promise, + resolveFn, + promise: new Promise((resolve) => { + resolveFn = resolve; + }), + }); + setTimeout(() => { + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + this.add(epoch, decisionRoot, { + type: CacheItemType.shuffling, + shuffling, + }); + }, 100); + } + + set(shuffling: EpochShuffling, decisionRoot: string): void { + const cacheItem: ShufflingCacheItem = { + shuffling, + type: CacheItemType.shuffling, + }; + this.add(shuffling.epoch, decisionRoot, cacheItem); + } + private add(shufflingEpoch: Epoch, decisionBlock: RootHex, cacheItem: CacheItem): void { this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionBlock, cacheItem); pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs); @@ -151,13 +210,14 @@ function isPromiseCacheItem(item: CacheItem): item is PromiseCacheItem { return item.type === CacheItemType.promise; } -/** - * Get the shuffling decision block root for the given epoch of given state - * - Special case close to genesis block, return the genesis block root - * - This is similar to forkchoice.getDependentRoot() function, otherwise we cannot get cached shuffing in attestation verification when syncing from genesis. - */ -function getDecisionBlock(state: CachedBeaconStateAllForks, epoch: Epoch): RootHex { - return state.slot > GENESIS_SLOT - ? getShufflingDecisionBlock(state, epoch) - : toHexString(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(state.config, state).blockHeader)); +export enum ShufflingCacheErrorCode { + NO_SHUFFLING_FOUND = "SHUFFLING_CACHE_ERROR_NO_SHUFFLING_FOUND", } + +type ShufflingCacheErrorType = { + code: ShufflingCacheErrorCode.NO_SHUFFLING_FOUND; + epoch: Epoch; + decisionRoot: RootHex; +}; + +export class ShufflingCacheError extends LodestarError {} diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index fc39534b45e6..389abd8a7c03 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -584,7 +584,7 @@ export async function getShufflingForAttestationVerification( const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); const shufflingDependentRoot = getShufflingDependentRoot(chain.forkChoice, attEpoch, blockEpoch, attHeadBlock); - const shuffling = await chain.shufflingCache.get(attEpoch, shufflingDependentRoot); + const shuffling = await chain.shufflingCache.getShufflingOrNull(attEpoch, shufflingDependentRoot); if (shuffling) { // most of the time, we should get the shuffling from cache chain.metrics?.gossipAttestation.shufflingCacheHit.inc({caller: regenCaller}); diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 53df533e829a..4c4a7d7b96f4 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -36,7 +36,7 @@ import {sumTargetUnslashedBalanceIncrements} from "../util/targetUnslashedBalanc import {getTotalSlashingsByIncrement} from "../epoch/processSlashings.js"; import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements.js"; import {Index2PubkeyCache, PubkeyIndexMap, syncPubkeys} from "./pubkeyCache.js"; -import {BeaconStateAllForks, BeaconStateAltair, ShufflingGetter} from "./types.js"; +import {BeaconStateAllForks, BeaconStateAltair} from "./types.js"; import { computeSyncCommitteeCache, getSyncCommitteeCache, @@ -577,14 +577,14 @@ export class EpochCache { this.currentDecisionRoot = this.nextDecisionRoot; this.currentShuffling = this.shufflingCache - ? this.shufflingCache.getOrBuild(this.nextEpoch, this.nextDecisionRoot, this.nextActiveIndices) + ? this.shufflingCache.getOrBuildSync(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices) : computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); const currentEpoch = this.nextEpoch; this.nextShuffling = null; this.nextDecisionRoot = getShufflingDecisionBlock(state, currentEpoch + 1); this.nextActiveIndices = epochTransitionCache.nextEpochShufflingActiveValidatorIndices; - this.shufflingCache?.build(currentEpoch + 1, this.nextDecisionRoot, this.nextActiveIndices); + this.shufflingCache?.build(currentEpoch + 1, this.nextDecisionRoot, state, this.nextActiveIndices); // Roll current proposers into previous proposers for metrics this.proposersPrevEpoch = this.proposers; diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index dd625deb549a..5033262216d7 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -15,8 +15,13 @@ import {getBlockRootAtSlot} from "./blockRoot.js"; export interface IShufflingCache { getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null; - getOrBuild(epoch: Epoch, decisionRoot: RootHex, activeIndices: ValidatorIndex[]): EpochShuffling; - build(epoch: Epoch, decisionRoot: RootHex, activeIndices: ValidatorIndex[]): void; + getOrBuildSync( + epoch: Epoch, + decisionRoot: RootHex, + state: BeaconStateAllForks, + activeIndices: ValidatorIndex[] + ): EpochShuffling; + build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void; set(shuffling: EpochShuffling, decisionRoot: RootHex): void; } From 4110dc05dcf09503865b129880548f044f96ecd5 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 4 Jul 2024 17:28:51 +0000 Subject: [PATCH 05/79] feat: pass shufflingCache when loading cached state from db --- .../stateCache/persistentCheckpointsCache.ts | 15 +-------------- packages/state-transition/src/cache/stateCache.ts | 3 +++ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 58aeca061bc0..0f89bb0394b1 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -212,20 +212,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } sszTimer?.(); const timer = this.metrics?.stateReloadDuration.startTimer(); - const newCachedState = loadCachedBeaconState( - seedState, - stateBytes, - { - shufflingGetter: (shufflingEpoch, decisionRootHex) => { - const shuffling = this.shufflingCache.getSync(shufflingEpoch, decisionRootHex); - if (shuffling == null) { - this.metrics?.stateReloadShufflingCacheMiss.inc(); - } - return shuffling; - }, - }, - validatorsBytes - ); + const newCachedState = loadCachedBeaconState(seedState, stateBytes, this.shufflingCache, {}, validatorsBytes); newCachedState.commit(); const stateRoot = toHexString(newCachedState.hashTreeRoot()); timer?.(); diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index f4e637e5d665..232b4a56134b 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -1,6 +1,7 @@ import {PublicKey} from "@chainsafe/blst"; import {BeaconConfig} from "@lodestar/config"; import {loadState} from "../util/loadState/loadState.js"; +import {IShufflingCache} from "../util/epochShuffling.js"; import {EpochCache, EpochCacheImmutableData, EpochCacheOpts} from "./epochCache.js"; import { BeaconStateAllForks, @@ -163,6 +164,7 @@ export function createCachedBeaconState( export function loadCachedBeaconState( cachedSeedState: T, stateBytes: Uint8Array, + shufflingCache: IShufflingCache, opts?: EpochCacheOpts, seedValidatorsBytes?: Uint8Array ): T { @@ -188,6 +190,7 @@ export function loadCachedBeaconState Date: Thu, 4 Jul 2024 23:56:26 +0000 Subject: [PATCH 06/79] test: fix state-transition tests for EpochCache changes --- .../state-transition/src/cache/epochCache.ts | 15 +++++-- .../state-transition/src/cache/stateCache.ts | 2 +- packages/state-transition/test/perf/util.ts | 4 ++ .../perf/util/loadState/loadState.test.ts | 6 +-- .../test/perf/util/shufflings.test.ts | 10 ++--- .../test/unit/cachedBeaconState.test.ts | 42 +++++++------------ .../test/unit/upgradeState.test.ts | 2 + .../test/unit/util/cachedBeaconState.test.ts | 1 + .../test/utils/mockShufflingCache.ts | 35 ++++++++++++++++ packages/state-transition/test/utils/state.ts | 3 ++ 10 files changed, 76 insertions(+), 44 deletions(-) create mode 100644 packages/state-transition/test/utils/mockShufflingCache.ts diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 4c4a7d7b96f4..8953cc9540d2 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -576,9 +576,14 @@ export class EpochCache { this.previousShuffling = this.currentShuffling; this.currentDecisionRoot = this.nextDecisionRoot; - this.currentShuffling = this.shufflingCache - ? this.shufflingCache.getOrBuildSync(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices) - : computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); + if (this.nextShuffling) { + // was already pulled by the api or another method on EpochCache + this.currentShuffling = this.nextShuffling; + } else { + this.currentShuffling = this.shufflingCache + ? this.shufflingCache.getOrBuildSync(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices) + : computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); + } const currentEpoch = this.nextEpoch; this.nextShuffling = null; @@ -1009,12 +1014,14 @@ export class EpochCacheError extends LodestarError {} export function createEmptyEpochCacheImmutableData( chainConfig: ChainConfig, - state: Pick + state: Pick, + shufflingCache?: IShufflingCache ): EpochCacheImmutableData { return { config: createBeaconConfig(chainConfig, state.genesisValidatorsRoot), // This is a test state, there's no need to have a global shared cache of keys pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + shufflingCache, }; } diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index 232b4a56134b..0c932391c92b 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -164,7 +164,7 @@ export function createCachedBeaconState( export function loadCachedBeaconState( cachedSeedState: T, stateBytes: Uint8Array, - shufflingCache: IShufflingCache, + shufflingCache: IShufflingCache, // should not be optional because this is used during node operation opts?: EpochCacheOpts, seedValidatorsBytes?: Uint8Array ): T { diff --git a/packages/state-transition/test/perf/util.ts b/packages/state-transition/test/perf/util.ts index 4b2a7da4a50e..6e607cdfdeb6 100644 --- a/packages/state-transition/test/perf/util.ts +++ b/packages/state-transition/test/perf/util.ts @@ -32,6 +32,7 @@ import {interopPubkeysCached} from "../utils/interop.js"; import {getNextSyncCommittee} from "../../src/util/syncCommittee.js"; import {getEffectiveBalanceIncrements} from "../../src/cache/effectiveBalanceIncrements.js"; import {processSlots} from "../../src/index.js"; +import {MockShufflingCache} from "../utils/mockShufflingCache.js"; let phase0State: BeaconStatePhase0 | null = null; let phase0CachedState23637: CachedBeaconStatePhase0 | null = null; @@ -129,6 +130,7 @@ export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean config: createBeaconConfig(config, state.genesisValidatorsRoot), pubkey2index, index2pubkey, + shufflingCache: new MockShufflingCache(), }); const currentEpoch = computeEpochAtSlot(state.slot - 1); @@ -234,6 +236,7 @@ export function generatePerfTestCachedStateAltair(opts?: { config: createBeaconConfig(altairConfig, state.genesisValidatorsRoot), pubkey2index, index2pubkey, + shufflingCache: new MockShufflingCache(), }); } if (!altairCachedState23638) { @@ -437,6 +440,7 @@ export function generateTestCachedBeaconStateOnlyValidators({ config: createBeaconConfig(config, state.genesisValidatorsRoot), pubkey2index, index2pubkey, + shufflingCache: new MockShufflingCache(), }, {skipSyncPubkeys: true} ); diff --git a/packages/state-transition/test/perf/util/loadState/loadState.test.ts b/packages/state-transition/test/perf/util/loadState/loadState.test.ts index a8a1b1399dc5..25b43e242d02 100644 --- a/packages/state-transition/test/perf/util/loadState/loadState.test.ts +++ b/packages/state-transition/test/perf/util/loadState/loadState.test.ts @@ -79,17 +79,15 @@ describe("loadState", function () { pubkey2index.set(pubkey, validatorIndex); index2pubkey[validatorIndex] = PublicKey.fromBytes(pubkey); } - // skip computimg shuffling in performance test because in reality we have a ShufflingCache - // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - const shufflingGetter = () => seedState.epochCtx.currentShuffling; createCachedBeaconState( migratedState, { config: seedState.config, pubkey2index, index2pubkey, + shufflingCache: seedState.epochCtx.shufflingCache, }, - {skipSyncPubkeys: true, skipSyncCommitteeCache: true, shufflingGetter} + {skipSyncPubkeys: true, skipSyncCommitteeCache: true} ); }, }); diff --git a/packages/state-transition/test/perf/util/shufflings.test.ts b/packages/state-transition/test/perf/util/shufflings.test.ts index 96c7878a46ac..5077f0ef6525 100644 --- a/packages/state-transition/test/perf/util/shufflings.test.ts +++ b/packages/state-transition/test/perf/util/shufflings.test.ts @@ -27,8 +27,8 @@ describe("epoch shufflings", () => { itBench({ id: `computeProposers - vc ${numValidators}`, fn: () => { - const epochSeed = getSeed(state, state.epochCtx.nextShuffling.epoch, DOMAIN_BEACON_PROPOSER); - computeProposers(epochSeed, state.epochCtx.nextShuffling, state.epochCtx.effectiveBalanceIncrements); + const epochSeed = getSeed(state, state.epochCtx.epoch, DOMAIN_BEACON_PROPOSER); + computeProposers(epochSeed, state.epochCtx.currentShuffling, state.epochCtx.effectiveBalanceIncrements); }, }); @@ -43,11 +43,7 @@ describe("epoch shufflings", () => { itBench({ id: `getNextSyncCommittee - vc ${numValidators}`, fn: () => { - getNextSyncCommittee( - state, - state.epochCtx.nextShuffling.activeIndices, - state.epochCtx.effectiveBalanceIncrements - ); + getNextSyncCommittee(state, state.epochCtx.nextActiveIndices, state.epochCtx.effectiveBalanceIncrements); }, }); }); diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 2891cd3e6216..8d944b2a4ad8 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -1,5 +1,5 @@ import {describe, it, expect} from "vitest"; -import {Epoch, ssz, RootHex} from "@lodestar/types"; +import {ssz} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {config as defaultConfig} from "@lodestar/config/default"; import {createBeaconConfig} from "@lodestar/config"; @@ -8,7 +8,8 @@ import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; import {interopPubkeysCached} from "../utils/interop.js"; import {modifyStateSameValidator, newStateWithValidators} from "../utils/capella.js"; -import {EpochShuffling, getShufflingDecisionBlock} from "../../src/util/epochShuffling.js"; +import {MockShufflingCache} from "../utils/mockShufflingCache.js"; +import {IShufflingCache} from "../../src/index.js"; describe("CachedBeaconState", () => { it("Clone and mutate", () => { @@ -65,6 +66,7 @@ describe("CachedBeaconState", () => { config, pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + shufflingCache: new MockShufflingCache(), }, {skipSyncCommitteeCache: true} ); @@ -129,42 +131,26 @@ describe("CachedBeaconState", () => { // confirm loadState() result const stateBytes = state.serialize(); - const newCachedState = loadCachedBeaconState(seedState, stateBytes, {skipSyncCommitteeCache: true}); + const newCachedState = loadCachedBeaconState( + seedState, + stateBytes, + seedState.epochCtx.shufflingCache as IShufflingCache, + { + skipSyncCommitteeCache: true, + } + ); const newStateBytes = newCachedState.serialize(); expect(newStateBytes).toEqual(stateBytes); expect(newCachedState.hashTreeRoot()).toEqual(state.hashTreeRoot()); - const shufflingGetter = (shufflingEpoch: Epoch, dependentRoot: RootHex): EpochShuffling | null => { - if ( - shufflingEpoch === seedState.epochCtx.epoch - 1 && - dependentRoot === getShufflingDecisionBlock(seedState, shufflingEpoch) - ) { - return seedState.epochCtx.previousShuffling; - } - - if ( - shufflingEpoch === seedState.epochCtx.epoch && - dependentRoot === getShufflingDecisionBlock(seedState, shufflingEpoch) - ) { - return seedState.epochCtx.currentShuffling; - } - - if ( - shufflingEpoch === seedState.epochCtx.epoch + 1 && - dependentRoot === getShufflingDecisionBlock(seedState, shufflingEpoch) - ) { - return seedState.epochCtx.nextShuffling; - } - - return null; - }; const cachedState = createCachedBeaconState( state, { config, pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + shufflingCache: seedState.epochCtx.shufflingCache, }, - {skipSyncCommitteeCache: true, shufflingGetter} + {skipSyncCommitteeCache: true} ); // validatorCountDelta < 0 is unrealistic and shuffling computation results in a different result if (validatorCountDelta >= 0) { diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index 2ea8eef182ac..c1b7cbdd646f 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -7,6 +7,7 @@ import {config as chainConfig} from "@lodestar/config/default"; import {upgradeStateToDeneb} from "../../src/slot/upgradeStateToDeneb.js"; import {createCachedBeaconState} from "../../src/cache/stateCache.js"; import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; +import {MockShufflingCache} from "../utils/mockShufflingCache.js"; describe("upgradeState", () => { it("upgradeStateToDeneb", () => { @@ -18,6 +19,7 @@ describe("upgradeState", () => { config: createBeaconConfig(config, capellaState.genesisValidatorsRoot), pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + shufflingCache: new MockShufflingCache(), }, {skipSyncCommitteeCache: true} ); diff --git a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts index 654e0752adb8..f3fb9734718c 100644 --- a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts @@ -12,6 +12,7 @@ describe("CachedBeaconState", () => { config: createBeaconConfig(config, emptyState.genesisValidatorsRoot), pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + shufflingCache: undefined, }); }); }); diff --git a/packages/state-transition/test/utils/mockShufflingCache.ts b/packages/state-transition/test/utils/mockShufflingCache.ts new file mode 100644 index 000000000000..04c11ec5bf26 --- /dev/null +++ b/packages/state-transition/test/utils/mockShufflingCache.ts @@ -0,0 +1,35 @@ +import {MapDef} from "@lodestar/utils"; +import {Epoch, RootHex} from "@lodestar/types"; +import {BeaconStateAllForks} from "../../src/types.js"; +import {EpochShuffling, IShufflingCache, computeEpochShuffling} from "../../src/util/epochShuffling.js"; + +export class MockShufflingCache implements IShufflingCache { + private readonly itemsByDecisionRootByEpoch: MapDef> = new MapDef( + () => new Map() + ); + build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): void { + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + this.set(shuffling, decisionRoot); + } + + getOrBuildSync( + epoch: number, + decisionRoot: string, + state: BeaconStateAllForks, + activeIndices: number[] + ): EpochShuffling { + const shuffling = this.getSync(epoch, decisionRoot); + if (!shuffling) { + this.build(epoch, decisionRoot, state, activeIndices); + } + return this.getSync(epoch, decisionRoot) as EpochShuffling; + } + + getSync(epoch: number, decisionRoot: string): EpochShuffling | null { + return this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot) ?? null; + } + + set(shuffling: EpochShuffling, decisionRoot: string): void { + this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch).set(decisionRoot, shuffling); + } +} diff --git a/packages/state-transition/test/utils/state.ts b/packages/state-transition/test/utils/state.ts index 29a1f98b5562..596fcc3afdfb 100644 --- a/packages/state-transition/test/utils/state.ts +++ b/packages/state-transition/test/utils/state.ts @@ -22,6 +22,7 @@ import { } from "../../src/index.js"; import {BeaconStateCache} from "../../src/cache/stateCache.js"; import {EpochCacheOpts} from "../../src/cache/epochCache.js"; +import {MockShufflingCache} from "./mockShufflingCache.js"; /** * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed. @@ -94,6 +95,7 @@ export function generateCachedState( // This is a test state, there's no need to have a global shared cache of keys pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + shufflingCache: undefined, }); } @@ -109,6 +111,7 @@ export function createCachedBeaconStateTest( // This is a test state, there's no need to have a global shared cache of keys pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + shufflingCache: new MockShufflingCache(), }, opts ); From ff2d52065874a5ff3c88f2ab16ee7d354f2004a9 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 4 Jul 2024 23:59:09 +0000 Subject: [PATCH 07/79] feat: Pass shufflingCache to EpochCache at startup --- packages/beacon-node/src/chain/chain.ts | 13 +++++++++++++ .../beacon-node/src/node/utils/interop/state.ts | 1 + packages/beacon-node/src/node/utils/state.ts | 3 +++ 3 files changed, 17 insertions(+) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 5c1c090f6257..26679a4aa51a 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -254,7 +254,20 @@ export class BeaconChain implements IBeaconChain { config, pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + shufflingCache: this.shufflingCache, }); + // If shufflingCache was not passed to EpochCache, add it now and cache shuffling from context. Happens in most + // situations including checkpoint sync + if (!cachedState.epochCtx.shufflingCache) { + cachedState.epochCtx.shufflingCache = this.shufflingCache; + this.shufflingCache.set(cachedState.epochCtx.currentShuffling, cachedState.epochCtx.currentDecisionRoot); + if (cachedState.epochCtx.currentShuffling !== cachedState.epochCtx.previousShuffling) { + this.shufflingCache.set(cachedState.epochCtx.previousShuffling, cachedState.epochCtx.previousDecisionRoot); + } + if (cachedState.epochCtx.nextShuffling) { + this.shufflingCache.set(cachedState.epochCtx.nextShuffling, cachedState.epochCtx.nextDecisionRoot); + } + } // Persist single global instance of state caches this.pubkey2index = cachedState.epochCtx.pubkey2index; diff --git a/packages/beacon-node/src/node/utils/interop/state.ts b/packages/beacon-node/src/node/utils/interop/state.ts index 6528bd392bc7..fe26afef2013 100644 --- a/packages/beacon-node/src/node/utils/interop/state.ts +++ b/packages/beacon-node/src/node/utils/interop/state.ts @@ -20,6 +20,7 @@ export type InteropStateOpts = { withEth1Credentials?: boolean; }; +// TODO: (@matthewkeil) - Only used by initDevState. Consider combining into that function export function getInteropState( config: ChainForkConfig, { diff --git a/packages/beacon-node/src/node/utils/state.ts b/packages/beacon-node/src/node/utils/state.ts index 25bd77c82274..05da7042eef4 100644 --- a/packages/beacon-node/src/node/utils/state.ts +++ b/packages/beacon-node/src/node/utils/state.ts @@ -5,6 +5,9 @@ import {IBeaconDb} from "../../db/index.js"; import {interopDeposits} from "./interop/deposits.js"; import {getInteropState, InteropStateOpts} from "./interop/state.js"; +/** + * Builds state for `dev` command, for sim testing and some other tests + */ export function initDevState( config: ChainForkConfig, validatorCount: number, From 53ed72e595655af80e66ad5047a469f46241d625 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 9 Jul 2024 13:56:34 +0000 Subject: [PATCH 08/79] test: fix slot off by one for decision root in perf test --- packages/state-transition/test/perf/epoch/epochAltair.test.ts | 2 ++ packages/state-transition/test/perf/epoch/epochCapella.test.ts | 2 ++ packages/state-transition/test/perf/epoch/epochPhase0.test.ts | 2 ++ 3 files changed, 6 insertions(+) diff --git a/packages/state-transition/test/perf/epoch/epochAltair.test.ts b/packages/state-transition/test/perf/epoch/epochAltair.test.ts index 273353d8632b..d143c03eed02 100644 --- a/packages/state-transition/test/perf/epoch/epochAltair.test.ts +++ b/packages/state-transition/test/perf/epoch/epochAltair.test.ts @@ -46,6 +46,7 @@ describe(`altair processEpoch - ${stateId}`, () => { fn: (state) => { const cache = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStateAltair, cache); + state.slot++; state.epochCtx.afterProcessEpoch(state, cache); // Simulate root computation through the next block to account for changes // 74184 hash64 ops - 92.730 ms @@ -182,6 +183,7 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue const state = stateOg.value.clone(); const cacheAfter = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStateAltair, cacheAfter); + state.slot++; return {state, cache: cacheAfter}; }, beforeEach: ({state, cache}) => ({state: state.clone(), cache}), diff --git a/packages/state-transition/test/perf/epoch/epochCapella.test.ts b/packages/state-transition/test/perf/epoch/epochCapella.test.ts index eeaf8bfc5400..45372a11c796 100644 --- a/packages/state-transition/test/perf/epoch/epochCapella.test.ts +++ b/packages/state-transition/test/perf/epoch/epochCapella.test.ts @@ -46,6 +46,7 @@ describe(`capella processEpoch - ${stateId}`, () => { fn: (state) => { const cache = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStateCapella, cache); + state.slot++; state.epochCtx.afterProcessEpoch(state, cache); // Simulate root computation through the next block to account for changes // 74184 hash64 ops - 92.730 ms @@ -154,6 +155,7 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue const state = stateOg.value.clone(); const cacheAfter = beforeProcessEpoch(state); processEpoch(fork, state, cacheAfter); + state.slot++; return {state, cache: cacheAfter}; }, beforeEach: ({state, cache}) => ({state: state.clone(), cache}), diff --git a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts index 4e43634b1669..abcb6a9fba8c 100644 --- a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts @@ -43,6 +43,7 @@ describe(`phase0 processEpoch - ${stateId}`, () => { fn: (state) => { const cache = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStatePhase0, cache); + state.slot++; state.epochCtx.afterProcessEpoch(state, cache); // Simulate root computation through the next block to account for changes state.hashTreeRoot(); @@ -157,6 +158,7 @@ function benchmarkPhase0EpochSteps(stateOg: LazyValue const state = stateOg.value.clone(); const cacheAfter = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStatePhase0, cacheAfter); + state.slot++; return {state, cache: cacheAfter}; }, beforeEach: ({state, cache}) => ({state: state.clone(), cache}), From 8faee3f68aed342baeb66965f064a7857e025622 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 9 Jul 2024 13:51:30 -0400 Subject: [PATCH 09/79] chore: use ?. syntax --- packages/state-transition/src/cache/epochCache.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 8953cc9540d2..34d15b910f63 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -580,9 +580,9 @@ export class EpochCache { // was already pulled by the api or another method on EpochCache this.currentShuffling = this.nextShuffling; } else { - this.currentShuffling = this.shufflingCache - ? this.shufflingCache.getOrBuildSync(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices) - : computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); + this.currentShuffling = + this.shufflingCache?.getOrBuildSync(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices) ?? + computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); } const currentEpoch = this.nextEpoch; @@ -895,9 +895,8 @@ export class EpochCache { return this.currentShuffling; case this.nextEpoch: if (!this.nextShuffling) { - this.nextShuffling = this.shufflingCache - ? this.shufflingCache.getSync(this.nextEpoch, this.getDecisionRoot(this.nextEpoch)) - : null; + this.nextShuffling = + this.shufflingCache?.getSync(this.nextEpoch, this.getDecisionRoot(this.nextEpoch)) ?? null; } return this.nextShuffling; default: From d69c8b183f2c17f3c2020e83701b65510765e738 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 9 Jul 2024 14:54:38 -0400 Subject: [PATCH 10/79] chore: refactoring --- packages/beacon-node/src/chain/chain.ts | 13 +- .../beacon-node/src/chain/shufflingCache.ts | 147 ++++++++---------- .../src/chain/validation/attestation.ts | 2 +- .../test/unit/chain/shufflingCache.test.ts | 23 +-- .../test/utils/validationData/attestation.ts | 8 +- .../state-transition/src/cache/epochCache.ts | 5 +- .../src/util/epochShuffling.ts | 30 +++- .../test/utils/mockShufflingCache.ts | 33 ++-- 8 files changed, 145 insertions(+), 116 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 26679a4aa51a..eaa72716cf74 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -876,9 +876,6 @@ export class BeaconChain implements IBeaconChain { attHeadBlock: ProtoBlock, regenCaller: RegenCaller ): Promise { - // this is to prevent multiple calls to get shuffling for the same epoch and dependent root - // any subsequent calls of the same epoch and dependent root will wait for this promise to resolve - this.shufflingCache.insertPromise(attEpoch, shufflingDependentRoot); const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); let state: CachedBeaconStateAllForks; @@ -903,8 +900,14 @@ export class BeaconChain implements IBeaconChain { state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller); } - // resolve the promise to unblock other calls of the same epoch and dependent root - return this.shufflingCache.get(attEpoch, getShufflingDecisionBlock(state, attEpoch)); + const decisionRoot = getShufflingDecisionBlock(state, attEpoch); + const shuffling = await this.shufflingCache.get(attEpoch, decisionRoot); + if (!shuffling) { + // This will be essentially unreachable considering regen should build the shuffling for this epoch + // but need to handle anyhow + throw Error(`UNREACHABLE: Shuffling not found for attestation epoch ${attEpoch} decisionRoot ${decisionRoot}`); + } + return shuffling; } /** diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 8753df94698a..b53157b4bd39 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -68,42 +68,12 @@ export class ShufflingCache implements IShufflingCache { this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS; } - /** - * Insert a promise to make sure we don't regen state for the same shuffling. - * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. - */ - insertPromise(shufflingEpoch: Epoch, decisionRootHex: RootHex): void { - const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) - .flatMap((innerMap) => Array.from(innerMap.values())) - .filter((item) => isPromiseCacheItem(item)).length; - if (promiseCount >= MAX_PROMISES) { - throw new Error( - `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRootHex}` - ); - } - let resolveFn: ((shuffling: EpochShuffling) => void) | null = null; - const promise = new Promise((resolve) => { - resolveFn = resolve; - }); - if (resolveFn === null) { - throw new Error("Promise Constructor was not executed immediately"); - } - - const cacheItem: PromiseCacheItem = { - type: CacheItemType.promise, - promise, - resolveFn, - }; - this.add(shufflingEpoch, decisionRootHex, cacheItem); - this.metrics?.shufflingCache.insertPromiseCount.inc(); - } - /** * Most of the time, this should return a shuffling immediately. * If there's a promise, it means we are computing the same shuffling, so we wait for the promise to resolve. * Return null if we don't have a shuffling for this epoch and dependentRootHex. */ - async getShufflingOrNull(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { + async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); if (cacheItem === undefined) { return null; @@ -117,18 +87,6 @@ export class ShufflingCache implements IShufflingCache { } } - async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { - const item = await this.getShufflingOrNull(shufflingEpoch, decisionRootHex); - if (!item) { - throw new ShufflingCacheError({ - code: ShufflingCacheErrorCode.NO_SHUFFLING_FOUND, - epoch: shufflingEpoch, - decisionRoot: decisionRootHex, - }); - } - return item; - } - /** * Same to getShufflingOrNull() function but synchronous. */ @@ -146,59 +104,82 @@ export class ShufflingCache implements IShufflingCache { return null; } - getOrBuildSync( + buildSync(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): EpochShuffling { + const cachedShuffling = this.getSync(epoch, decisionRoot); + if (cachedShuffling) { + return cachedShuffling; + } + + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + this.set(shuffling, decisionRoot); + return shuffling; + } + + async build( epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[] - ): EpochShuffling { - const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); - if (cacheItem && isShufflingCacheItem(cacheItem)) { - // this.metrics?.shufflingCache.cacheHitInEpochTransition(); - return cacheItem.shuffling; + ): Promise { + const cachedShuffling = await this.get(epoch, decisionRoot); + if (cachedShuffling) { + return cachedShuffling; } - // if (cacheItem) { - // this.metrics?.shufflingCache.cacheMissInEpochTransition(); - // } else { - // this.metrics?.shufflingCache.shufflingPromiseNotResolvedInEpochTransition(); - // } - const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.add(epoch, decisionRoot, { - type: CacheItemType.shuffling, - shuffling, - }); - return shuffling; - } - build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): void { - let resolveFn: (shuffling: EpochShuffling) => void = () => {}; - this.add(epoch, decisionRoot, { - type: CacheItemType.promise, - resolveFn, - promise: new Promise((resolve) => { - resolveFn = resolve; - }), - }); + // this is to prevent multiple calls to get shuffling for the same epoch and dependent root + // any subsequent calls of the same epoch and dependent root will wait for this promise to resolve + const cacheItem = this.insertPromise(epoch, decisionRoot); + // TODO: replace this call with a worker setTimeout(() => { const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.add(epoch, decisionRoot, { - type: CacheItemType.shuffling, - shuffling, - }); - }, 100); + this.set(shuffling, decisionRoot); + }, 5); + return cacheItem.promise; } set(shuffling: EpochShuffling, decisionRoot: string): void { - const cacheItem: ShufflingCacheItem = { - shuffling, - type: CacheItemType.shuffling, - }; - this.add(shuffling.epoch, decisionRoot, cacheItem); + const items = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch); + // if a pending shuffling promise exists, resolve it + const item = items.get(decisionRoot); + if (item !== undefined && isPromiseCacheItem(item)) { + item.resolveFn(shuffling); + } + // set the shuffling + items.set(decisionRoot, {type: CacheItemType.shuffling, shuffling}); + // prune the cache + pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs); } - private add(shufflingEpoch: Epoch, decisionBlock: RootHex, cacheItem: CacheItem): void { - this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionBlock, cacheItem); - pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs); + /** + * Insert a promise to make sure we don't regen state for the same shuffling. + * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. + */ + private insertPromise(shufflingEpoch: Epoch, decisionRoot: RootHex): PromiseCacheItem { + const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) + .flatMap((innerMap) => Array.from(innerMap.values())) + .filter((item) => isPromiseCacheItem(item)).length; + if (promiseCount >= MAX_PROMISES) { + throw new Error( + `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRoot}` + ); + } + let resolveFn: ((shuffling: EpochShuffling) => void) | null = null; + const promise = new Promise((resolve) => { + resolveFn = resolve; + }); + if (resolveFn === null) { + throw new Error("Promise Constructor was not executed immediately"); + } + + this.metrics?.shufflingCache.insertPromiseCount.inc(); + + const cacheItem: PromiseCacheItem = { + type: CacheItemType.promise, + promise, + resolveFn, + }; + this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionRoot, cacheItem); + return cacheItem; } } diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 389abd8a7c03..fc39534b45e6 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -584,7 +584,7 @@ export async function getShufflingForAttestationVerification( const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); const shufflingDependentRoot = getShufflingDependentRoot(chain.forkChoice, attEpoch, blockEpoch, attHeadBlock); - const shuffling = await chain.shufflingCache.getShufflingOrNull(attEpoch, shufflingDependentRoot); + const shuffling = await chain.shufflingCache.get(attEpoch, shufflingDependentRoot); if (shuffling) { // most of the time, we should get the shuffling from cache chain.metrics?.gossipAttestation.shufflingCacheHit.inc({caller: regenCaller}); diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index 6295a993c072..7323691ff14e 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -10,11 +10,13 @@ describe("ShufflingCache", function () { const stateSlot = 100; const state = generateTestCachedBeaconStateOnlyValidators({vc, slot: stateSlot}); const currentEpoch = state.epochCtx.currentShuffling.epoch; + const activeIndices = Array.from(state.epochCtx.currentShuffling.activeIndices); let shufflingCache: ShufflingCache; beforeEach(() => { shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - shufflingCache.processState(state, currentEpoch); + const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); + shufflingCache.buildSync(currentEpoch, decisionRoot, state, activeIndices); }); it("should get shuffling from cache", async function () { @@ -26,29 +28,30 @@ describe("ShufflingCache", function () { const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert promises at the same epoch does not prune the cache - shufflingCache.insertPromise(currentEpoch, "0x00"); + shufflingCache["insertPromise"](currentEpoch, "0x00"); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert shufflings at other epochs does prune the cache - shufflingCache.processState(state, currentEpoch + 1); + shufflingCache.buildSync(currentEpoch + 1, decisionRoot, state, activeIndices); // the current shuffling is not available anymore expect(await shufflingCache.get(currentEpoch, decisionRoot)).toBeNull(); }); it("should return shuffling from promise", async function () { + const nextEpoch = currentEpoch + 1; const nextDecisionRoot = getShufflingDecisionBlock(state, currentEpoch + 1); - shufflingCache.insertPromise(currentEpoch + 1, nextDecisionRoot); - const shufflingRequest0 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - const shufflingRequest1 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - shufflingCache.processState(state, currentEpoch + 1); + shufflingCache["insertPromise"](nextEpoch, nextDecisionRoot); + const shufflingRequest0 = shufflingCache.get(nextEpoch, nextDecisionRoot); + const shufflingRequest1 = shufflingCache.get(nextEpoch, nextDecisionRoot); + shufflingCache.buildSync(nextEpoch, nextDecisionRoot, state, activeIndices); expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling); expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling); }); it("should support up to 2 promises at a time", async function () { // insert 2 promises at the same epoch - shufflingCache.insertPromise(currentEpoch, "0x00"); - shufflingCache.insertPromise(currentEpoch, "0x01"); + shufflingCache["insertPromise"](currentEpoch, "0x00"); + shufflingCache["insertPromise"](currentEpoch, "0x01"); // inserting other promise should throw error - expect(() => shufflingCache.insertPromise(currentEpoch, "0x02")).toThrow(); + expect(() => shufflingCache["insertPromise"](currentEpoch, "0x02")).toThrow(); }); }); diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index c33d942dabc5..a94a7857bd3e 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -3,6 +3,7 @@ import { computeEpochAtSlot, computeSigningRoot, computeStartSlotAtEpoch, + getActiveValidatorIndices, getShufflingDecisionBlock, } from "@lodestar/state-transition"; import {ProtoBlock, IForkChoice, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; @@ -82,9 +83,12 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { }; const shufflingCache = new ShufflingCache(); - shufflingCache.processState(state, state.epochCtx.currentShuffling.epoch); - shufflingCache.processState(state, state.epochCtx.nextShuffling.epoch); const dependentRoot = getShufflingDecisionBlock(state, state.epochCtx.currentShuffling.epoch); + shufflingCache.set(state.epochCtx.currentShuffling, dependentRoot); + + const nextEpoch = state.epochCtx.currentShuffling.epoch + 1; + const nextDependentRoot = getShufflingDecisionBlock(state, nextEpoch); + shufflingCache.buildSync(nextEpoch, nextDependentRoot, state, getActiveValidatorIndices(state, nextEpoch)); const forkChoice = { getBlock: (root) => { diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 34d15b910f63..b388b3fbace3 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -581,7 +581,7 @@ export class EpochCache { this.currentShuffling = this.nextShuffling; } else { this.currentShuffling = - this.shufflingCache?.getOrBuildSync(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices) ?? + this.shufflingCache?.buildSync(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices) ?? computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); } @@ -589,7 +589,8 @@ export class EpochCache { this.nextShuffling = null; this.nextDecisionRoot = getShufflingDecisionBlock(state, currentEpoch + 1); this.nextActiveIndices = epochTransitionCache.nextEpochShufflingActiveValidatorIndices; - this.shufflingCache?.build(currentEpoch + 1, this.nextDecisionRoot, state, this.nextActiveIndices); + // TODO move this out to beacon node + void this.shufflingCache?.build(currentEpoch + 1, this.nextDecisionRoot, state, this.nextActiveIndices); // Roll current proposers into previous proposers for metrics this.proposersPrevEpoch = this.proposers; diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 5033262216d7..5175e40ed435 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -14,14 +14,40 @@ import {computeStartSlotAtEpoch} from "./epoch.js"; import {getBlockRootAtSlot} from "./blockRoot.js"; export interface IShufflingCache { + /** + * Will synchronously get a shuffling if it is available or will return null if not. + */ getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null; - getOrBuildSync( + + /** + * Will synchronously get a shuffling if it is available. + * + * If a shuffling is not immediately available, a shuffling will be calculated. + * + * NOTE: this may recalculate an already in-progress shuffling. + */ + buildSync( epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[] ): EpochShuffling; - build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void; + + /** + * Will immediately return a shuffling if it is available, or a promise to an in-progress shuffling calculation. + * + * If neither is available, a shuffling will be calculated. + */ + build( + epoch: Epoch, + decisionRoot: RootHex, + state: BeaconStateAllForks, + activeIndices: ValidatorIndex[] + ): Promise; + + /** + * Set a shuffling for a given epoch and decisionRoot. + */ set(shuffling: EpochShuffling, decisionRoot: RootHex): void; } diff --git a/packages/state-transition/test/utils/mockShufflingCache.ts b/packages/state-transition/test/utils/mockShufflingCache.ts index 04c11ec5bf26..7e600f5f1f4d 100644 --- a/packages/state-transition/test/utils/mockShufflingCache.ts +++ b/packages/state-transition/test/utils/mockShufflingCache.ts @@ -7,22 +7,33 @@ export class MockShufflingCache implements IShufflingCache { private readonly itemsByDecisionRootByEpoch: MapDef> = new MapDef( () => new Map() ); - build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): void { - const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.set(shuffling, decisionRoot); - } - - getOrBuildSync( + async build( epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[] - ): EpochShuffling { - const shuffling = this.getSync(epoch, decisionRoot); - if (!shuffling) { - this.build(epoch, decisionRoot, state, activeIndices); + ): Promise { + const cachedShuffling = this.getSync(epoch, decisionRoot); + if (cachedShuffling) { + return cachedShuffling; } - return this.getSync(epoch, decisionRoot) as EpochShuffling; + return new Promise((resolve) => { + setTimeout(() => { + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + this.set(shuffling, decisionRoot); + resolve(shuffling); + }); + }); + } + + buildSync(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): EpochShuffling { + const cachedShuffling = this.getSync(epoch, decisionRoot); + if (cachedShuffling) { + return cachedShuffling; + } + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + this.set(shuffling, decisionRoot); + return shuffling; } getSync(epoch: number, decisionRoot: string): EpochShuffling | null { From cfa97ad6178dab197627372d64e9063368196a4a Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 18:54:26 -0100 Subject: [PATCH 11/79] feat: add comments and clean up afterProcessEpoch --- .../state-transition/src/cache/epochCache.ts | 83 +++++++++++++------ 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index b388b3fbace3..1c6caee47168 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -562,7 +562,12 @@ export class EpochCache { /** * Called to re-use information, such as the shuffling of the next epoch, after transitioning into a - * new epoch. + * new epoch. Also handles pre-computation of values that may change during the upcoming epoch and + * that get used in the following epoch transition. Often those pre-computations are not used by the + * chain but are courtesy values that are served via the API for epoch look ahead of duties. + * + * Steps for afterProcessEpoch + * 1) update previous/current/next values of cached items */ afterProcessEpoch( state: BeaconStateAllForks, @@ -572,38 +577,71 @@ export class EpochCache { nextEpochTotalActiveBalanceByIncrement: number; } ): void { + // Advance time units + // state.slot is advanced right before calling this function + // ``` + // postState.slot++; + // afterProcessEpoch(postState, epochTransitionCache); + // ``` + + // After updating the "current" epoch the "cached epoch" will be out of sync with the clock + // epoch until the clock catches up (should be less than a second or so). Because this function + // and the remainder of the state-transition is all synchronous there is no chance that this value + // can be read incorrectly, however, if the synchronous-only paradigm changes it must be taken + // into account + this.epoch = this.nextEpoch; + // TODO: why was this what was used @twoeths? and not a simple increment like above if all the other + // stuff in this function was swapped curr->prev and next->curr? + // this.epoch = computeEpochAtSlot(state.slot); + + // Move current values to previous values as they are no longer for the current epoch this.previousDecisionRoot = this.currentDecisionRoot; this.previousShuffling = this.currentShuffling; + // Roll current proposers into previous proposers for metrics + this.proposersPrevEpoch = this.proposers; + // Move next values to be the current values of the upcoming epoch after epoch transition finishes this.currentDecisionRoot = this.nextDecisionRoot; if (this.nextShuffling) { // was already pulled by the api or another method on EpochCache this.currentShuffling = this.nextShuffling; } else { this.currentShuffling = - this.shufflingCache?.buildSync(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices) ?? - computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); + this.shufflingCache?.getOrBuildSync( + this.epoch, + this.currentDecisionRoot, + state, + // have to use the "nextActiveIndices" that were saved in the last transition here to calculate + // the upcoming shuffling if it is not already built (similar condition to the below computation) + this.nextActiveIndices + ) ?? + // allow for this case during testing where the ShufflingCache is not present, may affect perf testing + // so should be taken into account when structuring tests. Should not affect unit or other tests though + computeEpochShuffling(state, this.nextActiveIndices, this.epoch); } + this.proposers = computeProposers( + getSeed(state, this.epoch, DOMAIN_BEACON_PROPOSER), + this.currentShuffling, + this.effectiveBalanceIncrements + ); - const currentEpoch = this.nextEpoch; - this.nextShuffling = null; - this.nextDecisionRoot = getShufflingDecisionBlock(state, currentEpoch + 1); + /** + * Calculate look-ahead values for n+2 (will be n+1 after transition finishes) + */ + this.nextDecisionRoot = getShufflingDecisionBlock(state, this.nextEpoch); this.nextActiveIndices = epochTransitionCache.nextEpochShufflingActiveValidatorIndices; - // TODO move this out to beacon node - void this.shufflingCache?.build(currentEpoch + 1, this.nextDecisionRoot, state, this.nextActiveIndices); - - // Roll current proposers into previous proposers for metrics - this.proposersPrevEpoch = this.proposers; - - const currentProposerSeed = getSeed(state, this.currentShuffling.epoch, DOMAIN_BEACON_PROPOSER); - this.proposers = computeProposers(currentProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements); - + if (this.shufflingCache) { + this.nextShuffling = null; + this.shufflingCache?.build(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices); + } else { + this.nextShuffling = computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); + } // Only pre-compute the seed since it's very cheap. Do the expensive computeProposers() call only on demand. - this.proposersNextEpoch = {computed: false, seed: getSeed(state, currentEpoch + 1, DOMAIN_BEACON_PROPOSER)}; + this.proposersNextEpoch = {computed: false, seed: getSeed(state, this.nextEpoch, DOMAIN_BEACON_PROPOSER)}; // TODO: DEDUPLICATE from createEpochCache // - // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute everytime the + // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute every time the // active validator indices set changes in size. Validators change active status only when: // - validator.activation_epoch is set. Only changes in process_registry_updates() if validator can be activated. If // the value changes it will be set to `epoch + 1 + MAX_SEED_LOOKAHEAD`. @@ -625,14 +663,14 @@ export class EpochCache { ); // Maybe advance exitQueueEpoch at the end of the epoch if there haven't been any exists for a while - const exitQueueEpoch = computeActivationExitEpoch(currentEpoch); + const exitQueueEpoch = computeActivationExitEpoch(this.epoch); if (exitQueueEpoch > this.exitQueueEpoch) { this.exitQueueEpoch = exitQueueEpoch; this.exitQueueChurn = 0; } this.totalActiveBalanceIncrements = epochTransitionCache.nextEpochTotalActiveBalanceByIncrement; - if (currentEpoch >= this.config.ALTAIR_FORK_EPOCH) { + if (this.epoch >= this.config.ALTAIR_FORK_EPOCH) { this.syncParticipantReward = computeSyncParticipantReward(this.totalActiveBalanceIncrements); this.syncProposerReward = Math.floor(this.syncParticipantReward * PROPOSER_WEIGHT_FACTOR); this.baseRewardPerIncrement = computeBaseRewardPerIncrement(this.totalActiveBalanceIncrements); @@ -641,13 +679,6 @@ export class EpochCache { this.previousTargetUnslashedBalanceIncrements = this.currentTargetUnslashedBalanceIncrements; this.currentTargetUnslashedBalanceIncrements = 0; - // Advance time units - // state.slot is advanced right before calling this function - // ``` - // postState.slot++; - // afterProcessEpoch(postState, epochTransitionCache); - // ``` - this.epoch = computeEpochAtSlot(state.slot); this.syncPeriod = computeSyncPeriodAtEpoch(this.epoch); } From 78a60a26dc837ae7af3d4af76985cd47563f226d Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 18:54:53 -0100 Subject: [PATCH 12/79] fix: perf test slot incrementing --- .../state-transition/test/perf/epoch/epochAltair.test.ts | 6 ++++-- .../state-transition/test/perf/epoch/epochCapella.test.ts | 6 ++++-- .../state-transition/test/perf/epoch/epochPhase0.test.ts | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/state-transition/test/perf/epoch/epochAltair.test.ts b/packages/state-transition/test/perf/epoch/epochAltair.test.ts index d143c03eed02..6461ae545ca1 100644 --- a/packages/state-transition/test/perf/epoch/epochAltair.test.ts +++ b/packages/state-transition/test/perf/epoch/epochAltair.test.ts @@ -183,10 +183,12 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue const state = stateOg.value.clone(); const cacheAfter = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStateAltair, cacheAfter); - state.slot++; return {state, cache: cacheAfter}; }, beforeEach: ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => state.epochCtx.afterProcessEpoch(state, cache), + fn: ({state, cache}) => { + state.slot++; + state.epochCtx.afterProcessEpoch(state, cache); + }, }); } diff --git a/packages/state-transition/test/perf/epoch/epochCapella.test.ts b/packages/state-transition/test/perf/epoch/epochCapella.test.ts index 45372a11c796..f18815e4dac2 100644 --- a/packages/state-transition/test/perf/epoch/epochCapella.test.ts +++ b/packages/state-transition/test/perf/epoch/epochCapella.test.ts @@ -155,10 +155,12 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue const state = stateOg.value.clone(); const cacheAfter = beforeProcessEpoch(state); processEpoch(fork, state, cacheAfter); - state.slot++; return {state, cache: cacheAfter}; }, beforeEach: ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => state.epochCtx.afterProcessEpoch(state, cache), + fn: ({state, cache}) => { + state.slot++; + state.epochCtx.afterProcessEpoch(state, cache); + }, }); } diff --git a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts index abcb6a9fba8c..c22a2cc2fa9f 100644 --- a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts @@ -158,10 +158,12 @@ function benchmarkPhase0EpochSteps(stateOg: LazyValue const state = stateOg.value.clone(); const cacheAfter = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStatePhase0, cacheAfter); - state.slot++; return {state, cache: cacheAfter}; }, beforeEach: ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => state.epochCtx.afterProcessEpoch(state, cache), + fn: ({state, cache}) => { + state.slot++; + state.epochCtx.afterProcessEpoch(state, cache); + }, }); } From 9b0ffe21ac2bea69e2983ce20965e7e2558d6cb7 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 18:55:58 -0100 Subject: [PATCH 13/79] fix: remove MockShufflingCache --- packages/state-transition/test/perf/util.ts | 4 -- .../test/unit/cachedBeaconState.test.ts | 2 - .../test/unit/upgradeState.test.ts | 2 - .../test/utils/mockShufflingCache.ts | 46 ------------------- packages/state-transition/test/utils/state.ts | 2 - 5 files changed, 56 deletions(-) delete mode 100644 packages/state-transition/test/utils/mockShufflingCache.ts diff --git a/packages/state-transition/test/perf/util.ts b/packages/state-transition/test/perf/util.ts index 6e607cdfdeb6..4b2a7da4a50e 100644 --- a/packages/state-transition/test/perf/util.ts +++ b/packages/state-transition/test/perf/util.ts @@ -32,7 +32,6 @@ import {interopPubkeysCached} from "../utils/interop.js"; import {getNextSyncCommittee} from "../../src/util/syncCommittee.js"; import {getEffectiveBalanceIncrements} from "../../src/cache/effectiveBalanceIncrements.js"; import {processSlots} from "../../src/index.js"; -import {MockShufflingCache} from "../utils/mockShufflingCache.js"; let phase0State: BeaconStatePhase0 | null = null; let phase0CachedState23637: CachedBeaconStatePhase0 | null = null; @@ -130,7 +129,6 @@ export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean config: createBeaconConfig(config, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - shufflingCache: new MockShufflingCache(), }); const currentEpoch = computeEpochAtSlot(state.slot - 1); @@ -236,7 +234,6 @@ export function generatePerfTestCachedStateAltair(opts?: { config: createBeaconConfig(altairConfig, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - shufflingCache: new MockShufflingCache(), }); } if (!altairCachedState23638) { @@ -440,7 +437,6 @@ export function generateTestCachedBeaconStateOnlyValidators({ config: createBeaconConfig(config, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - shufflingCache: new MockShufflingCache(), }, {skipSyncPubkeys: true} ); diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 8d944b2a4ad8..faa2ff08ee65 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -8,7 +8,6 @@ import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; import {interopPubkeysCached} from "../utils/interop.js"; import {modifyStateSameValidator, newStateWithValidators} from "../utils/capella.js"; -import {MockShufflingCache} from "../utils/mockShufflingCache.js"; import {IShufflingCache} from "../../src/index.js"; describe("CachedBeaconState", () => { @@ -66,7 +65,6 @@ describe("CachedBeaconState", () => { config, pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - shufflingCache: new MockShufflingCache(), }, {skipSyncCommitteeCache: true} ); diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index c1b7cbdd646f..2ea8eef182ac 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -7,7 +7,6 @@ import {config as chainConfig} from "@lodestar/config/default"; import {upgradeStateToDeneb} from "../../src/slot/upgradeStateToDeneb.js"; import {createCachedBeaconState} from "../../src/cache/stateCache.js"; import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; -import {MockShufflingCache} from "../utils/mockShufflingCache.js"; describe("upgradeState", () => { it("upgradeStateToDeneb", () => { @@ -19,7 +18,6 @@ describe("upgradeState", () => { config: createBeaconConfig(config, capellaState.genesisValidatorsRoot), pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - shufflingCache: new MockShufflingCache(), }, {skipSyncCommitteeCache: true} ); diff --git a/packages/state-transition/test/utils/mockShufflingCache.ts b/packages/state-transition/test/utils/mockShufflingCache.ts deleted file mode 100644 index 7e600f5f1f4d..000000000000 --- a/packages/state-transition/test/utils/mockShufflingCache.ts +++ /dev/null @@ -1,46 +0,0 @@ -import {MapDef} from "@lodestar/utils"; -import {Epoch, RootHex} from "@lodestar/types"; -import {BeaconStateAllForks} from "../../src/types.js"; -import {EpochShuffling, IShufflingCache, computeEpochShuffling} from "../../src/util/epochShuffling.js"; - -export class MockShufflingCache implements IShufflingCache { - private readonly itemsByDecisionRootByEpoch: MapDef> = new MapDef( - () => new Map() - ); - async build( - epoch: number, - decisionRoot: string, - state: BeaconStateAllForks, - activeIndices: number[] - ): Promise { - const cachedShuffling = this.getSync(epoch, decisionRoot); - if (cachedShuffling) { - return cachedShuffling; - } - return new Promise((resolve) => { - setTimeout(() => { - const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.set(shuffling, decisionRoot); - resolve(shuffling); - }); - }); - } - - buildSync(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): EpochShuffling { - const cachedShuffling = this.getSync(epoch, decisionRoot); - if (cachedShuffling) { - return cachedShuffling; - } - const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.set(shuffling, decisionRoot); - return shuffling; - } - - getSync(epoch: number, decisionRoot: string): EpochShuffling | null { - return this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot) ?? null; - } - - set(shuffling: EpochShuffling, decisionRoot: string): void { - this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch).set(decisionRoot, shuffling); - } -} diff --git a/packages/state-transition/test/utils/state.ts b/packages/state-transition/test/utils/state.ts index 596fcc3afdfb..0c70d01c226b 100644 --- a/packages/state-transition/test/utils/state.ts +++ b/packages/state-transition/test/utils/state.ts @@ -22,7 +22,6 @@ import { } from "../../src/index.js"; import {BeaconStateCache} from "../../src/cache/stateCache.js"; import {EpochCacheOpts} from "../../src/cache/epochCache.js"; -import {MockShufflingCache} from "./mockShufflingCache.js"; /** * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed. @@ -111,7 +110,6 @@ export function createCachedBeaconStateTest( // This is a test state, there's no need to have a global shared cache of keys pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - shufflingCache: new MockShufflingCache(), }, opts ); From 13f59964c4f1ba4afb4b35451ceb4b90e8e8c8c5 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 19:27:51 -0100 Subject: [PATCH 14/79] Revert "chore: refactoring" This reverts commit 104aa56d84ce3ca33d045705591767ef578dec3a. --- packages/beacon-node/src/chain/chain.ts | 13 +- .../beacon-node/src/chain/shufflingCache.ts | 147 ++++++++++-------- .../src/chain/validation/attestation.ts | 2 +- .../test/unit/chain/shufflingCache.test.ts | 23 ++- .../test/utils/validationData/attestation.ts | 8 +- .../src/util/epochShuffling.ts | 30 +--- 6 files changed, 103 insertions(+), 120 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index eaa72716cf74..26679a4aa51a 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -876,6 +876,9 @@ export class BeaconChain implements IBeaconChain { attHeadBlock: ProtoBlock, regenCaller: RegenCaller ): Promise { + // this is to prevent multiple calls to get shuffling for the same epoch and dependent root + // any subsequent calls of the same epoch and dependent root will wait for this promise to resolve + this.shufflingCache.insertPromise(attEpoch, shufflingDependentRoot); const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); let state: CachedBeaconStateAllForks; @@ -900,14 +903,8 @@ export class BeaconChain implements IBeaconChain { state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller); } - const decisionRoot = getShufflingDecisionBlock(state, attEpoch); - const shuffling = await this.shufflingCache.get(attEpoch, decisionRoot); - if (!shuffling) { - // This will be essentially unreachable considering regen should build the shuffling for this epoch - // but need to handle anyhow - throw Error(`UNREACHABLE: Shuffling not found for attestation epoch ${attEpoch} decisionRoot ${decisionRoot}`); - } - return shuffling; + // resolve the promise to unblock other calls of the same epoch and dependent root + return this.shufflingCache.get(attEpoch, getShufflingDecisionBlock(state, attEpoch)); } /** diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index b53157b4bd39..8753df94698a 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -68,12 +68,42 @@ export class ShufflingCache implements IShufflingCache { this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS; } + /** + * Insert a promise to make sure we don't regen state for the same shuffling. + * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. + */ + insertPromise(shufflingEpoch: Epoch, decisionRootHex: RootHex): void { + const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) + .flatMap((innerMap) => Array.from(innerMap.values())) + .filter((item) => isPromiseCacheItem(item)).length; + if (promiseCount >= MAX_PROMISES) { + throw new Error( + `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRootHex}` + ); + } + let resolveFn: ((shuffling: EpochShuffling) => void) | null = null; + const promise = new Promise((resolve) => { + resolveFn = resolve; + }); + if (resolveFn === null) { + throw new Error("Promise Constructor was not executed immediately"); + } + + const cacheItem: PromiseCacheItem = { + type: CacheItemType.promise, + promise, + resolveFn, + }; + this.add(shufflingEpoch, decisionRootHex, cacheItem); + this.metrics?.shufflingCache.insertPromiseCount.inc(); + } + /** * Most of the time, this should return a shuffling immediately. * If there's a promise, it means we are computing the same shuffling, so we wait for the promise to resolve. * Return null if we don't have a shuffling for this epoch and dependentRootHex. */ - async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { + async getShufflingOrNull(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); if (cacheItem === undefined) { return null; @@ -87,6 +117,18 @@ export class ShufflingCache implements IShufflingCache { } } + async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { + const item = await this.getShufflingOrNull(shufflingEpoch, decisionRootHex); + if (!item) { + throw new ShufflingCacheError({ + code: ShufflingCacheErrorCode.NO_SHUFFLING_FOUND, + epoch: shufflingEpoch, + decisionRoot: decisionRootHex, + }); + } + return item; + } + /** * Same to getShufflingOrNull() function but synchronous. */ @@ -104,82 +146,59 @@ export class ShufflingCache implements IShufflingCache { return null; } - buildSync(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): EpochShuffling { - const cachedShuffling = this.getSync(epoch, decisionRoot); - if (cachedShuffling) { - return cachedShuffling; - } - - const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.set(shuffling, decisionRoot); - return shuffling; - } - - async build( + getOrBuildSync( epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[] - ): Promise { - const cachedShuffling = await this.get(epoch, decisionRoot); - if (cachedShuffling) { - return cachedShuffling; + ): EpochShuffling { + const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); + if (cacheItem && isShufflingCacheItem(cacheItem)) { + // this.metrics?.shufflingCache.cacheHitInEpochTransition(); + return cacheItem.shuffling; } + // if (cacheItem) { + // this.metrics?.shufflingCache.cacheMissInEpochTransition(); + // } else { + // this.metrics?.shufflingCache.shufflingPromiseNotResolvedInEpochTransition(); + // } + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + this.add(epoch, decisionRoot, { + type: CacheItemType.shuffling, + shuffling, + }); + return shuffling; + } - // this is to prevent multiple calls to get shuffling for the same epoch and dependent root - // any subsequent calls of the same epoch and dependent root will wait for this promise to resolve - const cacheItem = this.insertPromise(epoch, decisionRoot); - // TODO: replace this call with a worker + build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): void { + let resolveFn: (shuffling: EpochShuffling) => void = () => {}; + this.add(epoch, decisionRoot, { + type: CacheItemType.promise, + resolveFn, + promise: new Promise((resolve) => { + resolveFn = resolve; + }), + }); setTimeout(() => { const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.set(shuffling, decisionRoot); - }, 5); - return cacheItem.promise; + this.add(epoch, decisionRoot, { + type: CacheItemType.shuffling, + shuffling, + }); + }, 100); } set(shuffling: EpochShuffling, decisionRoot: string): void { - const items = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch); - // if a pending shuffling promise exists, resolve it - const item = items.get(decisionRoot); - if (item !== undefined && isPromiseCacheItem(item)) { - item.resolveFn(shuffling); - } - // set the shuffling - items.set(decisionRoot, {type: CacheItemType.shuffling, shuffling}); - // prune the cache - pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs); + const cacheItem: ShufflingCacheItem = { + shuffling, + type: CacheItemType.shuffling, + }; + this.add(shuffling.epoch, decisionRoot, cacheItem); } - /** - * Insert a promise to make sure we don't regen state for the same shuffling. - * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. - */ - private insertPromise(shufflingEpoch: Epoch, decisionRoot: RootHex): PromiseCacheItem { - const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) - .flatMap((innerMap) => Array.from(innerMap.values())) - .filter((item) => isPromiseCacheItem(item)).length; - if (promiseCount >= MAX_PROMISES) { - throw new Error( - `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRoot}` - ); - } - let resolveFn: ((shuffling: EpochShuffling) => void) | null = null; - const promise = new Promise((resolve) => { - resolveFn = resolve; - }); - if (resolveFn === null) { - throw new Error("Promise Constructor was not executed immediately"); - } - - this.metrics?.shufflingCache.insertPromiseCount.inc(); - - const cacheItem: PromiseCacheItem = { - type: CacheItemType.promise, - promise, - resolveFn, - }; - this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionRoot, cacheItem); - return cacheItem; + private add(shufflingEpoch: Epoch, decisionBlock: RootHex, cacheItem: CacheItem): void { + this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionBlock, cacheItem); + pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs); } } diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index fc39534b45e6..389abd8a7c03 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -584,7 +584,7 @@ export async function getShufflingForAttestationVerification( const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); const shufflingDependentRoot = getShufflingDependentRoot(chain.forkChoice, attEpoch, blockEpoch, attHeadBlock); - const shuffling = await chain.shufflingCache.get(attEpoch, shufflingDependentRoot); + const shuffling = await chain.shufflingCache.getShufflingOrNull(attEpoch, shufflingDependentRoot); if (shuffling) { // most of the time, we should get the shuffling from cache chain.metrics?.gossipAttestation.shufflingCacheHit.inc({caller: regenCaller}); diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index 7323691ff14e..6295a993c072 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -10,13 +10,11 @@ describe("ShufflingCache", function () { const stateSlot = 100; const state = generateTestCachedBeaconStateOnlyValidators({vc, slot: stateSlot}); const currentEpoch = state.epochCtx.currentShuffling.epoch; - const activeIndices = Array.from(state.epochCtx.currentShuffling.activeIndices); let shufflingCache: ShufflingCache; beforeEach(() => { shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); - shufflingCache.buildSync(currentEpoch, decisionRoot, state, activeIndices); + shufflingCache.processState(state, currentEpoch); }); it("should get shuffling from cache", async function () { @@ -28,30 +26,29 @@ describe("ShufflingCache", function () { const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert promises at the same epoch does not prune the cache - shufflingCache["insertPromise"](currentEpoch, "0x00"); + shufflingCache.insertPromise(currentEpoch, "0x00"); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert shufflings at other epochs does prune the cache - shufflingCache.buildSync(currentEpoch + 1, decisionRoot, state, activeIndices); + shufflingCache.processState(state, currentEpoch + 1); // the current shuffling is not available anymore expect(await shufflingCache.get(currentEpoch, decisionRoot)).toBeNull(); }); it("should return shuffling from promise", async function () { - const nextEpoch = currentEpoch + 1; const nextDecisionRoot = getShufflingDecisionBlock(state, currentEpoch + 1); - shufflingCache["insertPromise"](nextEpoch, nextDecisionRoot); - const shufflingRequest0 = shufflingCache.get(nextEpoch, nextDecisionRoot); - const shufflingRequest1 = shufflingCache.get(nextEpoch, nextDecisionRoot); - shufflingCache.buildSync(nextEpoch, nextDecisionRoot, state, activeIndices); + shufflingCache.insertPromise(currentEpoch + 1, nextDecisionRoot); + const shufflingRequest0 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); + const shufflingRequest1 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); + shufflingCache.processState(state, currentEpoch + 1); expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling); expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling); }); it("should support up to 2 promises at a time", async function () { // insert 2 promises at the same epoch - shufflingCache["insertPromise"](currentEpoch, "0x00"); - shufflingCache["insertPromise"](currentEpoch, "0x01"); + shufflingCache.insertPromise(currentEpoch, "0x00"); + shufflingCache.insertPromise(currentEpoch, "0x01"); // inserting other promise should throw error - expect(() => shufflingCache["insertPromise"](currentEpoch, "0x02")).toThrow(); + expect(() => shufflingCache.insertPromise(currentEpoch, "0x02")).toThrow(); }); }); diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index a94a7857bd3e..c33d942dabc5 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -3,7 +3,6 @@ import { computeEpochAtSlot, computeSigningRoot, computeStartSlotAtEpoch, - getActiveValidatorIndices, getShufflingDecisionBlock, } from "@lodestar/state-transition"; import {ProtoBlock, IForkChoice, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; @@ -83,12 +82,9 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { }; const shufflingCache = new ShufflingCache(); + shufflingCache.processState(state, state.epochCtx.currentShuffling.epoch); + shufflingCache.processState(state, state.epochCtx.nextShuffling.epoch); const dependentRoot = getShufflingDecisionBlock(state, state.epochCtx.currentShuffling.epoch); - shufflingCache.set(state.epochCtx.currentShuffling, dependentRoot); - - const nextEpoch = state.epochCtx.currentShuffling.epoch + 1; - const nextDependentRoot = getShufflingDecisionBlock(state, nextEpoch); - shufflingCache.buildSync(nextEpoch, nextDependentRoot, state, getActiveValidatorIndices(state, nextEpoch)); const forkChoice = { getBlock: (root) => { diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 5175e40ed435..5033262216d7 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -14,40 +14,14 @@ import {computeStartSlotAtEpoch} from "./epoch.js"; import {getBlockRootAtSlot} from "./blockRoot.js"; export interface IShufflingCache { - /** - * Will synchronously get a shuffling if it is available or will return null if not. - */ getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null; - - /** - * Will synchronously get a shuffling if it is available. - * - * If a shuffling is not immediately available, a shuffling will be calculated. - * - * NOTE: this may recalculate an already in-progress shuffling. - */ - buildSync( + getOrBuildSync( epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[] ): EpochShuffling; - - /** - * Will immediately return a shuffling if it is available, or a promise to an in-progress shuffling calculation. - * - * If neither is available, a shuffling will be calculated. - */ - build( - epoch: Epoch, - decisionRoot: RootHex, - state: BeaconStateAllForks, - activeIndices: ValidatorIndex[] - ): Promise; - - /** - * Set a shuffling for a given epoch and decisionRoot. - */ + build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void; set(shuffling: EpochShuffling, decisionRoot: RootHex): void; } From d8d69c96fd0f6b8b5d053e9467b0749955dfc878 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 19:35:32 -0100 Subject: [PATCH 15/79] refactor: shufflingCache getters --- packages/beacon-node/src/chain/chain.ts | 8 ++++++- .../beacon-node/src/chain/shufflingCache.ts | 22 +++---------------- .../src/chain/validation/attestation.ts | 2 +- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 26679a4aa51a..81e415c80593 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -904,7 +904,13 @@ export class BeaconChain implements IBeaconChain { } // resolve the promise to unblock other calls of the same epoch and dependent root - return this.shufflingCache.get(attEpoch, getShufflingDecisionBlock(state, attEpoch)); + const shuffling = await this.shufflingCache.get(attEpoch, getShufflingDecisionBlock(state, attEpoch)); + if (!shuffling) { + // This will be essentially unreachable considering regen should build the shuffling for this epoch + // but need to handle anyhow + throw Error(`UNREACHABLE: Shuffling not found for attestation epoch ${attEpoch} decisionRoot ${decisionRoot}`); + } + return shuffling; } /** diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 8753df94698a..2c92f1ea7cfb 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -103,7 +103,7 @@ export class ShufflingCache implements IShufflingCache { * If there's a promise, it means we are computing the same shuffling, so we wait for the promise to resolve. * Return null if we don't have a shuffling for this epoch and dependentRootHex. */ - async getShufflingOrNull(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { + async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); if (cacheItem === undefined) { return null; @@ -117,32 +117,16 @@ export class ShufflingCache implements IShufflingCache { } } - async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { - const item = await this.getShufflingOrNull(shufflingEpoch, decisionRootHex); - if (!item) { - throw new ShufflingCacheError({ - code: ShufflingCacheErrorCode.NO_SHUFFLING_FOUND, - epoch: shufflingEpoch, - decisionRoot: decisionRootHex, - }); - } - return item; - } - /** * Same to getShufflingOrNull() function but synchronous. */ getSync(shufflingEpoch: Epoch, decisionRootHex: RootHex): EpochShuffling | null { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); - if (cacheItem === undefined) { - return null; - } - - if (isShufflingCacheItem(cacheItem)) { + if (cacheItem && isShufflingCacheItem(cacheItem)) { return cacheItem.shuffling; } - // ignore promise + // ignore promise and cache misses return null; } diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 389abd8a7c03..fc39534b45e6 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -584,7 +584,7 @@ export async function getShufflingForAttestationVerification( const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); const shufflingDependentRoot = getShufflingDependentRoot(chain.forkChoice, attEpoch, blockEpoch, attHeadBlock); - const shuffling = await chain.shufflingCache.getShufflingOrNull(attEpoch, shufflingDependentRoot); + const shuffling = await chain.shufflingCache.get(attEpoch, shufflingDependentRoot); if (shuffling) { // most of the time, we should get the shuffling from cache chain.metrics?.gossipAttestation.shufflingCacheHit.inc({caller: regenCaller}); From 89798a5981e622b5a2b0f9331106db9cd5e21b28 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 19:45:41 -0100 Subject: [PATCH 16/79] refactor: shufflingCache setters --- .../beacon-node/src/chain/shufflingCache.ts | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 2c92f1ea7cfb..9ee0562711f5 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -72,13 +72,13 @@ export class ShufflingCache implements IShufflingCache { * Insert a promise to make sure we don't regen state for the same shuffling. * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. */ - insertPromise(shufflingEpoch: Epoch, decisionRootHex: RootHex): void { + insertPromise(shufflingEpoch: Epoch, decisionRoot: RootHex): void { const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) .flatMap((innerMap) => Array.from(innerMap.values())) .filter((item) => isPromiseCacheItem(item)).length; if (promiseCount >= MAX_PROMISES) { throw new Error( - `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRootHex}` + `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRoot}` ); } let resolveFn: ((shuffling: EpochShuffling) => void) | null = null; @@ -94,7 +94,7 @@ export class ShufflingCache implements IShufflingCache { promise, resolveFn, }; - this.add(shufflingEpoch, decisionRootHex, cacheItem); + this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionRoot, cacheItem); this.metrics?.shufflingCache.insertPromiseCount.inc(); } @@ -147,7 +147,7 @@ export class ShufflingCache implements IShufflingCache { // this.metrics?.shufflingCache.shufflingPromiseNotResolvedInEpochTransition(); // } const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.add(epoch, decisionRoot, { + this.set(shuffling, decisionRoot, { type: CacheItemType.shuffling, shuffling, }); @@ -156,7 +156,7 @@ export class ShufflingCache implements IShufflingCache { build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): void { let resolveFn: (shuffling: EpochShuffling) => void = () => {}; - this.add(epoch, decisionRoot, { + this.set(shuffling, decisionRoot, { type: CacheItemType.promise, resolveFn, promise: new Promise((resolve) => { @@ -165,7 +165,7 @@ export class ShufflingCache implements IShufflingCache { }); setTimeout(() => { const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.add(epoch, decisionRoot, { + this.set(shuffling, decisionRoot, { type: CacheItemType.shuffling, shuffling, }); @@ -173,15 +173,19 @@ export class ShufflingCache implements IShufflingCache { } set(shuffling: EpochShuffling, decisionRoot: string): void { - const cacheItem: ShufflingCacheItem = { - shuffling, - type: CacheItemType.shuffling, - }; - this.add(shuffling.epoch, decisionRoot, cacheItem); - } - - private add(shufflingEpoch: Epoch, decisionBlock: RootHex, cacheItem: CacheItem): void { - this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionBlock, cacheItem); + const items = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch); + // if a pending shuffling promise exists, resolve it + const item = items.get(decisionRoot); + if (item) { + if (isPromiseCacheItem(item)) { + item.resolveFn(shuffling); + } else { + // metric for throwing away previously calculated shuffling + } + } + // set the shuffling + items.set(decisionRoot, {type: CacheItemType.shuffling, shuffling}); + // prune the cache pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs); } } From 81449e421dc429b6d9d0db9fc2e8b25a91778f7b Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 20:06:07 -0100 Subject: [PATCH 17/79] refactor: build and getOrBuild --- .../beacon-node/src/chain/shufflingCache.ts | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 9ee0562711f5..278378301fd7 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -106,13 +106,14 @@ export class ShufflingCache implements IShufflingCache { async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); if (cacheItem === undefined) { + // this.metrics?.shufflingCache.miss(); return null; } if (isShufflingCacheItem(cacheItem)) { return cacheItem.shuffling; } else { - // promise + // this.metrics?.shufflingCache.shufflingPromiseNotResolved({type: async}); return cacheItem.promise; } } @@ -122,10 +123,15 @@ export class ShufflingCache implements IShufflingCache { */ getSync(shufflingEpoch: Epoch, decisionRootHex: RootHex): EpochShuffling | null { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); - if (cacheItem && isShufflingCacheItem(cacheItem)) { - return cacheItem.shuffling; + if (cacheItem) { + if (isShufflingCacheItem(cacheItem)) { + // this.metrics?.shufflingCache.hit() + return cacheItem.shuffling; + } + // this.metrics?.shufflingCache.shufflingPromiseNotResolved({type: sync}); } + // this.metrics?.shufflingCache.miss(); // ignore promise and cache misses return null; } @@ -136,16 +142,11 @@ export class ShufflingCache implements IShufflingCache { state: BeaconStateAllForks, activeIndices: number[] ): EpochShuffling { - const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); - if (cacheItem && isShufflingCacheItem(cacheItem)) { - // this.metrics?.shufflingCache.cacheHitInEpochTransition(); - return cacheItem.shuffling; + const cacheItem = this.getSync(epoch, decisionRoot); + if (cacheItem) { + return cacheItem; } - // if (cacheItem) { - // this.metrics?.shufflingCache.cacheMissInEpochTransition(); - // } else { - // this.metrics?.shufflingCache.shufflingPromiseNotResolvedInEpochTransition(); - // } + // this.metrics?.shufflingCache.cacheMissInEpochTransition(); const shuffling = computeEpochShuffling(state, activeIndices, epoch); this.set(shuffling, decisionRoot, { type: CacheItemType.shuffling, @@ -155,21 +156,19 @@ export class ShufflingCache implements IShufflingCache { } build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): void { - let resolveFn: (shuffling: EpochShuffling) => void = () => {}; - this.set(shuffling, decisionRoot, { - type: CacheItemType.promise, - resolveFn, - promise: new Promise((resolve) => { - resolveFn = resolve; - }), - }); + this.insertPromise(epoch, decisionRoot); + /** + * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations + * on a NICE thread with a rust implementation + */ setTimeout(() => { const shuffling = computeEpochShuffling(state, activeIndices, epoch); this.set(shuffling, decisionRoot, { type: CacheItemType.shuffling, shuffling, }); - }, 100); + // wait until after the first slot to help with attestation and block proposal performance + }, 12_000); } set(shuffling: EpochShuffling, decisionRoot: string): void { From 5799676b3d9936c43d42bad287387a72be5889b2 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 20:25:56 -0100 Subject: [PATCH 18/79] docs: add comments to ShufflingCache methods --- .../beacon-node/src/chain/shufflingCache.ts | 35 ++++++++++++++----- .../src/util/epochShuffling.ts | 20 +++++++++++ 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 278378301fd7..b3e8d15d9aa4 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -113,13 +113,12 @@ export class ShufflingCache implements IShufflingCache { if (isShufflingCacheItem(cacheItem)) { return cacheItem.shuffling; } else { - // this.metrics?.shufflingCache.shufflingPromiseNotResolved({type: async}); return cacheItem.promise; } } /** - * Same to getShufflingOrNull() function but synchronous. + * Will synchronously get a shuffling if it is available or will return null if not. */ getSync(shufflingEpoch: Epoch, decisionRootHex: RootHex): EpochShuffling | null { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); @@ -128,26 +127,37 @@ export class ShufflingCache implements IShufflingCache { // this.metrics?.shufflingCache.hit() return cacheItem.shuffling; } - // this.metrics?.shufflingCache.shufflingPromiseNotResolved({type: sync}); + // this.metrics?.shufflingCache.shufflingPromiseNotResolved(); } - // this.metrics?.shufflingCache.miss(); - // ignore promise and cache misses return null; } + /** + * Gets a cached shuffling via the epoch and decision root. If the shuffling is not + * available it will build it synchronously and return the shuffling. + * + * NOTE: If a shuffling is already queued and not calculated it will build and resolve + * the promise but the already queued build will happen at some later time + */ getOrBuildSync( epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[] ): EpochShuffling { - const cacheItem = this.getSync(epoch, decisionRoot); - if (cacheItem) { - return cacheItem; + const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); + if (cacheItem && isShufflingCacheItem(cacheItem)) { + // this.metrics?.shufflingCache.cacheHitEpochTransition(); + return cacheItem.shuffling; } - // this.metrics?.shufflingCache.cacheMissInEpochTransition(); const shuffling = computeEpochShuffling(state, activeIndices, epoch); + if (cacheItem) { + // this.metrics?.shufflingCache.shufflingPromiseNotResolvedEpochTransition(); + cacheItem.resolveFn(shuffling); + } else { + // this.metrics?.shufflingCache.cacheMissEpochTransition(); + } this.set(shuffling, decisionRoot, { type: CacheItemType.shuffling, shuffling, @@ -155,6 +165,9 @@ export class ShufflingCache implements IShufflingCache { return shuffling; } + /** + * Queue asynchronous build for an EpochShuffling + */ build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): void { this.insertPromise(epoch, decisionRoot); /** @@ -171,6 +184,10 @@ export class ShufflingCache implements IShufflingCache { }, 12_000); } + /** + * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will + * resolve the promise with the built shuffling + */ set(shuffling: EpochShuffling, decisionRoot: string): void { const items = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch); // if a pending shuffling promise exists, resolve it diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 5033262216d7..561c196715bc 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -14,14 +14,34 @@ import {computeStartSlotAtEpoch} from "./epoch.js"; import {getBlockRootAtSlot} from "./blockRoot.js"; export interface IShufflingCache { + /** + * Will synchronously get a shuffling if it is available or will return null if not. + */ getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null; + + /** + * Gets a cached shuffling via the epoch and decision root. If the shuffling is not + * available it will build it synchronously and return the shuffling. + * + * NOTE: If a shuffling is already queued and not calculated it will build and resolve + * the promise but the already queued build will happen at some later time + */ getOrBuildSync( epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[] ): EpochShuffling; + + /** + * Queue asynchronous build for an EpochShuffling + */ build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void; + + /** + * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will + * resolve the promise with the built shuffling + */ set(shuffling: EpochShuffling, decisionRoot: RootHex): void; } From 9a0ca700ca48ee71fef343a11cbbdaf821a69ecd Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 22:06:19 -0100 Subject: [PATCH 19/79] chore: lint issues --- packages/beacon-node/src/chain/chain.ts | 3 ++- .../beacon-node/src/chain/shufflingCache.ts | 26 +++++++------------ 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 81e415c80593..3e39a22ac831 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -904,7 +904,8 @@ export class BeaconChain implements IBeaconChain { } // resolve the promise to unblock other calls of the same epoch and dependent root - const shuffling = await this.shufflingCache.get(attEpoch, getShufflingDecisionBlock(state, attEpoch)); + const decisionRoot = getShufflingDecisionBlock(state, attEpoch); + const shuffling = await this.shufflingCache.get(attEpoch, decisionRoot); if (!shuffling) { // This will be essentially unreachable considering regen should build the shuffling for this epoch // but need to handle anyhow diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index b3e8d15d9aa4..f4ac971870c1 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -72,13 +72,13 @@ export class ShufflingCache implements IShufflingCache { * Insert a promise to make sure we don't regen state for the same shuffling. * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. */ - insertPromise(shufflingEpoch: Epoch, decisionRoot: RootHex): void { + insertPromise(epoch: Epoch, decisionRoot: RootHex): void { const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) .flatMap((innerMap) => Array.from(innerMap.values())) .filter((item) => isPromiseCacheItem(item)).length; if (promiseCount >= MAX_PROMISES) { throw new Error( - `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRoot}` + `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${epoch}, decisionRootHex: ${decisionRoot}` ); } let resolveFn: ((shuffling: EpochShuffling) => void) | null = null; @@ -94,7 +94,7 @@ export class ShufflingCache implements IShufflingCache { promise, resolveFn, }; - this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionRoot, cacheItem); + this.itemsByDecisionRootByEpoch.getOrDefault(epoch).set(decisionRoot, cacheItem); this.metrics?.shufflingCache.insertPromiseCount.inc(); } @@ -103,8 +103,8 @@ export class ShufflingCache implements IShufflingCache { * If there's a promise, it means we are computing the same shuffling, so we wait for the promise to resolve. * Return null if we don't have a shuffling for this epoch and dependentRootHex. */ - async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { - const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); + async get(epoch: Epoch, decisionRoot: RootHex): Promise { + const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); if (cacheItem === undefined) { // this.metrics?.shufflingCache.miss(); return null; @@ -120,8 +120,8 @@ export class ShufflingCache implements IShufflingCache { /** * Will synchronously get a shuffling if it is available or will return null if not. */ - getSync(shufflingEpoch: Epoch, decisionRootHex: RootHex): EpochShuffling | null { - const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); + getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null { + const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); if (cacheItem) { if (isShufflingCacheItem(cacheItem)) { // this.metrics?.shufflingCache.hit() @@ -146,7 +146,7 @@ export class ShufflingCache implements IShufflingCache { state: BeaconStateAllForks, activeIndices: number[] ): EpochShuffling { - const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); + const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); if (cacheItem && isShufflingCacheItem(cacheItem)) { // this.metrics?.shufflingCache.cacheHitEpochTransition(); return cacheItem.shuffling; @@ -158,10 +158,7 @@ export class ShufflingCache implements IShufflingCache { } else { // this.metrics?.shufflingCache.cacheMissEpochTransition(); } - this.set(shuffling, decisionRoot, { - type: CacheItemType.shuffling, - shuffling, - }); + this.set(shuffling, decisionRoot); return shuffling; } @@ -176,10 +173,7 @@ export class ShufflingCache implements IShufflingCache { */ setTimeout(() => { const shuffling = computeEpochShuffling(state, activeIndices, epoch); - this.set(shuffling, decisionRoot, { - type: CacheItemType.shuffling, - shuffling, - }); + this.set(shuffling, decisionRoot); // wait until after the first slot to help with attestation and block proposal performance }, 12_000); } From 10dd105035366bce4208490b43c6b5884d35a95e Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 22:11:51 -0100 Subject: [PATCH 20/79] test: update tests in beacon-node --- .../test/spec/presets/genesis.test.ts | 14 ++++++++---- .../test/unit/chain/shufflingCache.test.ts | 22 ++++++++++--------- packages/beacon-node/test/utils/state.ts | 1 + .../test/utils/validationData/attestation.ts | 9 ++++---- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/genesis.test.ts b/packages/beacon-node/test/spec/presets/genesis.test.ts index f03f2595a566..30d361e58786 100644 --- a/packages/beacon-node/test/spec/presets/genesis.test.ts +++ b/packages/beacon-node/test/spec/presets/genesis.test.ts @@ -19,6 +19,7 @@ import {getConfig} from "../../utils/config.js"; import {RunnerType} from "../utils/types.js"; import {specTestIterator} from "../utils/specTestIterator.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; +import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; // The aim of the genesis tests is to provide a baseline to test genesis-state initialization and test if the // proposed genesis-validity conditions are working. @@ -42,10 +43,15 @@ const genesisInitialization: TestRunnerFn { shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - shufflingCache.processState(state, currentEpoch); + shufflingCache.set(state.epochCtx.currentShuffling, state.epochCtx.currentDecisionRoot); }); it("should get shuffling from cache", async function () { @@ -29,19 +31,19 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch, "0x00"); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert shufflings at other epochs does prune the cache - shufflingCache.processState(state, currentEpoch + 1); + shufflingCache.set(state.epochCtx.nextShuffling!, state.epochCtx.nextDecisionRoot); // the current shuffling is not available anymore expect(await shufflingCache.get(currentEpoch, decisionRoot)).toBeNull(); }); it("should return shuffling from promise", async function () { - const nextDecisionRoot = getShufflingDecisionBlock(state, currentEpoch + 1); - shufflingCache.insertPromise(currentEpoch + 1, nextDecisionRoot); - const shufflingRequest0 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - const shufflingRequest1 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - shufflingCache.processState(state, currentEpoch + 1); - expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling); - expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling); + const nextDecisionRoot = getShufflingDecisionBlock(state, nextEpoch); + shufflingCache.insertPromise(nextEpoch, nextDecisionRoot); + const shufflingRequest0 = shufflingCache.get(nextEpoch, nextDecisionRoot); + const shufflingRequest1 = shufflingCache.get(nextEpoch, nextDecisionRoot); + shufflingCache.set(state.epochCtx.nextShuffling!, state.epochCtx.nextDecisionRoot); + expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling!); + expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling!); }); it("should support up to 2 promises at a time", async function () { diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index 1e9f614e8093..68b7e98c7b01 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -97,6 +97,7 @@ export function generateState( /** * This generates state with default pubkey + * TODO: (@matthewkeil) - this is duplicated and exists in state-transition as well */ export function generateCachedState(opts?: TestBeaconState): CachedBeaconStateAllForks { const config = getConfig(ForkName.phase0); diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index c33d942dabc5..8be754b4627c 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -82,9 +82,10 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { }; const shufflingCache = new ShufflingCache(); - shufflingCache.processState(state, state.epochCtx.currentShuffling.epoch); - shufflingCache.processState(state, state.epochCtx.nextShuffling.epoch); - const dependentRoot = getShufflingDecisionBlock(state, state.epochCtx.currentShuffling.epoch); + shufflingCache.set(state.epochCtx.previousShuffling, state.epochCtx.previousDecisionRoot); + shufflingCache.set(state.epochCtx.currentShuffling, state.epochCtx.currentDecisionRoot); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + shufflingCache.set(state.epochCtx.nextShuffling!, state.epochCtx.nextDecisionRoot); const forkChoice = { getBlock: (root) => { @@ -95,7 +96,7 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { if (rootHex !== toHexString(beaconBlockRoot)) return null; return headBlock; }, - getDependentRoot: () => dependentRoot, + getDependentRoot: () => state.epochCtx.currentDecisionRoot, } as Partial as IForkChoice; const committeeIndices = state.epochCtx.getBeaconCommittee(attSlot, attIndex); From bb3fd1c1ff6c119d978b6d963dc9435117811f58 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 22:16:45 -0100 Subject: [PATCH 21/79] chore: lint --- .../beacon-node/test/utils/validationData/attestation.ts | 7 +------ .../light-client/test/unit/webEsmBundle.browser.test.ts | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index 8be754b4627c..a2bd76452b0c 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -1,10 +1,5 @@ import {BitArray, toHexString} from "@chainsafe/ssz"; -import { - computeEpochAtSlot, - computeSigningRoot, - computeStartSlotAtEpoch, - getShufflingDecisionBlock, -} from "@lodestar/state-transition"; +import {computeEpochAtSlot, computeSigningRoot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ProtoBlock, IForkChoice, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import {phase0, Slot, ssz} from "@lodestar/types"; diff --git a/packages/light-client/test/unit/webEsmBundle.browser.test.ts b/packages/light-client/test/unit/webEsmBundle.browser.test.ts index defc421d7071..05afe1fba8e7 100644 --- a/packages/light-client/test/unit/webEsmBundle.browser.test.ts +++ b/packages/light-client/test/unit/webEsmBundle.browser.test.ts @@ -1,4 +1,4 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-member-access*/ import {expect, describe, it, vi, beforeAll} from "vitest"; import {sleep} from "@lodestar/utils"; import {Lightclient, LightclientEvent, utils, transport} from "../../dist/lightclient.min.mjs"; From 59f6d3f8fea015ba76a5a1a643a46735b7d69d43 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 23:13:59 -0100 Subject: [PATCH 22/79] feat: get shufflings from cache for API --- .../src/api/impl/beacon/state/index.ts | 9 +++- .../src/api/impl/validator/index.ts | 11 ++++- .../state-transition/src/cache/epochCache.ts | 41 +++--------------- .../src/util/calculateCommitteeAssignments.ts | 43 +++++++++++++++++++ packages/state-transition/src/util/index.ts | 1 + 5 files changed, 69 insertions(+), 36 deletions(-) create mode 100644 packages/state-transition/src/util/calculateCommitteeAssignments.ts diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index 9d9646ee8cf3..4dd028ed1978 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -202,7 +202,14 @@ export function getBeaconStateApi({ const epoch = filters.epoch ?? computeEpochAtSlot(state.slot); const startSlot = computeStartSlotAtEpoch(epoch); - const shuffling = stateCached.epochCtx.getShufflingAtEpoch(epoch); + const decisionRoot = stateCached.epochCtx.getDecisionRoot(epoch); + const shuffling = await chain.shufflingCache.get(epoch, decisionRoot); + if (!shuffling) { + throw new ApiError( + 400, + `No shuffling found to calculate committees for epoch: ${epoch} and decisionRoot: ${decisionRoot}` + ); + } const committees = shuffling.committees; const committeesFlat = committees.flatMap((slotCommittees, slotInEpoch) => { const slot = startSlot + slotInEpoch; diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index b2a0b8575f5c..bd31373b27e0 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -3,6 +3,7 @@ import {ApplicationMethods} from "@lodestar/api/server"; import { CachedBeaconStateAllForks, computeStartSlotAtEpoch, + calculateCommitteeAssignments, proposerShufflingDecisionRoot, attesterShufflingDecisionRoot, getBlockRootAtSlot, @@ -988,7 +989,15 @@ export function getValidatorApi( // Check that all validatorIndex belong to the state before calling getCommitteeAssignments() const pubkeys = getPubkeysForIndices(state.validators, indices); - const committeeAssignments = state.epochCtx.getCommitteeAssignments(epoch, indices); + const decisionRoot = state.epochCtx.getDecisionRoot(epoch); + const shuffling = await chain.shufflingCache.get(epoch, decisionRoot); + if (!shuffling) { + throw new ApiError( + 400, + `No shuffling found to calculate committee assignments for epoch: ${epoch} and decisionRoot: ${decisionRoot}` + ); + } + const committeeAssignments = calculateCommitteeAssignments(shuffling, indices); const duties: routes.validator.AttesterDuty[] = []; for (let i = 0, len = indices.length; i < len; i++) { const validatorIndex = indices[i]; diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 1c6caee47168..8ef5625628a7 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -34,6 +34,7 @@ import { import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../util/syncCommittee.js"; import {sumTargetUnslashedBalanceIncrements} from "../util/targetUnslashedBalance.js"; import {getTotalSlashingsByIncrement} from "../epoch/processSlashings.js"; +import {AttesterDuty, calculateCommitteeAssignments} from "../util/calculateCommitteeAssignments.js"; import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements.js"; import {Index2PubkeyCache, PubkeyIndexMap, syncPubkeys} from "./pubkeyCache.js"; import {BeaconStateAllForks, BeaconStateAltair} from "./types.js"; @@ -380,6 +381,7 @@ export class EpochCache { } else { currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentEpoch); shufflingCache?.set(currentShuffling, currentDecisionRoot); + // shufflingCache?.metrics?.shufflingCache.miss(); } let previousShuffling: EpochShuffling; @@ -391,6 +393,7 @@ export class EpochCache { } else { previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); shufflingCache?.set(previousShuffling, previousDecisionRoot); + // shufflingCache?.metrics?.shufflingCache.miss(); } let nextShuffling: EpochShuffling; @@ -399,6 +402,7 @@ export class EpochCache { } else { nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); shufflingCache?.set(nextShuffling, nextDecisionRoot); + // shufflingCache?.metrics?.shufflingCache.miss(); } const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER); @@ -605,6 +609,7 @@ export class EpochCache { if (this.nextShuffling) { // was already pulled by the api or another method on EpochCache this.currentShuffling = this.nextShuffling; + // shufflingCache?.metrics?.shufflingCache.nextShufflingOnEpochCache(); } else { this.currentShuffling = this.shufflingCache?.getOrBuildSync( @@ -804,30 +809,8 @@ export class EpochCache { epoch: Epoch, requestedValidatorIndices: ValidatorIndex[] ): Map { - const requestedValidatorIndicesSet = new Set(requestedValidatorIndices); - const duties = new Map(); - - const epochCommittees = this.getShufflingAtEpoch(epoch).committees; - for (let epochSlot = 0; epochSlot < SLOTS_PER_EPOCH; epochSlot++) { - const slotCommittees = epochCommittees[epochSlot]; - for (let i = 0, committeesAtSlot = slotCommittees.length; i < committeesAtSlot; i++) { - for (let j = 0, committeeLength = slotCommittees[i].length; j < committeeLength; j++) { - const validatorIndex = slotCommittees[i][j]; - if (requestedValidatorIndicesSet.has(validatorIndex)) { - duties.set(validatorIndex, { - validatorIndex, - committeeLength, - committeesAtSlot, - validatorCommitteeIndex: j, - committeeIndex: i, - slot: epoch * SLOTS_PER_EPOCH + epochSlot, - }); - } - } - } - } - - return duties; + const shuffling = this.getShufflingAtEpoch(epoch); + return calculateCommitteeAssignments(shuffling, requestedValidatorIndices); } /** @@ -991,16 +974,6 @@ function getEffectiveBalanceIncrementsByteLen(validatorCount: number): number { return 1024 * Math.ceil(validatorCount / 1024); } -// Copied from lodestar-api package to avoid depending on the package -type AttesterDuty = { - validatorIndex: ValidatorIndex; - committeeIndex: CommitteeIndex; - committeeLength: number; - committeesAtSlot: number; - validatorCommitteeIndex: number; - slot: Slot; -}; - export enum EpochCacheErrorCode { COMMITTEE_INDEX_OUT_OF_RANGE = "EPOCH_CONTEXT_ERROR_COMMITTEE_INDEX_OUT_OF_RANGE", COMMITTEE_EPOCH_OUT_OF_RANGE = "EPOCH_CONTEXT_ERROR_COMMITTEE_EPOCH_OUT_OF_RANGE", diff --git a/packages/state-transition/src/util/calculateCommitteeAssignments.ts b/packages/state-transition/src/util/calculateCommitteeAssignments.ts new file mode 100644 index 000000000000..992c5efbdaaa --- /dev/null +++ b/packages/state-transition/src/util/calculateCommitteeAssignments.ts @@ -0,0 +1,43 @@ +import {CommitteeIndex, Slot, ValidatorIndex} from "@lodestar/types"; +import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {EpochShuffling} from "./epochShuffling.js"; + +// Copied from lodestar-api package to avoid depending on the package +export interface AttesterDuty { + validatorIndex: ValidatorIndex; + committeeIndex: CommitteeIndex; + committeeLength: number; + committeesAtSlot: number; + validatorCommitteeIndex: number; + slot: Slot; +} + +export function calculateCommitteeAssignments( + epochShuffling: EpochShuffling, + requestedValidatorIndices: ValidatorIndex[] +): Map { + const requestedValidatorIndicesSet = new Set(requestedValidatorIndices); + const duties = new Map(); + + const epochCommittees = epochShuffling.committees; + for (let epochSlot = 0; epochSlot < SLOTS_PER_EPOCH; epochSlot++) { + const slotCommittees = epochCommittees[epochSlot]; + for (let i = 0, committeesAtSlot = slotCommittees.length; i < committeesAtSlot; i++) { + for (let j = 0, committeeLength = slotCommittees[i].length; j < committeeLength; j++) { + const validatorIndex = slotCommittees[i][j]; + if (requestedValidatorIndicesSet.has(validatorIndex)) { + duties.set(validatorIndex, { + validatorIndex, + committeeLength, + committeesAtSlot, + validatorCommitteeIndex: j, + committeeIndex: i, + slot: epochShuffling.epoch * SLOTS_PER_EPOCH + epochSlot, + }); + } + } + } + } + + return duties; +} diff --git a/packages/state-transition/src/util/index.ts b/packages/state-transition/src/util/index.ts index 3f2e91da9a77..bef3ae2f0511 100644 --- a/packages/state-transition/src/util/index.ts +++ b/packages/state-transition/src/util/index.ts @@ -4,6 +4,7 @@ export * from "./attestation.js"; export * from "./attesterStatus.js"; export * from "./balance.js"; export * from "./blindedBlock.js"; +export * from "./calculateCommitteeAssignments.js"; export * from "./capella.js"; export * from "./execution.js"; export * from "./blockRoot.js"; From bae40cc19037e570e094bdc405c871587b2d38ad Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 23:36:26 -0100 Subject: [PATCH 23/79] feat: minTimeDelayToBuildShuffling cli flag --- packages/beacon-node/src/chain/shufflingCache.ts | 11 ++++++++--- packages/cli/src/options/beaconNodeOptions/chain.ts | 11 +++++++++++ .../cli/test/unit/options/beaconNodeOptions.test.ts | 2 ++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index f4ac971870c1..60f115ed59ba 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -15,7 +15,9 @@ const MAX_EPOCHS = 4; * With default chain option of maxSkipSlots = 32, there should be no shuffling promise. If that happens a lot, it could blow up Lodestar, * with MAX_EPOCHS = 4, only allow 2 promise at a time. Note that regen already bounds number of concurrent requests at 1 already. */ -const MAX_PROMISES = 2; +const MAX_PROMISES = 4; + +const DEFAULT_MIN_TIME_DELAY_IN_MS = 5; enum CacheItemType { shuffling, @@ -37,6 +39,7 @@ type CacheItem = ShufflingCacheItem | PromiseCacheItem; export type ShufflingCacheOpts = { maxShufflingCacheEpochs?: number; + minTimeDelayToBuildShuffling?: number; }; /** @@ -52,9 +55,10 @@ export class ShufflingCache implements IShufflingCache { ); private readonly maxEpochs: number; + private readonly minTimeDelayToBuild: number; constructor( - private readonly metrics: Metrics | null = null, + readonly metrics: Metrics | null = null, opts: ShufflingCacheOpts = {} ) { if (metrics) { @@ -66,6 +70,7 @@ export class ShufflingCache implements IShufflingCache { } this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS; + this.minTimeDelayToBuild = opts.minTimeDelayToBuildShuffling ?? DEFAULT_MIN_TIME_DELAY_IN_MS; } /** @@ -175,7 +180,7 @@ export class ShufflingCache implements IShufflingCache { const shuffling = computeEpochShuffling(state, activeIndices, epoch); this.set(shuffling, decisionRoot); // wait until after the first slot to help with attestation and block proposal performance - }, 12_000); + }, this.minTimeDelayToBuild); } /** diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index aae97b6db68f..c905f86ad6ac 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -27,6 +27,7 @@ export type ChainArgs = { broadcastValidationStrictness?: string; "chain.minSameMessageSignatureSetsToBatch"?: number; "chain.maxShufflingCacheEpochs"?: number; + "chain.minTimeDelayToBuildShuffling"?: number; "chain.archiveBlobEpochs"?: number; "chain.nHistoricalStates"?: boolean; "chain.nHistoricalStatesFileDataStore"?: boolean; @@ -60,6 +61,8 @@ export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] { minSameMessageSignatureSetsToBatch: args["chain.minSameMessageSignatureSetsToBatch"] ?? defaultOptions.chain.minSameMessageSignatureSetsToBatch, maxShufflingCacheEpochs: args["chain.maxShufflingCacheEpochs"] ?? defaultOptions.chain.maxShufflingCacheEpochs, + minTimeDelayToBuildShuffling: + args["chain.minTimeDelayToBuildShuffling"] ?? defaultOptions.chain.minTimeDelayToBuildShuffling, archiveBlobEpochs: args["chain.archiveBlobEpochs"], nHistoricalStates: args["chain.nHistoricalStates"] ?? defaultOptions.chain.nHistoricalStates, nHistoricalStatesFileDataStore: @@ -235,6 +238,14 @@ Will double processing times. Use only for debugging purposes.", group: "chain", }, + "chain.minTimeDelayToBuildShuffling": { + hidden: true, + description: "Minimum amount of time to delay before building a shuffling after its queued", + type: "number", + default: defaultOptions.chain.minTimeDelayToBuildShuffling, + group: "chain", + }, + "chain.archiveBlobEpochs": { description: "Number of epochs to retain finalized blobs (minimum of MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS)", type: "number", diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index d74ae73b966f..cdcd831ed15f 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -37,6 +37,7 @@ describe("options / beaconNodeOptions", () => { "chain.trustedSetup": "", "chain.minSameMessageSignatureSetsToBatch": 32, "chain.maxShufflingCacheEpochs": 100, + "chain.minTimeDelayToBuildShuffling": 5, "chain.archiveBlobEpochs": 10000, "chain.nHistoricalStates": true, "chain.nHistoricalStatesFileDataStore": true, @@ -145,6 +146,7 @@ describe("options / beaconNodeOptions", () => { trustedSetup: "", minSameMessageSignatureSetsToBatch: 32, maxShufflingCacheEpochs: 100, + minTimeDelayToBuildShuffling: 5, archiveBlobEpochs: 10000, nHistoricalStates: true, nHistoricalStatesFileDataStore: true, From b8269efeaebd5d4c564744a806b42447620812e0 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 11 Jul 2024 23:54:08 -0100 Subject: [PATCH 24/79] test: fix shufflingCache promise insertion test --- packages/beacon-node/test/unit/chain/shufflingCache.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index 972c9713c1be..6479bafbe0e0 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -46,11 +46,13 @@ describe("ShufflingCache", function () { expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling!); }); - it("should support up to 2 promises at a time", async function () { + it("should support up to 4 promises at a time", async function () { // insert 2 promises at the same epoch shufflingCache.insertPromise(currentEpoch, "0x00"); shufflingCache.insertPromise(currentEpoch, "0x01"); + shufflingCache.insertPromise(currentEpoch, "0x02"); + shufflingCache.insertPromise(currentEpoch, "0x03"); // inserting other promise should throw error - expect(() => shufflingCache.insertPromise(currentEpoch, "0x02")).toThrow(); + expect(() => shufflingCache.insertPromise(currentEpoch, "0x04")).toThrow(); }); }); From a0e7e601c5e291f4074c3db2bb46e96e67cfed68 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Sun, 11 Aug 2024 23:43:06 -0400 Subject: [PATCH 25/79] fix: rebase conflicts --- .../beacon-node/src/chain/shufflingCache.ts | 15 ++++++-- .../state-transition/src/cache/epochCache.ts | 37 +++++++++++++++---- .../src/util/epochShuffling.ts | 11 +++++- .../test/perf/util/shufflings.test.ts | 4 +- 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 60f115ed59ba..eddedb2406d2 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -149,14 +149,15 @@ export class ShufflingCache implements IShufflingCache { epoch: number, decisionRoot: string, state: BeaconStateAllForks, - activeIndices: number[] + activeIndices: number[], + activeIndicesLength: number ): EpochShuffling { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); if (cacheItem && isShufflingCacheItem(cacheItem)) { // this.metrics?.shufflingCache.cacheHitEpochTransition(); return cacheItem.shuffling; } - const shuffling = computeEpochShuffling(state, activeIndices, epoch); + const shuffling = computeEpochShuffling(state, activeIndices, activeIndicesLength, epoch); if (cacheItem) { // this.metrics?.shufflingCache.shufflingPromiseNotResolvedEpochTransition(); cacheItem.resolveFn(shuffling); @@ -170,14 +171,20 @@ export class ShufflingCache implements IShufflingCache { /** * Queue asynchronous build for an EpochShuffling */ - build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: number[]): void { + build( + epoch: number, + decisionRoot: string, + state: BeaconStateAllForks, + activeIndices: number[], + activeIndicesLength: number + ): void { this.insertPromise(epoch, decisionRoot); /** * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations * on a NICE thread with a rust implementation */ setTimeout(() => { - const shuffling = computeEpochShuffling(state, activeIndices, epoch); + const shuffling = computeEpochShuffling(state, activeIndices, activeIndicesLength, epoch); this.set(shuffling, decisionRoot); // wait until after the first slot to help with attestation and block proposal performance }, this.minTimeDelayToBuild); diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 8ef5625628a7..05d573b7545b 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -149,6 +149,7 @@ export class EpochCache { * in case it is not built or the ShufflingCache is not available */ nextActiveIndices: ValidatorIndex[]; + nextActiveIndicesLength: number; /** * Effective balances, for altair processAttestations() */ @@ -241,6 +242,7 @@ export class EpochCache { currentShuffling: EpochShuffling; nextShuffling: EpochShuffling | null; nextActiveIndices: ValidatorIndex[]; + nextActiveIndicesLength: number; effectiveBalanceIncrements: EffectiveBalanceIncrements; totalSlashingsByIncrement: number; syncParticipantReward: number; @@ -272,6 +274,7 @@ export class EpochCache { this.currentShuffling = data.currentShuffling; this.nextShuffling = data.nextShuffling; this.nextActiveIndices = data.nextActiveIndices; + this.nextActiveIndicesLength = data.nextActiveIndicesLength; this.effectiveBalanceIncrements = data.effectiveBalanceIncrements; this.totalSlashingsByIncrement = data.totalSlashingsByIncrement; this.syncParticipantReward = data.syncParticipantReward; @@ -379,7 +382,7 @@ export class EpochCache { if (cachedCurrentShuffling) { currentShuffling = cachedCurrentShuffling; } else { - currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentEpoch); + currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentActiveIndices.length, currentEpoch); shufflingCache?.set(currentShuffling, currentDecisionRoot); // shufflingCache?.metrics?.shufflingCache.miss(); } @@ -391,7 +394,12 @@ export class EpochCache { // TODO: (@matthewkeil) does this need to be added to the cache at previousEpoch and previousDecisionRoot? previousShuffling = currentShuffling; } else { - previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); + previousShuffling = computeEpochShuffling( + state, + previousActiveIndices, + previousActiveIndices.length, + previousEpoch + ); shufflingCache?.set(previousShuffling, previousDecisionRoot); // shufflingCache?.metrics?.shufflingCache.miss(); } @@ -400,7 +408,7 @@ export class EpochCache { if (cachedNextShuffling) { nextShuffling = cachedNextShuffling; } else { - nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); + nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextActiveIndices.length, nextEpoch); shufflingCache?.set(nextShuffling, nextDecisionRoot); // shufflingCache?.metrics?.shufflingCache.miss(); } @@ -499,6 +507,7 @@ export class EpochCache { currentShuffling, nextShuffling, nextActiveIndices, + nextActiveIndicesLength: nextActiveIndices.length, effectiveBalanceIncrements, totalSlashingsByIncrement, syncParticipantReward, @@ -542,6 +551,7 @@ export class EpochCache { currentShuffling: this.currentShuffling, nextShuffling: this.nextShuffling, nextActiveIndices: this.nextActiveIndices, + nextActiveIndicesLength: this.nextActiveIndicesLength, // Uint8Array, requires cloning, but it is cloned only when necessary before an epoch transition // See EpochCache.beforeEpochTransition() effectiveBalanceIncrements: this.effectiveBalanceIncrements, @@ -618,11 +628,12 @@ export class EpochCache { state, // have to use the "nextActiveIndices" that were saved in the last transition here to calculate // the upcoming shuffling if it is not already built (similar condition to the below computation) - this.nextActiveIndices + this.nextActiveIndices, + this.nextActiveIndicesLength ) ?? // allow for this case during testing where the ShufflingCache is not present, may affect perf testing // so should be taken into account when structuring tests. Should not affect unit or other tests though - computeEpochShuffling(state, this.nextActiveIndices, this.epoch); + computeEpochShuffling(state, this.nextActiveIndices, this.nextActiveIndicesLength, this.epoch); } this.proposers = computeProposers( getSeed(state, this.epoch, DOMAIN_BEACON_PROPOSER), @@ -635,11 +646,23 @@ export class EpochCache { */ this.nextDecisionRoot = getShufflingDecisionBlock(state, this.nextEpoch); this.nextActiveIndices = epochTransitionCache.nextEpochShufflingActiveValidatorIndices; + this.nextActiveIndicesLength = epochTransitionCache.nextEpochShufflingActiveIndicesLength; if (this.shufflingCache) { this.nextShuffling = null; - this.shufflingCache?.build(this.nextEpoch, this.nextDecisionRoot, state, this.nextActiveIndices); + this.shufflingCache?.build( + this.nextEpoch, + this.nextDecisionRoot, + state, + this.nextActiveIndices, + this.nextActiveIndicesLength + ); } else { - this.nextShuffling = computeEpochShuffling(state, this.nextActiveIndices, this.nextEpoch); + this.nextShuffling = computeEpochShuffling( + state, + this.nextActiveIndices, + this.nextActiveIndicesLength, + this.nextEpoch + ); } // Only pre-compute the seed since it's very cheap. Do the expensive computeProposers() call only on demand. this.proposersNextEpoch = {computed: false, seed: getSeed(state, this.nextEpoch, DOMAIN_BEACON_PROPOSER)}; diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 561c196715bc..3e98051962e2 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -30,13 +30,20 @@ export interface IShufflingCache { epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, - activeIndices: ValidatorIndex[] + activeIndices: ValidatorIndex[], + activeIndicesLength: number ): EpochShuffling; /** * Queue asynchronous build for an EpochShuffling */ - build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void; + build( + epoch: Epoch, + decisionRoot: RootHex, + state: BeaconStateAllForks, + activeIndices: ValidatorIndex[], + activeIndicesLength: number + ): void; /** * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will diff --git a/packages/state-transition/test/perf/util/shufflings.test.ts b/packages/state-transition/test/perf/util/shufflings.test.ts index 5077f0ef6525..a18d46e5ff25 100644 --- a/packages/state-transition/test/perf/util/shufflings.test.ts +++ b/packages/state-transition/test/perf/util/shufflings.test.ts @@ -35,8 +35,8 @@ describe("epoch shufflings", () => { itBench({ id: `computeEpochShuffling - vc ${numValidators}`, fn: () => { - const {activeIndices} = state.epochCtx.nextShuffling; - computeEpochShuffling(state, activeIndices, activeIndices.length, nextEpoch); + const {nextActiveIndices} = state.epochCtx; + computeEpochShuffling(state, nextActiveIndices, nextActiveIndices.length, nextEpoch); }, }); From ff7df85a442dfd87ff5bed9208dc28382713c905 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Sun, 11 Aug 2024 23:58:21 -0400 Subject: [PATCH 26/79] fix: changes from debugging sim tests --- packages/beacon-node/src/chain/chain.ts | 29 ++++---------- .../beacon-node/src/chain/shufflingCache.ts | 4 +- .../stateCache/persistentCheckpointsCache.ts | 2 +- packages/params/src/index.ts | 2 + .../state-transition/src/cache/epochCache.ts | 11 +++--- .../state-transition/src/cache/stateCache.ts | 4 +- .../src/util/computeAnchorCheckpoint.ts | 38 +++++++++++++++++++ .../src/util/epochShuffling.ts | 18 ++++++++- 8 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 packages/state-transition/src/util/computeAnchorCheckpoint.ts diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 3e39a22ac831..78327c00e53a 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -254,19 +254,13 @@ export class BeaconChain implements IBeaconChain { config, pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - shufflingCache: this.shufflingCache, }); - // If shufflingCache was not passed to EpochCache, add it now and cache shuffling from context. Happens in most - // situations including checkpoint sync - if (!cachedState.epochCtx.shufflingCache) { - cachedState.epochCtx.shufflingCache = this.shufflingCache; - this.shufflingCache.set(cachedState.epochCtx.currentShuffling, cachedState.epochCtx.currentDecisionRoot); - if (cachedState.epochCtx.currentShuffling !== cachedState.epochCtx.previousShuffling) { - this.shufflingCache.set(cachedState.epochCtx.previousShuffling, cachedState.epochCtx.previousDecisionRoot); - } - if (cachedState.epochCtx.nextShuffling) { - this.shufflingCache.set(cachedState.epochCtx.nextShuffling, cachedState.epochCtx.nextDecisionRoot); - } + + cachedState.epochCtx.shufflingCache = this.shufflingCache; + this.shufflingCache.set(cachedState.epochCtx.previousShuffling, cachedState.epochCtx.previousDecisionRoot); + this.shufflingCache.set(cachedState.epochCtx.currentShuffling, cachedState.epochCtx.currentDecisionRoot); + if (cachedState.epochCtx.nextShuffling !== null) { + this.shufflingCache.set(cachedState.epochCtx.nextShuffling, cachedState.epochCtx.nextDecisionRoot); } // Persist single global instance of state caches @@ -903,15 +897,8 @@ export class BeaconChain implements IBeaconChain { state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller); } - // resolve the promise to unblock other calls of the same epoch and dependent root - const decisionRoot = getShufflingDecisionBlock(state, attEpoch); - const shuffling = await this.shufflingCache.get(attEpoch, decisionRoot); - if (!shuffling) { - // This will be essentially unreachable considering regen should build the shuffling for this epoch - // but need to handle anyhow - throw Error(`UNREACHABLE: Shuffling not found for attestation epoch ${attEpoch} decisionRoot ${decisionRoot}`); - } - return shuffling; + // should always be the current epoch of the active context so no need to await a result from the ShufflingCache + return state.epochCtx.getShufflingAtEpoch(attEpoch); } /** diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index eddedb2406d2..b20cc37d54b0 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -13,9 +13,9 @@ const MAX_EPOCHS = 4; /** * With default chain option of maxSkipSlots = 32, there should be no shuffling promise. If that happens a lot, it could blow up Lodestar, - * with MAX_EPOCHS = 4, only allow 2 promise at a time. Note that regen already bounds number of concurrent requests at 1 already. + * with MAX_EPOCHS = 2, only allow 2 promise at a time. Note that regen already bounds number of concurrent requests at 1 already. */ -const MAX_PROMISES = 4; +const MAX_PROMISES = 2; const DEFAULT_MIN_TIME_DELAY_IN_MS = 5; diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 0f89bb0394b1..25bba27d5045 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -212,7 +212,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } sszTimer?.(); const timer = this.metrics?.stateReloadDuration.startTimer(); - const newCachedState = loadCachedBeaconState(seedState, stateBytes, this.shufflingCache, {}, validatorsBytes); + const newCachedState = loadCachedBeaconState(seedState, stateBytes, {}, validatorsBytes); newCachedState.commit(); const stateRoot = toHexString(newCachedState.hashTreeRoot()); timer?.(); diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 6a95e3ca632e..4218d14629df 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -109,6 +109,8 @@ export const FAR_FUTURE_EPOCH = Infinity; export const BASE_REWARDS_PER_EPOCH = 4; export const DEPOSIT_CONTRACT_TREE_DEPTH = 2 ** 5; // 32 export const JUSTIFICATION_BITS_LENGTH = 4; +export const ZERO_HASH = Buffer.alloc(32, 0); +export const ZERO_HASH_HEX = "0x" + "00".repeat(32); // Withdrawal prefixes // Since the prefixes are just 1 byte, we define and use them as number diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 05d573b7545b..f71f5b2b66fa 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -44,6 +44,7 @@ import { SyncCommitteeCache, SyncCommitteeCacheEmpty, } from "./syncCommitteeCache.js"; +import {CachedBeaconStateAllForks} from "./stateCache.js"; /** `= PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)` */ export const PROPOSER_WEIGHT_FACTOR = PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT); @@ -330,11 +331,11 @@ export class EpochCache { // BeaconChain could provide a shuffling cache to avoid re-computing shuffling every epoch // in that case, we don't need to compute shufflings again - const previousDecisionRoot = getShufflingDecisionBlock(state, previousEpoch); + const previousDecisionRoot = getShufflingDecisionBlock(config, state, previousEpoch); const cachedPreviousShuffling = shufflingCache?.getSync(previousEpoch, previousDecisionRoot); - const currentDecisionRoot = getShufflingDecisionBlock(state, currentEpoch); + const currentDecisionRoot = getShufflingDecisionBlock(config, state, currentEpoch); const cachedCurrentShuffling = shufflingCache?.getSync(currentEpoch, currentDecisionRoot); - const nextDecisionRoot = getShufflingDecisionBlock(state, nextEpoch); + const nextDecisionRoot = getShufflingDecisionBlock(config, state, nextEpoch); const cachedNextShuffling = shufflingCache?.getSync(nextEpoch, nextDecisionRoot); for (let i = 0; i < validatorCount; i++) { @@ -584,7 +585,7 @@ export class EpochCache { * 1) update previous/current/next values of cached items */ afterProcessEpoch( - state: BeaconStateAllForks, + state: CachedBeaconStateAllForks, epochTransitionCache: { nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; nextEpochShufflingActiveIndicesLength: number; @@ -644,7 +645,7 @@ export class EpochCache { /** * Calculate look-ahead values for n+2 (will be n+1 after transition finishes) */ - this.nextDecisionRoot = getShufflingDecisionBlock(state, this.nextEpoch); + this.nextDecisionRoot = getShufflingDecisionBlock(state.config, state, this.nextEpoch); this.nextActiveIndices = epochTransitionCache.nextEpochShufflingActiveValidatorIndices; this.nextActiveIndicesLength = epochTransitionCache.nextEpochShufflingActiveIndicesLength; if (this.shufflingCache) { diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index 0c932391c92b..e3b435c684d8 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -1,7 +1,6 @@ import {PublicKey} from "@chainsafe/blst"; import {BeaconConfig} from "@lodestar/config"; import {loadState} from "../util/loadState/loadState.js"; -import {IShufflingCache} from "../util/epochShuffling.js"; import {EpochCache, EpochCacheImmutableData, EpochCacheOpts} from "./epochCache.js"; import { BeaconStateAllForks, @@ -164,7 +163,6 @@ export function createCachedBeaconState( export function loadCachedBeaconState( cachedSeedState: T, stateBytes: Uint8Array, - shufflingCache: IShufflingCache, // should not be optional because this is used during node operation opts?: EpochCacheOpts, seedValidatorsBytes?: Uint8Array ): T { @@ -174,7 +172,7 @@ export function loadCachedBeaconState GENESIS_SLOT + ? getDecisionBlock(state, epoch) + : toHexString(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(config, state).blockHeader)); +} From 746841942ca6a9e21426787b2e819d01113e4b79 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 13 Aug 2024 01:12:26 -0400 Subject: [PATCH 27/79] refactor: minimize changes in afterProcessEpoch --- .../state-transition/src/cache/epochCache.ts | 69 ++++++++----------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index f71f5b2b66fa..35bcc0ef4e2c 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -592,30 +592,15 @@ export class EpochCache { nextEpochTotalActiveBalanceByIncrement: number; } ): void { - // Advance time units - // state.slot is advanced right before calling this function - // ``` - // postState.slot++; - // afterProcessEpoch(postState, epochTransitionCache); - // ``` + const upcomingEpoch = this.nextEpoch; + const epochAfterNext = upcomingEpoch + 1; - // After updating the "current" epoch the "cached epoch" will be out of sync with the clock - // epoch until the clock catches up (should be less than a second or so). Because this function - // and the remainder of the state-transition is all synchronous there is no chance that this value - // can be read incorrectly, however, if the synchronous-only paradigm changes it must be taken - // into account - this.epoch = this.nextEpoch; - // TODO: why was this what was used @twoeths? and not a simple increment like above if all the other - // stuff in this function was swapped curr->prev and next->curr? - // this.epoch = computeEpochAtSlot(state.slot); - - // Move current values to previous values as they are no longer for the current epoch - this.previousDecisionRoot = this.currentDecisionRoot; + // move current to previous this.previousShuffling = this.currentShuffling; - // Roll current proposers into previous proposers for metrics + this.previousDecisionRoot = this.currentDecisionRoot; this.proposersPrevEpoch = this.proposers; - // Move next values to be the current values of the upcoming epoch after epoch transition finishes + // move next to current or calculate upcoming this.currentDecisionRoot = this.nextDecisionRoot; if (this.nextShuffling) { // was already pulled by the api or another method on EpochCache @@ -624,8 +609,8 @@ export class EpochCache { } else { this.currentShuffling = this.shufflingCache?.getOrBuildSync( - this.epoch, - this.currentDecisionRoot, + upcomingEpoch, + this.nextDecisionRoot, state, // have to use the "nextActiveIndices" that were saved in the last transition here to calculate // the upcoming shuffling if it is not already built (similar condition to the below computation) @@ -634,24 +619,20 @@ export class EpochCache { ) ?? // allow for this case during testing where the ShufflingCache is not present, may affect perf testing // so should be taken into account when structuring tests. Should not affect unit or other tests though - computeEpochShuffling(state, this.nextActiveIndices, this.nextActiveIndicesLength, this.epoch); + computeEpochShuffling(state, this.nextActiveIndices, this.nextActiveIndicesLength, upcomingEpoch); } - this.proposers = computeProposers( - getSeed(state, this.epoch, DOMAIN_BEACON_PROPOSER), - this.currentShuffling, - this.effectiveBalanceIncrements - ); + const upcomingProposerSeed = getSeed(state, upcomingEpoch, DOMAIN_BEACON_PROPOSER); + // next epoch was moved to current epoch so use current here + this.proposers = computeProposers(upcomingProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements); - /** - * Calculate look-ahead values for n+2 (will be n+1 after transition finishes) - */ - this.nextDecisionRoot = getShufflingDecisionBlock(state.config, state, this.nextEpoch); + // calculate next values + this.nextDecisionRoot = getShufflingDecisionBlock(state.config, state, epochAfterNext); this.nextActiveIndices = epochTransitionCache.nextEpochShufflingActiveValidatorIndices; this.nextActiveIndicesLength = epochTransitionCache.nextEpochShufflingActiveIndicesLength; if (this.shufflingCache) { this.nextShuffling = null; this.shufflingCache?.build( - this.nextEpoch, + epochAfterNext, this.nextDecisionRoot, state, this.nextActiveIndices, @@ -662,15 +643,16 @@ export class EpochCache { state, this.nextActiveIndices, this.nextActiveIndicesLength, - this.nextEpoch + epochAfterNext ); } + // Only pre-compute the seed since it's very cheap. Do the expensive computeProposers() call only on demand. - this.proposersNextEpoch = {computed: false, seed: getSeed(state, this.nextEpoch, DOMAIN_BEACON_PROPOSER)}; + this.proposersNextEpoch = {computed: false, seed: getSeed(state, epochAfterNext, DOMAIN_BEACON_PROPOSER)}; // TODO: DEDUPLICATE from createEpochCache // - // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute every time the + // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute everytime the // active validator indices set changes in size. Validators change active status only when: // - validator.activation_epoch is set. Only changes in process_registry_updates() if validator can be activated. If // the value changes it will be set to `epoch + 1 + MAX_SEED_LOOKAHEAD`. @@ -692,14 +674,14 @@ export class EpochCache { ); // Maybe advance exitQueueEpoch at the end of the epoch if there haven't been any exists for a while - const exitQueueEpoch = computeActivationExitEpoch(this.epoch); + const exitQueueEpoch = computeActivationExitEpoch(upcomingEpoch); if (exitQueueEpoch > this.exitQueueEpoch) { this.exitQueueEpoch = exitQueueEpoch; this.exitQueueChurn = 0; } this.totalActiveBalanceIncrements = epochTransitionCache.nextEpochTotalActiveBalanceByIncrement; - if (this.epoch >= this.config.ALTAIR_FORK_EPOCH) { + if (upcomingEpoch >= this.config.ALTAIR_FORK_EPOCH) { this.syncParticipantReward = computeSyncParticipantReward(this.totalActiveBalanceIncrements); this.syncProposerReward = Math.floor(this.syncParticipantReward * PROPOSER_WEIGHT_FACTOR); this.baseRewardPerIncrement = computeBaseRewardPerIncrement(this.totalActiveBalanceIncrements); @@ -708,6 +690,13 @@ export class EpochCache { this.previousTargetUnslashedBalanceIncrements = this.currentTargetUnslashedBalanceIncrements; this.currentTargetUnslashedBalanceIncrements = 0; + // Advance time units + // state.slot is advanced right before calling this function + // ``` + // postState.slot++; + // afterProcessEpoch(postState, epochTransitionCache); + // ``` + this.epoch = computeEpochAtSlot(state.slot); this.syncPeriod = computeSyncPeriodAtEpoch(this.epoch); } @@ -1042,14 +1031,12 @@ export class EpochCacheError extends LodestarError {} export function createEmptyEpochCacheImmutableData( chainConfig: ChainConfig, - state: Pick, - shufflingCache?: IShufflingCache + state: Pick ): EpochCacheImmutableData { return { config: createBeaconConfig(chainConfig, state.genesisValidatorsRoot), // This is a test state, there's no need to have a global shared cache of keys pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - shufflingCache, }; } From 9e5e4b087d27453db9bb75eeea039b9291a51746 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 13 Aug 2024 01:15:33 -0400 Subject: [PATCH 28/79] chore: fix lint --- packages/beacon-node/src/chain/chain.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 78327c00e53a..93725441751e 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -13,7 +13,6 @@ import { PubkeyIndexMap, EpochShuffling, computeEndSlotAtEpoch, - getShufflingDecisionBlock, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import { From 198856843367c79c96f93c5ef4baff96bac373e1 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 13 Aug 2024 01:19:12 -0400 Subject: [PATCH 29/79] chore: fix check-types --- .../test/unit/chain/shufflingCache.test.ts | 12 ++++++------ .../test/unit/cachedBeaconState.test.ts | 12 +++--------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index 6479bafbe0e0..ab0e62712e82 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -20,12 +20,12 @@ describe("ShufflingCache", function () { }); it("should get shuffling from cache", async function () { - const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); + const decisionRoot = getShufflingDecisionBlock(state.config, state, currentEpoch); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); }); it("should bound by maxSize(=1)", async function () { - const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); + const decisionRoot = getShufflingDecisionBlock(state.config, state, currentEpoch); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert promises at the same epoch does not prune the cache shufflingCache.insertPromise(currentEpoch, "0x00"); @@ -37,7 +37,7 @@ describe("ShufflingCache", function () { }); it("should return shuffling from promise", async function () { - const nextDecisionRoot = getShufflingDecisionBlock(state, nextEpoch); + const nextDecisionRoot = getShufflingDecisionBlock(state.config, state, nextEpoch); shufflingCache.insertPromise(nextEpoch, nextDecisionRoot); const shufflingRequest0 = shufflingCache.get(nextEpoch, nextDecisionRoot); const shufflingRequest1 = shufflingCache.get(nextEpoch, nextDecisionRoot); @@ -46,12 +46,12 @@ describe("ShufflingCache", function () { expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling!); }); - it("should support up to 4 promises at a time", async function () { + it("should support up to 2 promises at a time", async function () { // insert 2 promises at the same epoch shufflingCache.insertPromise(currentEpoch, "0x00"); shufflingCache.insertPromise(currentEpoch, "0x01"); - shufflingCache.insertPromise(currentEpoch, "0x02"); - shufflingCache.insertPromise(currentEpoch, "0x03"); + // shufflingCache.insertPromise(currentEpoch, "0x02"); + // shufflingCache.insertPromise(currentEpoch, "0x03"); // inserting other promise should throw error expect(() => shufflingCache.insertPromise(currentEpoch, "0x04")).toThrow(); }); diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index faa2ff08ee65..6ea3e9ce8a38 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -8,7 +8,6 @@ import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; import {interopPubkeysCached} from "../utils/interop.js"; import {modifyStateSameValidator, newStateWithValidators} from "../utils/capella.js"; -import {IShufflingCache} from "../../src/index.js"; describe("CachedBeaconState", () => { it("Clone and mutate", () => { @@ -129,14 +128,9 @@ describe("CachedBeaconState", () => { // confirm loadState() result const stateBytes = state.serialize(); - const newCachedState = loadCachedBeaconState( - seedState, - stateBytes, - seedState.epochCtx.shufflingCache as IShufflingCache, - { - skipSyncCommitteeCache: true, - } - ); + const newCachedState = loadCachedBeaconState(seedState, stateBytes, { + skipSyncCommitteeCache: true, + }); const newStateBytes = newCachedState.serialize(); expect(newStateBytes).toEqual(stateBytes); expect(newCachedState.hashTreeRoot()).toEqual(state.hashTreeRoot()); From 955c66c10994790fc9754bb56bbc609b98fcf9c0 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 13 Aug 2024 01:25:53 -0400 Subject: [PATCH 30/79] chore: fix check-types --- .../beacon-node/test/spec/presets/genesis.test.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/genesis.test.ts b/packages/beacon-node/test/spec/presets/genesis.test.ts index 30d361e58786..0732d2bd7ec0 100644 --- a/packages/beacon-node/test/spec/presets/genesis.test.ts +++ b/packages/beacon-node/test/spec/presets/genesis.test.ts @@ -19,7 +19,7 @@ import {getConfig} from "../../utils/config.js"; import {RunnerType} from "../utils/types.js"; import {specTestIterator} from "../utils/specTestIterator.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; -import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; +// import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; // The aim of the genesis tests is to provide a baseline to test genesis-state initialization and test if the // proposed genesis-validity conditions are working. @@ -43,15 +43,10 @@ const genesisInitialization: TestRunnerFn Date: Thu, 15 Aug 2024 02:41:16 -0400 Subject: [PATCH 31/79] feat: add diff utility --- packages/utils/src/diff.ts | 232 ++++++++++++++++++++++++++++++++++++ packages/utils/src/index.ts | 1 + 2 files changed, 233 insertions(+) create mode 100644 packages/utils/src/diff.ts diff --git a/packages/utils/src/diff.ts b/packages/utils/src/diff.ts new file mode 100644 index 000000000000..9ffa8cfe6126 --- /dev/null +++ b/packages/utils/src/diff.ts @@ -0,0 +1,232 @@ +/* eslint-disable no-console */ +import fs from "node:fs"; + +const primitiveTypeof = ["number", "string", "bigint", "boolean"]; +export type BufferType = Uint8Array | Uint32Array; +export type PrimitiveType = number | string | bigint | boolean | BufferType; +export type DiffableCollection = Record; +export type Diffable = PrimitiveType | Array | DiffableCollection; + +export interface Diff { + objectPath: string; + errorMessage?: string; + val1: Diffable; + val2: Diffable; +} + +export function diffUint8Array(val1: Uint8Array, val2: PrimitiveType, objectPath: string): Diff[] { + if (!(val2 instanceof Uint8Array)) { + return [ + { + objectPath, + errorMessage: `val1${objectPath} is a Uint8Array, but val2${objectPath} is not`, + val1, + val2, + }, + ]; + } + const hex1 = Buffer.from(val1).toString("hex"); + const hex2 = Buffer.from(val2).toString("hex"); + if (hex1 !== hex2) { + return [ + { + objectPath, + val1: `0x${hex1}`, + val2: `0x${hex2}`, + }, + ]; + } + return []; +} + +export function diffUint32Array(val1: Uint32Array, val2: PrimitiveType, objectPath: string): Diff[] { + if (!(val2 instanceof Uint32Array)) { + return [ + { + objectPath, + errorMessage: `val1${objectPath} is a Uint32Array, but val2${objectPath} is not`, + val1, + val2, + }, + ]; + } + const diffs: Diff[] = []; + val1.forEach((value, index) => { + const value2 = val2[index]; + if (value !== value2) { + diffs.push({ + objectPath: `${objectPath}[${index}]`, + val1: `0x${value.toString(16).padStart(8, "0")}`, + val2: value2 ? `0x${val2[index].toString(16).padStart(8, "0")}` : "undefined", + }); + } + }); + return diffs; +} + +function diffPrimitiveValue(val1: PrimitiveType, val2: PrimitiveType, objectPath: string): Diff[] { + if (val1 instanceof Uint8Array) { + return diffUint8Array(val1, val2, objectPath); + } + if (val1 instanceof Uint32Array) { + return diffUint32Array(val1, val2, objectPath); + } + + const diff = {objectPath, val1, val2} as Diff; + const type1 = typeof val1; + if (!primitiveTypeof.includes(type1)) { + diff.errorMessage = `val1${objectPath} is not a supported type`; + } + const type2 = typeof val2; + if (!primitiveTypeof.includes(type2)) { + diff.errorMessage = `val2${objectPath} is not a supported type`; + } + if (type1 !== type2) { + diff.errorMessage = `val1${objectPath} is not the same type as val2${objectPath}`; + } + if (val1 !== val2) { + return [diff]; + } + return []; +} + +function isPrimitiveValue(val: unknown): val is PrimitiveType { + if (Array.isArray(val)) return false; + if (typeof val === "object") { + return val instanceof Uint8Array || val instanceof Uint32Array; + } + return true; +} + +function isDiffable(val: unknown): val is Diffable { + return !(typeof val === "function" || typeof val === "symbol" || typeof val === "undefined" || val === null); +} + +export function getDiffs(val1: Diffable, val2: Diffable, objectPath: string): Diff[] { + if (isPrimitiveValue(val1)) { + if (!isPrimitiveValue(val2)) { + return [ + { + objectPath, + errorMessage: `val1${objectPath} is a primitive value and val2${objectPath} is not`, + val1, + val2, + }, + ]; + } + return diffPrimitiveValue(val1, val2, objectPath); + } + + const isArray = Array.isArray(val1); + let errorMessage: string | undefined; + if (isArray && !Array.isArray(val2)) { + errorMessage = `val1${objectPath} is an array and val2${objectPath} is not`; + } else if (typeof val1 === "object" && typeof val2 !== "object") { + errorMessage = `val1${objectPath} is a nested object and val2${objectPath} is not`; + } + if (errorMessage) { + return [ + { + objectPath, + errorMessage, + val1, + val2, + }, + ]; + } + + const diffs: Diff[] = []; + for (const [index, value] of Object.entries(val1)) { + if (!isDiffable(value)) { + diffs.push({objectPath, val1, val2, errorMessage: `val1${objectPath} is not Diffable`}); + continue; + } + const value2 = (val2 as DiffableCollection)[index]; + if (!isDiffable(value2)) { + diffs.push({objectPath, val1, val2, errorMessage: `val2${objectPath} is not Diffable`}); + continue; + } + const innerPath = isArray ? `${objectPath}[${index}]` : `${objectPath}.${index}`; + diffs.push(...getDiffs(value, value2, innerPath)); + } + return diffs; +} + +/** + * Find the different values on complex, nested objects. Outputs the path through the object to + * each value that does not match from val1 and val2. Optionally can output the values that differ. + * + * For objects that differ greatly, can write to a file instead of the terminal for analysis + * + * ## Example + * ```ts + * const obj1 = { + * key1: { + * key2: [ + * { key3: 1 }, + * { key3: new Uint8Array([1, 2, 3]) } + * ] + * }, + * key4: new Uint8Array([1, 2, 3]), + * key5: 362436 + * }; + * + * const obj2 = { + * key1: { + * key2: [ + * { key3: 1 }, + * { key3: new Uint8Array([1, 2, 4]) } + * ] + * }, + * key4: new Uint8Array([1, 2, 4]) + * key5: true + * }; + * + * diffObjects(obj1, obj2, true); + * + * + * ``` + * + * ## Output + * ```sh + * val.key1.key2[1].key3 + * - 0x010203 + * - 0x010204 + * val.key4 + * - 0x010203 + * - 0x010204 + * val.key5 + * val1.key5 is not the same type as val2.key5 + * - 362436 + * - true + * ``` + */ +export function diff(val1: unknown, val2: unknown, outputValues = false, filename?: string): void { + if (!isDiffable(val1)) { + console.log("val1 is not Diffable"); + return; + } + if (!isDiffable(val2)) { + console.log("val2 is not Diffable"); + return; + } + const diffs = getDiffs(val1, val2, ""); + let output = ""; + if (diffs.length) { + diffs.forEach((diff) => { + let diffOutput = `value${diff.objectPath}`; + if (diff.errorMessage) { + diffOutput += `\n ${diff.errorMessage}`; + } + if (outputValues) { + diffOutput += `\n - ${diff.val1.toString()}\n - ${diff.val2.toString()}\n`; + } + output += `${diffOutput}\n`; + }); + if (filename) { + fs.writeFileSync(filename, output); + } else { + console.log(output); + } + } +} diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 2057e50e07bc..5ba509047829 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -3,6 +3,7 @@ export * from "./assert.js"; export * from "./base64.js"; export * from "./bytes.js"; export * from "./command.js"; +export * from "./diff.js"; export * from "./err.js"; export * from "./errors.js"; export * from "./format.js"; From 529d85c2d266a6768f7912f0ebacde9b6189d346 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 15 Aug 2024 03:01:37 -0400 Subject: [PATCH 32/79] fix: bug in spec tests from invalid nextActiveIndices --- .../beacon-node/src/chain/shufflingCache.ts | 6 ++--- .../state-transition/src/cache/epochCache.ts | 23 ++++++++++++++----- .../src/util/epochShuffling.ts | 12 ++-------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index b20cc37d54b0..17922adb0fc1 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -1,5 +1,5 @@ import {BeaconStateAllForks, EpochShuffling, IShufflingCache, computeEpochShuffling} from "@lodestar/state-transition"; -import {Epoch, RootHex} from "@lodestar/types"; +import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; import {LodestarError, MapDef, pruneSetToMax} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; @@ -149,7 +149,7 @@ export class ShufflingCache implements IShufflingCache { epoch: number, decisionRoot: string, state: BeaconStateAllForks, - activeIndices: number[], + activeIndices: ValidatorIndex[], activeIndicesLength: number ): EpochShuffling { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); @@ -175,7 +175,7 @@ export class ShufflingCache implements IShufflingCache { epoch: number, decisionRoot: string, state: BeaconStateAllForks, - activeIndices: number[], + activeIndices: ValidatorIndex[], activeIndicesLength: number ): void { this.insertPromise(epoch, decisionRoot); diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 35bcc0ef4e2c..195ebed4dca0 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -593,7 +593,7 @@ export class EpochCache { } ): void { const upcomingEpoch = this.nextEpoch; - const epochAfterNext = upcomingEpoch + 1; + const epochAfterUpcoming = upcomingEpoch + 1; // move current to previous this.previousShuffling = this.currentShuffling; @@ -626,13 +626,24 @@ export class EpochCache { this.proposers = computeProposers(upcomingProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements); // calculate next values - this.nextDecisionRoot = getShufflingDecisionBlock(state.config, state, epochAfterNext); - this.nextActiveIndices = epochTransitionCache.nextEpochShufflingActiveValidatorIndices; + this.nextDecisionRoot = getShufflingDecisionBlock(state.config, state, epochAfterUpcoming); this.nextActiveIndicesLength = epochTransitionCache.nextEpochShufflingActiveIndicesLength; + this.nextActiveIndices = new Array(this.nextActiveIndicesLength); + + if (this.nextActiveIndicesLength > epochTransitionCache.nextEpochShufflingActiveValidatorIndices.length) { + throw new Error( + `Invalid activeValidatorCount: ${this.nextActiveIndicesLength} > ${epochTransitionCache.nextEpochShufflingActiveValidatorIndices.length}` + ); + } + // only the first `activeValidatorCount` elements are copied to `activeIndices` + for (let i = 0; i < this.nextActiveIndicesLength; i++) { + this.nextActiveIndices[i] = epochTransitionCache.nextEpochShufflingActiveValidatorIndices[i]; + } + if (this.shufflingCache) { this.nextShuffling = null; this.shufflingCache?.build( - epochAfterNext, + epochAfterUpcoming, this.nextDecisionRoot, state, this.nextActiveIndices, @@ -643,12 +654,12 @@ export class EpochCache { state, this.nextActiveIndices, this.nextActiveIndicesLength, - epochAfterNext + epochAfterUpcoming ); } // Only pre-compute the seed since it's very cheap. Do the expensive computeProposers() call only on demand. - this.proposersNextEpoch = {computed: false, seed: getSeed(state, epochAfterNext, DOMAIN_BEACON_PROPOSER)}; + this.proposersNextEpoch = {computed: false, seed: getSeed(state, epochAfterUpcoming, DOMAIN_BEACON_PROPOSER)}; // TODO: DEDUPLICATE from createEpochCache // diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index da2a46d3c73e..312b5d54eb90 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -103,20 +103,12 @@ export function computeCommitteeCount(activeValidatorCount: number): number { export function computeEpochShuffling( state: BeaconStateAllForks, - activeIndices: ArrayLike, + activeIndices: ValidatorIndex[], activeValidatorCount: number, epoch: Epoch ): EpochShuffling { const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); - - if (activeValidatorCount > activeIndices.length) { - throw new Error(`Invalid activeValidatorCount: ${activeValidatorCount} > ${activeIndices.length}`); - } - // only the first `activeValidatorCount` elements are copied to `activeIndices` - const _activeIndices = new Uint32Array(activeValidatorCount); - for (let i = 0; i < activeValidatorCount; i++) { - _activeIndices[i] = activeIndices[i]; - } + const _activeIndices = new Uint32Array(activeIndices); const shuffling = _activeIndices.slice(); unshuffleList(shuffling, seed); From dbf8ff17adfaee04730f4d786669f8632958c479 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 15 Aug 2024 03:02:03 -0400 Subject: [PATCH 33/79] refactor: add/remove comments --- .../beacon-node/test/unit/chain/shufflingCache.test.ts | 2 -- packages/utils/src/diff.ts | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index ab0e62712e82..b0e6257a109a 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -50,8 +50,6 @@ describe("ShufflingCache", function () { // insert 2 promises at the same epoch shufflingCache.insertPromise(currentEpoch, "0x00"); shufflingCache.insertPromise(currentEpoch, "0x01"); - // shufflingCache.insertPromise(currentEpoch, "0x02"); - // shufflingCache.insertPromise(currentEpoch, "0x03"); // inserting other promise should throw error expect(() => shufflingCache.insertPromise(currentEpoch, "0x04")).toThrow(); }); diff --git a/packages/utils/src/diff.ts b/packages/utils/src/diff.ts index 9ffa8cfe6126..204989016b46 100644 --- a/packages/utils/src/diff.ts +++ b/packages/utils/src/diff.ts @@ -167,7 +167,7 @@ export function getDiffs(val1: Diffable, val2: Diffable, objectPath: string): Di * { key3: new Uint8Array([1, 2, 3]) } * ] * }, - * key4: new Uint8Array([1, 2, 3]), + * key4: new Uint32Array([1, 2, 3]), * key5: 362436 * }; * @@ -178,7 +178,7 @@ export function getDiffs(val1: Diffable, val2: Diffable, objectPath: string): Di * { key3: new Uint8Array([1, 2, 4]) } * ] * }, - * key4: new Uint8Array([1, 2, 4]) + * key4: new Uint32Array([1, 2, 4]) * key5: true * }; * @@ -192,9 +192,9 @@ export function getDiffs(val1: Diffable, val2: Diffable, objectPath: string): Di * val.key1.key2[1].key3 * - 0x010203 * - 0x010204 - * val.key4 - * - 0x010203 - * - 0x010204 + * val.key4[2] + * - 0x00000003 + * - 0x00000004 * val.key5 * val1.key5 is not the same type as val2.key5 * - 362436 From 5540ce11a3217454380d59a3860200bcd74b7ff0 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 15 Aug 2024 23:44:03 -0400 Subject: [PATCH 34/79] refactor: remove this.activeIndicesLength from EpochCache --- .../beacon-node/src/chain/shufflingCache.ts | 17 ++----- .../state-transition/src/cache/epochCache.ts | 48 +++++-------------- .../src/util/epochShuffling.ts | 16 ++----- .../test/perf/util/shufflings.test.ts | 2 +- 4 files changed, 24 insertions(+), 59 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 17922adb0fc1..afde4cccf956 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -13,7 +13,7 @@ const MAX_EPOCHS = 4; /** * With default chain option of maxSkipSlots = 32, there should be no shuffling promise. If that happens a lot, it could blow up Lodestar, - * with MAX_EPOCHS = 2, only allow 2 promise at a time. Note that regen already bounds number of concurrent requests at 1 already. + * with MAX_EPOCHS = 4, only allow 2 promise at a time. Note that regen already bounds number of concurrent requests at 1 already. */ const MAX_PROMISES = 2; @@ -149,15 +149,14 @@ export class ShufflingCache implements IShufflingCache { epoch: number, decisionRoot: string, state: BeaconStateAllForks, - activeIndices: ValidatorIndex[], - activeIndicesLength: number + activeIndices: ValidatorIndex[] ): EpochShuffling { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); if (cacheItem && isShufflingCacheItem(cacheItem)) { // this.metrics?.shufflingCache.cacheHitEpochTransition(); return cacheItem.shuffling; } - const shuffling = computeEpochShuffling(state, activeIndices, activeIndicesLength, epoch); + const shuffling = computeEpochShuffling(state, activeIndices, epoch); if (cacheItem) { // this.metrics?.shufflingCache.shufflingPromiseNotResolvedEpochTransition(); cacheItem.resolveFn(shuffling); @@ -171,20 +170,14 @@ export class ShufflingCache implements IShufflingCache { /** * Queue asynchronous build for an EpochShuffling */ - build( - epoch: number, - decisionRoot: string, - state: BeaconStateAllForks, - activeIndices: ValidatorIndex[], - activeIndicesLength: number - ): void { + build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void { this.insertPromise(epoch, decisionRoot); /** * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations * on a NICE thread with a rust implementation */ setTimeout(() => { - const shuffling = computeEpochShuffling(state, activeIndices, activeIndicesLength, epoch); + const shuffling = computeEpochShuffling(state, activeIndices, epoch); this.set(shuffling, decisionRoot); // wait until after the first slot to help with attestation and block proposal performance }, this.minTimeDelayToBuild); diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 195ebed4dca0..79c8733d4c2e 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -150,7 +150,6 @@ export class EpochCache { * in case it is not built or the ShufflingCache is not available */ nextActiveIndices: ValidatorIndex[]; - nextActiveIndicesLength: number; /** * Effective balances, for altair processAttestations() */ @@ -243,7 +242,6 @@ export class EpochCache { currentShuffling: EpochShuffling; nextShuffling: EpochShuffling | null; nextActiveIndices: ValidatorIndex[]; - nextActiveIndicesLength: number; effectiveBalanceIncrements: EffectiveBalanceIncrements; totalSlashingsByIncrement: number; syncParticipantReward: number; @@ -275,7 +273,6 @@ export class EpochCache { this.currentShuffling = data.currentShuffling; this.nextShuffling = data.nextShuffling; this.nextActiveIndices = data.nextActiveIndices; - this.nextActiveIndicesLength = data.nextActiveIndicesLength; this.effectiveBalanceIncrements = data.effectiveBalanceIncrements; this.totalSlashingsByIncrement = data.totalSlashingsByIncrement; this.syncParticipantReward = data.syncParticipantReward; @@ -383,7 +380,7 @@ export class EpochCache { if (cachedCurrentShuffling) { currentShuffling = cachedCurrentShuffling; } else { - currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentActiveIndices.length, currentEpoch); + currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentEpoch); shufflingCache?.set(currentShuffling, currentDecisionRoot); // shufflingCache?.metrics?.shufflingCache.miss(); } @@ -395,12 +392,7 @@ export class EpochCache { // TODO: (@matthewkeil) does this need to be added to the cache at previousEpoch and previousDecisionRoot? previousShuffling = currentShuffling; } else { - previousShuffling = computeEpochShuffling( - state, - previousActiveIndices, - previousActiveIndices.length, - previousEpoch - ); + previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); shufflingCache?.set(previousShuffling, previousDecisionRoot); // shufflingCache?.metrics?.shufflingCache.miss(); } @@ -409,7 +401,7 @@ export class EpochCache { if (cachedNextShuffling) { nextShuffling = cachedNextShuffling; } else { - nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextActiveIndices.length, nextEpoch); + nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); shufflingCache?.set(nextShuffling, nextDecisionRoot); // shufflingCache?.metrics?.shufflingCache.miss(); } @@ -508,7 +500,6 @@ export class EpochCache { currentShuffling, nextShuffling, nextActiveIndices, - nextActiveIndicesLength: nextActiveIndices.length, effectiveBalanceIncrements, totalSlashingsByIncrement, syncParticipantReward, @@ -552,7 +543,6 @@ export class EpochCache { currentShuffling: this.currentShuffling, nextShuffling: this.nextShuffling, nextActiveIndices: this.nextActiveIndices, - nextActiveIndicesLength: this.nextActiveIndicesLength, // Uint8Array, requires cloning, but it is cloned only when necessary before an epoch transition // See EpochCache.beforeEpochTransition() effectiveBalanceIncrements: this.effectiveBalanceIncrements, @@ -594,6 +584,7 @@ export class EpochCache { ): void { const upcomingEpoch = this.nextEpoch; const epochAfterUpcoming = upcomingEpoch + 1; + const nextActiveIndicesLength = epochTransitionCache.nextEpochShufflingActiveIndicesLength; // move current to previous this.previousShuffling = this.currentShuffling; @@ -614,12 +605,11 @@ export class EpochCache { state, // have to use the "nextActiveIndices" that were saved in the last transition here to calculate // the upcoming shuffling if it is not already built (similar condition to the below computation) - this.nextActiveIndices, - this.nextActiveIndicesLength + this.nextActiveIndices ) ?? // allow for this case during testing where the ShufflingCache is not present, may affect perf testing // so should be taken into account when structuring tests. Should not affect unit or other tests though - computeEpochShuffling(state, this.nextActiveIndices, this.nextActiveIndicesLength, upcomingEpoch); + computeEpochShuffling(state, this.nextActiveIndices, upcomingEpoch); } const upcomingProposerSeed = getSeed(state, upcomingEpoch, DOMAIN_BEACON_PROPOSER); // next epoch was moved to current epoch so use current here @@ -627,35 +617,23 @@ export class EpochCache { // calculate next values this.nextDecisionRoot = getShufflingDecisionBlock(state.config, state, epochAfterUpcoming); - this.nextActiveIndicesLength = epochTransitionCache.nextEpochShufflingActiveIndicesLength; - this.nextActiveIndices = new Array(this.nextActiveIndicesLength); + this.nextActiveIndices = new Array(nextActiveIndicesLength); - if (this.nextActiveIndicesLength > epochTransitionCache.nextEpochShufflingActiveValidatorIndices.length) { + if (nextActiveIndicesLength > epochTransitionCache.nextEpochShufflingActiveValidatorIndices.length) { throw new Error( - `Invalid activeValidatorCount: ${this.nextActiveIndicesLength} > ${epochTransitionCache.nextEpochShufflingActiveValidatorIndices.length}` + `Invalid activeValidatorCount: ${nextActiveIndicesLength} > ${epochTransitionCache.nextEpochShufflingActiveValidatorIndices.length}` ); } // only the first `activeValidatorCount` elements are copied to `activeIndices` - for (let i = 0; i < this.nextActiveIndicesLength; i++) { + for (let i = 0; i < nextActiveIndicesLength; i++) { this.nextActiveIndices[i] = epochTransitionCache.nextEpochShufflingActiveValidatorIndices[i]; } if (this.shufflingCache) { this.nextShuffling = null; - this.shufflingCache?.build( - epochAfterUpcoming, - this.nextDecisionRoot, - state, - this.nextActiveIndices, - this.nextActiveIndicesLength - ); + this.shufflingCache?.build(epochAfterUpcoming, this.nextDecisionRoot, state, this.nextActiveIndices); } else { - this.nextShuffling = computeEpochShuffling( - state, - this.nextActiveIndices, - this.nextActiveIndicesLength, - epochAfterUpcoming - ); + this.nextShuffling = computeEpochShuffling(state, this.nextActiveIndices, epochAfterUpcoming); } // Only pre-compute the seed since it's very cheap. Do the expensive computeProposers() call only on demand. @@ -663,7 +641,7 @@ export class EpochCache { // TODO: DEDUPLICATE from createEpochCache // - // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute everytime the + // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute every time the // active validator indices set changes in size. Validators change active status only when: // - validator.activation_epoch is set. Only changes in process_registry_updates() if validator can be activated. If // the value changes it will be set to `epoch + 1 + MAX_SEED_LOOKAHEAD`. diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index b2a9980dbeda..7a3da2d41dcd 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -33,20 +33,13 @@ export interface IShufflingCache { epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, - activeIndices: ValidatorIndex[], - activeIndicesLength: number + activeIndices: ValidatorIndex[] ): EpochShuffling; /** * Queue asynchronous build for an EpochShuffling */ - build( - epoch: Epoch, - decisionRoot: RootHex, - state: BeaconStateAllForks, - activeIndices: ValidatorIndex[], - activeIndicesLength: number - ): void; + build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void; /** * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will @@ -104,12 +97,13 @@ export function computeCommitteeCount(activeValidatorCount: number): number { export function computeEpochShuffling( state: BeaconStateAllForks, activeIndices: ValidatorIndex[], - activeValidatorCount: number, epoch: Epoch ): EpochShuffling { - const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); const _activeIndices = new Uint32Array(activeIndices); + const activeValidatorCount = activeIndices.length; + const shuffling = _activeIndices.slice(); + const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); unshuffleList(shuffling, seed); const committeesPerSlot = computeCommitteeCount(activeValidatorCount); diff --git a/packages/state-transition/test/perf/util/shufflings.test.ts b/packages/state-transition/test/perf/util/shufflings.test.ts index a18d46e5ff25..664e6355ca37 100644 --- a/packages/state-transition/test/perf/util/shufflings.test.ts +++ b/packages/state-transition/test/perf/util/shufflings.test.ts @@ -36,7 +36,7 @@ describe("epoch shufflings", () => { id: `computeEpochShuffling - vc ${numValidators}`, fn: () => { const {nextActiveIndices} = state.epochCtx; - computeEpochShuffling(state, nextActiveIndices, nextActiveIndices.length, nextEpoch); + computeEpochShuffling(state, nextActiveIndices, nextEpoch); }, }); From eb30b7505297963cb6ebf1f266b66e5c7c1c512b Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 16 Aug 2024 00:27:45 -0400 Subject: [PATCH 35/79] refactor: simplify shufflingCache.getSync --- .../beacon-node/src/chain/shufflingCache.ts | 64 +++++++++---------- .../state-transition/src/cache/epochCache.ts | 14 ++-- .../src/util/epochShuffling.ts | 22 +++---- 3 files changed, 48 insertions(+), 52 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index afde4cccf956..32e827f1f2f7 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -1,4 +1,10 @@ -import {BeaconStateAllForks, EpochShuffling, IShufflingCache, computeEpochShuffling} from "@lodestar/state-transition"; +import { + BeaconStateAllForks, + EpochShuffling, + IShufflingCache, + ShufflingBuildProps, + computeEpochShuffling, +} from "@lodestar/state-transition"; import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; import {LodestarError, MapDef, pruneSetToMax} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; @@ -116,28 +122,14 @@ export class ShufflingCache implements IShufflingCache { } if (isShufflingCacheItem(cacheItem)) { + // this.metrics?.shufflingCache.cacheHit(); return cacheItem.shuffling; } else { + // this.metrics?.shufflingCache.shufflingPromiseNotResolved(); return cacheItem.promise; } } - /** - * Will synchronously get a shuffling if it is available or will return null if not. - */ - getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null { - const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); - if (cacheItem) { - if (isShufflingCacheItem(cacheItem)) { - // this.metrics?.shufflingCache.hit() - return cacheItem.shuffling; - } - // this.metrics?.shufflingCache.shufflingPromiseNotResolved(); - } - // this.metrics?.shufflingCache.miss(); - return null; - } - /** * Gets a cached shuffling via the epoch and decision root. If the shuffling is not * available it will build it synchronously and return the shuffling. @@ -145,26 +137,32 @@ export class ShufflingCache implements IShufflingCache { * NOTE: If a shuffling is already queued and not calculated it will build and resolve * the promise but the already queued build will happen at some later time */ - getOrBuildSync( - epoch: number, - decisionRoot: string, - state: BeaconStateAllForks, - activeIndices: ValidatorIndex[] - ): EpochShuffling { + getSync( + epoch: Epoch, + decisionRoot: RootHex, + buildProps?: T + ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | undefined { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); - if (cacheItem && isShufflingCacheItem(cacheItem)) { - // this.metrics?.shufflingCache.cacheHitEpochTransition(); + if (!cacheItem) { + // this.metrics?.shufflingCache.miss(); + } else if (isShufflingCacheItem(cacheItem)) { + // this.metrics?.shufflingCache.cacheHit(); return cacheItem.shuffling; - } - const shuffling = computeEpochShuffling(state, activeIndices, epoch); - if (cacheItem) { - // this.metrics?.shufflingCache.shufflingPromiseNotResolvedEpochTransition(); - cacheItem.resolveFn(shuffling); + } else if (buildProps) { + // this.metrics?.shufflingCache.shufflingPromiseNotResolvedAndThrownAway(); } else { - // this.metrics?.shufflingCache.cacheMissEpochTransition(); + // this.metrics?.shufflingCache.shufflingPromiseNotResolved(); + } + + let shuffling: EpochShuffling | undefined; + if (buildProps) { + shuffling = computeEpochShuffling(buildProps.state, buildProps.activeIndices, epoch); + if (cacheItem) { + cacheItem.resolveFn(shuffling); + } + this.set(shuffling, decisionRoot); } - this.set(shuffling, decisionRoot); - return shuffling; + return shuffling as T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | undefined; } /** diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 79c8733d4c2e..4ca0eff498a1 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -329,11 +329,11 @@ export class EpochCache { // BeaconChain could provide a shuffling cache to avoid re-computing shuffling every epoch // in that case, we don't need to compute shufflings again const previousDecisionRoot = getShufflingDecisionBlock(config, state, previousEpoch); - const cachedPreviousShuffling = shufflingCache?.getSync(previousEpoch, previousDecisionRoot); + const cachedPreviousShuffling = shufflingCache?.getSync(previousEpoch, previousDecisionRoot) ?? null; const currentDecisionRoot = getShufflingDecisionBlock(config, state, currentEpoch); - const cachedCurrentShuffling = shufflingCache?.getSync(currentEpoch, currentDecisionRoot); + const cachedCurrentShuffling = shufflingCache?.getSync(currentEpoch, currentDecisionRoot) ?? null; const nextDecisionRoot = getShufflingDecisionBlock(config, state, nextEpoch); - const cachedNextShuffling = shufflingCache?.getSync(nextEpoch, nextDecisionRoot); + const cachedNextShuffling = shufflingCache?.getSync(nextEpoch, nextDecisionRoot) ?? null; for (let i = 0; i < validatorCount; i++) { const validator = validators[i]; @@ -599,14 +599,12 @@ export class EpochCache { // shufflingCache?.metrics?.shufflingCache.nextShufflingOnEpochCache(); } else { this.currentShuffling = - this.shufflingCache?.getOrBuildSync( - upcomingEpoch, - this.nextDecisionRoot, + this.shufflingCache?.getSync(upcomingEpoch, this.nextDecisionRoot, { state, // have to use the "nextActiveIndices" that were saved in the last transition here to calculate // the upcoming shuffling if it is not already built (similar condition to the below computation) - this.nextActiveIndices - ) ?? + activeIndices: this.nextActiveIndices, + }) ?? // allow for this case during testing where the ShufflingCache is not present, may affect perf testing // so should be taken into account when structuring tests. Should not affect unit or other tests though computeEpochShuffling(state, this.nextActiveIndices, upcomingEpoch); diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 7a3da2d41dcd..00dd3c917f07 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -16,25 +16,25 @@ import {computeStartSlotAtEpoch} from "./epoch.js"; import {getBlockRootAtSlot} from "./blockRoot.js"; import {computeAnchorCheckpoint} from "./computeAnchorCheckpoint.js"; +export interface ShufflingBuildProps { + state: BeaconStateAllForks; + activeIndices: ValidatorIndex[]; +} export interface IShufflingCache { /** - * Will synchronously get a shuffling if it is available or will return null if not. - */ - getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null; - - /** - * Gets a cached shuffling via the epoch and decision root. If the shuffling is not - * available it will build it synchronously and return the shuffling. + * Gets a cached shuffling via the epoch and decision root. If the state and + * activeIndices are passed and a shuffling is not available it will be built + * synchronously. If the state is not passed and the shuffling is not available + * nothing will be returned. * * NOTE: If a shuffling is already queued and not calculated it will build and resolve * the promise but the already queued build will happen at some later time */ - getOrBuildSync( + getSync( epoch: Epoch, decisionRoot: RootHex, - state: BeaconStateAllForks, - activeIndices: ValidatorIndex[] - ): EpochShuffling; + buildProps?: T + ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | undefined; /** * Queue asynchronous build for an EpochShuffling From 6388a15ff04711b5f26280bf98194d1fa1b8f8f7 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 16 Aug 2024 00:42:55 -0400 Subject: [PATCH 36/79] refactor: remove unnecessary undefined's --- .../state-transition/test/unit/util/cachedBeaconState.test.ts | 1 - packages/state-transition/test/utils/state.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts index f3fb9734718c..654e0752adb8 100644 --- a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts @@ -12,7 +12,6 @@ describe("CachedBeaconState", () => { config: createBeaconConfig(config, emptyState.genesisValidatorsRoot), pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - shufflingCache: undefined, }); }); }); diff --git a/packages/state-transition/test/utils/state.ts b/packages/state-transition/test/utils/state.ts index 0c70d01c226b..29a1f98b5562 100644 --- a/packages/state-transition/test/utils/state.ts +++ b/packages/state-transition/test/utils/state.ts @@ -94,7 +94,6 @@ export function generateCachedState( // This is a test state, there's no need to have a global shared cache of keys pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - shufflingCache: undefined, }); } From b2a7aa2545748509315234fe00b9e59a72b4d99b Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 16 Aug 2024 01:02:16 -0400 Subject: [PATCH 37/79] refactor: clean up ShufflingCache unit test --- .../beacon-node/src/chain/shufflingCache.ts | 7 ++-- .../test/unit/chain/shufflingCache.test.ts | 37 +++++++++---------- .../state-transition/src/cache/epochCache.ts | 6 +-- .../src/util/epochShuffling.ts | 2 +- 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 32e827f1f2f7..0bec8992a55f 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -141,7 +141,7 @@ export class ShufflingCache implements IShufflingCache { epoch: Epoch, decisionRoot: RootHex, buildProps?: T - ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | undefined { + ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); if (!cacheItem) { // this.metrics?.shufflingCache.miss(); @@ -149,12 +149,13 @@ export class ShufflingCache implements IShufflingCache { // this.metrics?.shufflingCache.cacheHit(); return cacheItem.shuffling; } else if (buildProps) { + // TODO: (@matthewkeil) This should possible log a warning?? // this.metrics?.shufflingCache.shufflingPromiseNotResolvedAndThrownAway(); } else { // this.metrics?.shufflingCache.shufflingPromiseNotResolved(); } - let shuffling: EpochShuffling | undefined; + let shuffling: EpochShuffling | null = null; if (buildProps) { shuffling = computeEpochShuffling(buildProps.state, buildProps.activeIndices, epoch); if (cacheItem) { @@ -162,7 +163,7 @@ export class ShufflingCache implements IShufflingCache { } this.set(shuffling, decisionRoot); } - return shuffling as T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | undefined; + return shuffling as T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null; } /** diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index b0e6257a109a..dd2df8ee8f30 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -1,7 +1,5 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import {describe, it, expect, beforeEach} from "vitest"; - -import {getShufflingDecisionBlock} from "@lodestar/state-transition"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../state-transition/test/perf/util.js"; import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; @@ -11,39 +9,38 @@ describe("ShufflingCache", function () { const stateSlot = 100; const state = generateTestCachedBeaconStateOnlyValidators({vc, slot: stateSlot}); const currentEpoch = state.epochCtx.epoch; - const nextEpoch = state.epochCtx.nextEpoch; + const currentDecisionRoot = state.epochCtx.currentDecisionRoot; let shufflingCache: ShufflingCache; beforeEach(() => { shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - shufflingCache.set(state.epochCtx.currentShuffling, state.epochCtx.currentDecisionRoot); + shufflingCache.set(state.epochCtx.currentShuffling, currentDecisionRoot); }); it("should get shuffling from cache", async function () { - const decisionRoot = getShufflingDecisionBlock(state.config, state, currentEpoch); - expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); + expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toEqual(state.epochCtx.currentShuffling); }); it("should bound by maxSize(=1)", async function () { - const decisionRoot = getShufflingDecisionBlock(state.config, state, currentEpoch); - expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); + expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert promises at the same epoch does not prune the cache shufflingCache.insertPromise(currentEpoch, "0x00"); - expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); - // insert shufflings at other epochs does prune the cache - shufflingCache.set(state.epochCtx.nextShuffling!, state.epochCtx.nextDecisionRoot); + expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toEqual(state.epochCtx.currentShuffling); + // insert shuffling at other epochs does prune the cache + shufflingCache.set(state.epochCtx.previousShuffling, state.epochCtx.previousDecisionRoot); // the current shuffling is not available anymore - expect(await shufflingCache.get(currentEpoch, decisionRoot)).toBeNull(); + expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toBeNull(); }); it("should return shuffling from promise", async function () { - const nextDecisionRoot = getShufflingDecisionBlock(state.config, state, nextEpoch); - shufflingCache.insertPromise(nextEpoch, nextDecisionRoot); - const shufflingRequest0 = shufflingCache.get(nextEpoch, nextDecisionRoot); - const shufflingRequest1 = shufflingCache.get(nextEpoch, nextDecisionRoot); - shufflingCache.set(state.epochCtx.nextShuffling!, state.epochCtx.nextDecisionRoot); - expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling!); - expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling!); + const previousEpoch = state.epochCtx.epoch - 1; + const previousDecisionRoot = state.epochCtx.previousDecisionRoot; + shufflingCache.insertPromise(previousEpoch, previousDecisionRoot); + const shufflingRequest0 = shufflingCache.get(previousEpoch, previousDecisionRoot); + const shufflingRequest1 = shufflingCache.get(previousEpoch, previousDecisionRoot); + shufflingCache.set(state.epochCtx.previousShuffling, previousDecisionRoot); + expect(await shufflingRequest0).toEqual(state.epochCtx.previousShuffling); + expect(await shufflingRequest1).toEqual(state.epochCtx.previousShuffling); }); it("should support up to 2 promises at a time", async function () { @@ -51,6 +48,6 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch, "0x00"); shufflingCache.insertPromise(currentEpoch, "0x01"); // inserting other promise should throw error - expect(() => shufflingCache.insertPromise(currentEpoch, "0x04")).toThrow(); + expect(() => shufflingCache.insertPromise(currentEpoch, "0x02")).toThrow(); }); }); diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 4ca0eff498a1..fd93227c2630 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -329,11 +329,11 @@ export class EpochCache { // BeaconChain could provide a shuffling cache to avoid re-computing shuffling every epoch // in that case, we don't need to compute shufflings again const previousDecisionRoot = getShufflingDecisionBlock(config, state, previousEpoch); - const cachedPreviousShuffling = shufflingCache?.getSync(previousEpoch, previousDecisionRoot) ?? null; + const cachedPreviousShuffling = shufflingCache?.getSync(previousEpoch, previousDecisionRoot); const currentDecisionRoot = getShufflingDecisionBlock(config, state, currentEpoch); - const cachedCurrentShuffling = shufflingCache?.getSync(currentEpoch, currentDecisionRoot) ?? null; + const cachedCurrentShuffling = shufflingCache?.getSync(currentEpoch, currentDecisionRoot); const nextDecisionRoot = getShufflingDecisionBlock(config, state, nextEpoch); - const cachedNextShuffling = shufflingCache?.getSync(nextEpoch, nextDecisionRoot) ?? null; + const cachedNextShuffling = shufflingCache?.getSync(nextEpoch, nextDecisionRoot); for (let i = 0; i < validatorCount; i++) { const validator = validators[i]; diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 00dd3c917f07..02ed64dfff59 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -34,7 +34,7 @@ export interface IShufflingCache { epoch: Epoch, decisionRoot: RootHex, buildProps?: T - ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | undefined; + ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null; /** * Queue asynchronous build for an EpochShuffling From a8c0a6d236d7d862cbcf28fb21d6af9959a5c31e Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 16 Aug 2024 03:20:32 -0400 Subject: [PATCH 38/79] feat: add metrics for ShufflingCache --- .../beacon-node/src/chain/shufflingCache.ts | 38 ++++++++------ .../src/chain/validation/attestation.ts | 3 -- .../src/metrics/metrics/lodestar.ts | 52 +++++++++++-------- .../state-transition/src/cache/epochCache.ts | 22 +++++--- 4 files changed, 66 insertions(+), 49 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 0bec8992a55f..ad5e8507a02e 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -37,6 +37,7 @@ type ShufflingCacheItem = { type PromiseCacheItem = { type: CacheItemType.promise; + timeInserted: number; promise: Promise; resolveFn: (shuffling: EpochShuffling) => void; }; @@ -102,6 +103,7 @@ export class ShufflingCache implements IShufflingCache { const cacheItem: PromiseCacheItem = { type: CacheItemType.promise, + timeInserted: Date.now(), promise, resolveFn, }; @@ -117,15 +119,15 @@ export class ShufflingCache implements IShufflingCache { async get(epoch: Epoch, decisionRoot: RootHex): Promise { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); if (cacheItem === undefined) { - // this.metrics?.shufflingCache.miss(); + this.metrics?.shufflingCache.miss.inc(); return null; } if (isShufflingCacheItem(cacheItem)) { - // this.metrics?.shufflingCache.cacheHit(); + this.metrics?.shufflingCache.hit.inc(); return cacheItem.shuffling; } else { - // this.metrics?.shufflingCache.shufflingPromiseNotResolved(); + this.metrics?.shufflingCache.shufflingPromiseNotResolved.inc(); return cacheItem.promise; } } @@ -144,23 +146,22 @@ export class ShufflingCache implements IShufflingCache { ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null { const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); if (!cacheItem) { - // this.metrics?.shufflingCache.miss(); + this.metrics?.shufflingCache.miss.inc(); } else if (isShufflingCacheItem(cacheItem)) { - // this.metrics?.shufflingCache.cacheHit(); + this.metrics?.shufflingCache.hit.inc(); return cacheItem.shuffling; } else if (buildProps) { // TODO: (@matthewkeil) This should possible log a warning?? - // this.metrics?.shufflingCache.shufflingPromiseNotResolvedAndThrownAway(); + this.metrics?.shufflingCache.shufflingPromiseNotResolvedAndThrownAway.inc(); } else { - // this.metrics?.shufflingCache.shufflingPromiseNotResolved(); + this.metrics?.shufflingCache.shufflingPromiseNotResolved.inc(); } let shuffling: EpochShuffling | null = null; if (buildProps) { + const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer(); shuffling = computeEpochShuffling(buildProps.state, buildProps.activeIndices, epoch); - if (cacheItem) { - cacheItem.resolveFn(shuffling); - } + timer?.(); this.set(shuffling, decisionRoot); } return shuffling as T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null; @@ -176,7 +177,9 @@ export class ShufflingCache implements IShufflingCache { * on a NICE thread with a rust implementation */ setTimeout(() => { + const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer(); const shuffling = computeEpochShuffling(state, activeIndices, epoch); + timer?.(); this.set(shuffling, decisionRoot); // wait until after the first slot to help with attestation and block proposal performance }, this.minTimeDelayToBuild); @@ -187,18 +190,19 @@ export class ShufflingCache implements IShufflingCache { * resolve the promise with the built shuffling */ set(shuffling: EpochShuffling, decisionRoot: string): void { - const items = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch); + const shufflingAtEpoch = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch); // if a pending shuffling promise exists, resolve it - const item = items.get(decisionRoot); - if (item) { - if (isPromiseCacheItem(item)) { - item.resolveFn(shuffling); + const cacheItem = shufflingAtEpoch.get(decisionRoot); + if (cacheItem) { + if (isPromiseCacheItem(cacheItem)) { + cacheItem.resolveFn(shuffling); + this.metrics?.shufflingCache.shufflingPromiseResolutionTime.observe(Date.now() - cacheItem.timeInserted); } else { - // metric for throwing away previously calculated shuffling + this.metrics?.shufflingCache.shufflingRecalculated.inc(); } } // set the shuffling - items.set(decisionRoot, {type: CacheItemType.shuffling, shuffling}); + shufflingAtEpoch.set(decisionRoot, {type: CacheItemType.shuffling, shuffling}); // prune the cache pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs); } diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 1116e87e1d25..0d574999e9a3 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -586,12 +586,9 @@ export async function getShufflingForAttestationVerification( const shuffling = await chain.shufflingCache.get(attEpoch, shufflingDependentRoot); if (shuffling) { - // most of the time, we should get the shuffling from cache - chain.metrics?.gossipAttestation.shufflingCacheHit.inc({caller: regenCaller}); return shuffling; } - chain.metrics?.gossipAttestation.shufflingCacheMiss.inc({caller: regenCaller}); try { // for the 1st time of the same epoch and dependent root, it awaits for the regen state // from the 2nd time, it should use the same cached promise and it should reach the above code diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index f43a3f1cdbe6..2ff494e14413 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -630,16 +630,6 @@ export function createLodestarMetrics( labelNames: ["caller"], buckets: [0, 1, 2, 4, 8, 16, 32, 64], }), - shufflingCacheHit: register.gauge<{caller: RegenCaller}>({ - name: "lodestar_gossip_attestation_shuffling_cache_hit_count", - help: "Count of gossip attestation verification shuffling cache hit", - labelNames: ["caller"], - }), - shufflingCacheMiss: register.gauge<{caller: RegenCaller}>({ - name: "lodestar_gossip_attestation_shuffling_cache_miss_count", - help: "Count of gossip attestation verification shuffling cache miss", - labelNames: ["caller"], - }), shufflingCacheRegenHit: register.gauge<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_shuffling_cache_regen_hit_count", help: "Count of gossip attestation verification shuffling cache regen hit", @@ -1278,22 +1268,40 @@ export function createLodestarMetrics( name: "lodestar_shuffling_cache_size", help: "Shuffling cache size", }), - processStateInsertNew: register.gauge({ - name: "lodestar_shuffling_cache_process_state_insert_new_total", - help: "Total number of times processState is called resulting a new shuffling", - }), - processStateUpdatePromise: register.gauge({ - name: "lodestar_shuffling_cache_process_state_update_promise_total", - help: "Total number of times processState is called resulting a promise being updated with shuffling", - }), - processStateNoOp: register.gauge({ - name: "lodestar_shuffling_cache_process_state_no_op_total", - help: "Total number of times processState is called resulting no changes", - }), insertPromiseCount: register.gauge({ name: "lodestar_shuffling_cache_insert_promise_count", help: "Total number of times insertPromise is called", }), + hit: register.gauge({ + name: "lodestar_gossip_attestation_shuffling_cache_hit_count", + help: "Count of shuffling cache hit", + }), + miss: register.gauge({ + name: "lodestar_gossip_attestation_shuffling_cache_miss_count", + help: "Count of shuffling cache miss", + }), + shufflingRecalculated: register.gauge({ + name: "lodestar_shuffling_cache_recalculated_shuffling_count", + help: "Count of shuffling cache promises that were discarded and the shuffling was built synchronously", + }), + shufflingPromiseNotResolvedAndThrownAway: register.gauge({ + name: "lodestar_shuffling_cache_promise_not_resolved_and_thrown_away_count", + help: "Count of shuffling cache promises that were discarded and the shuffling was built synchronously", + }), + shufflingPromiseNotResolved: register.gauge({ + name: "lodestar_shuffling_cache_promise_not_resolved_count", + help: "Count of shuffling cache promises that were requested before the promise was resolved", + }), + shufflingPromiseResolutionTime: register.histogram({ + name: "lodestar_shuffling_cache_promise_resolution_time", + help: "Count of shuffling cache promises that were requested before the promise was resolved", + buckets: [1, 10, 100, 1000], + }), + shufflingCalculationTime: register.histogram({ + name: "lodestar_shuffling_cache_shuffling_calculation_time", + help: "Count of shuffling cache promises that were requested before the promise was resolved", + buckets: [0.5, 0.75, 1, 1.25, 1.5], + }), }, seenCache: { diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index fd93227c2630..82b1fe5fff3a 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -379,10 +379,13 @@ export class EpochCache { let currentShuffling: EpochShuffling; if (cachedCurrentShuffling) { currentShuffling = cachedCurrentShuffling; + } else if (shufflingCache) { + currentShuffling = shufflingCache.getSync(currentEpoch, currentDecisionRoot, { + state, + activeIndices: currentActiveIndices, + }); } else { currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentEpoch); - shufflingCache?.set(currentShuffling, currentDecisionRoot); - // shufflingCache?.metrics?.shufflingCache.miss(); } let previousShuffling: EpochShuffling; @@ -391,19 +394,25 @@ export class EpochCache { } else if (isGenesis) { // TODO: (@matthewkeil) does this need to be added to the cache at previousEpoch and previousDecisionRoot? previousShuffling = currentShuffling; + } else if (shufflingCache) { + previousShuffling = shufflingCache.getSync(previousEpoch, previousDecisionRoot, { + state, + activeIndices: previousActiveIndices, + }); } else { previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); - shufflingCache?.set(previousShuffling, previousDecisionRoot); - // shufflingCache?.metrics?.shufflingCache.miss(); } let nextShuffling: EpochShuffling; if (cachedNextShuffling) { nextShuffling = cachedNextShuffling; + } else if (shufflingCache) { + nextShuffling = shufflingCache.getSync(nextEpoch, nextDecisionRoot, { + state, + activeIndices: nextActiveIndices, + }); } else { nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); - shufflingCache?.set(nextShuffling, nextDecisionRoot); - // shufflingCache?.metrics?.shufflingCache.miss(); } const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER); @@ -596,7 +605,6 @@ export class EpochCache { if (this.nextShuffling) { // was already pulled by the api or another method on EpochCache this.currentShuffling = this.nextShuffling; - // shufflingCache?.metrics?.shufflingCache.nextShufflingOnEpochCache(); } else { this.currentShuffling = this.shufflingCache?.getSync(upcomingEpoch, this.nextDecisionRoot, { From 40bb572e07347c50ff59fbbba1c7ea4a6929c56b Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 16 Aug 2024 03:39:48 -0400 Subject: [PATCH 39/79] feat: add shufflingCache metrics to state-transition --- packages/beacon-node/src/metrics/metrics/lodestar.ts | 4 ++++ packages/state-transition/src/cache/epochCache.ts | 1 + packages/state-transition/src/util/epochShuffling.ts | 9 ++++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 2ff494e14413..5b234fe6423e 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1292,6 +1292,10 @@ export function createLodestarMetrics( name: "lodestar_shuffling_cache_promise_not_resolved_count", help: "Count of shuffling cache promises that were requested before the promise was resolved", }), + nextShufflingOnEpochCache: register.gauge({ + name: "lodestar_shuffling_cache_promise_not_resolved_count", + help: "The next shuffling was already pulled to the epoch cache before the epoch transition", + }), shufflingPromiseResolutionTime: register.histogram({ name: "lodestar_shuffling_cache_promise_resolution_time", help: "Count of shuffling cache promises that were requested before the promise was resolved", diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 82b1fe5fff3a..a0ab4e881761 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -605,6 +605,7 @@ export class EpochCache { if (this.nextShuffling) { // was already pulled by the api or another method on EpochCache this.currentShuffling = this.nextShuffling; + this.shufflingCache?.metrics?.shufflingCache.nextShufflingOnEpochCache.inc(); } else { this.currentShuffling = this.shufflingCache?.getSync(upcomingEpoch, this.nextDecisionRoot, { diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 02ed64dfff59..094245ccc8f0 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -1,6 +1,6 @@ import {toHexString} from "@chainsafe/ssz"; import {Epoch, RootHex, ssz, ValidatorIndex} from "@lodestar/types"; -import {intDiv, toRootHex} from "@lodestar/utils"; +import {GaugeExtra, intDiv, NoLabels, toRootHex} from "@lodestar/utils"; import { DOMAIN_BEACON_ATTESTER, GENESIS_SLOT, @@ -20,7 +20,14 @@ export interface ShufflingBuildProps { state: BeaconStateAllForks; activeIndices: ValidatorIndex[]; } + +export interface PublicShufflingCacheMetrics { + shufflingCache: { + nextShufflingOnEpochCache: GaugeExtra; + }; +} export interface IShufflingCache { + metrics: PublicShufflingCacheMetrics | null; /** * Gets a cached shuffling via the epoch and decision root. If the state and * activeIndices are passed and a shuffling is not available it will be built From dbe25c9bd86cadae4b1ef4eee289b2d40110b121 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 16 Aug 2024 03:41:29 -0400 Subject: [PATCH 40/79] chore: lint --- packages/beacon-node/test/unit/chain/shufflingCache.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index dd2df8ee8f30..5bf8e5835b3d 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-non-null-assertion */ import {describe, it, expect, beforeEach} from "vitest"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../state-transition/test/perf/util.js"; From 70dc68330bda3a0b8fb0b0da0aafe205446c082f Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 16 Aug 2024 03:58:27 -0400 Subject: [PATCH 41/79] fix: metric name clash --- packages/beacon-node/src/metrics/metrics/lodestar.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 5b234fe6423e..ca8df880e60c 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1293,7 +1293,7 @@ export function createLodestarMetrics( help: "Count of shuffling cache promises that were requested before the promise was resolved", }), nextShufflingOnEpochCache: register.gauge({ - name: "lodestar_shuffling_cache_promise_not_resolved_count", + name: "lodestar_shuffling_cache_next_shuffling_on_epoch_cache", help: "The next shuffling was already pulled to the epoch cache before the epoch transition", }), shufflingPromiseResolutionTime: register.histogram({ From 27081b29cb48951ec42408a5defcea31fc73ebbb Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 00:54:08 -0400 Subject: [PATCH 42/79] refactor: add comment about not having ShufflingCache in EpochCache --- .../state-transition/src/cache/epochCache.ts | 65 +++++++++---------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index a0ab4e881761..36eff3bb796d 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -376,43 +376,39 @@ export class EpochCache { throw Error("totalActiveBalanceIncrements >= Number.MAX_SAFE_INTEGER. MAX_EFFECTIVE_BALANCE is too low."); } - let currentShuffling: EpochShuffling; - if (cachedCurrentShuffling) { - currentShuffling = cachedCurrentShuffling; - } else if (shufflingCache) { - currentShuffling = shufflingCache.getSync(currentEpoch, currentDecisionRoot, { - state, - activeIndices: currentActiveIndices, - }); - } else { - currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentEpoch); - } - let previousShuffling: EpochShuffling; - if (cachedPreviousShuffling) { - previousShuffling = cachedPreviousShuffling; - } else if (isGenesis) { - // TODO: (@matthewkeil) does this need to be added to the cache at previousEpoch and previousDecisionRoot? - previousShuffling = currentShuffling; - } else if (shufflingCache) { - previousShuffling = shufflingCache.getSync(previousEpoch, previousDecisionRoot, { - state, - activeIndices: previousActiveIndices, - }); - } else { - previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); - } - + let currentShuffling: EpochShuffling; let nextShuffling: EpochShuffling; - if (cachedNextShuffling) { - nextShuffling = cachedNextShuffling; - } else if (shufflingCache) { - nextShuffling = shufflingCache.getSync(nextEpoch, nextDecisionRoot, { - state, - activeIndices: nextActiveIndices, - }); - } else { + if (!shufflingCache) { + // Only for testing. shufflingCache should always be available in prod + previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); + currentShuffling = isGenesis + ? previousShuffling + : computeEpochShuffling(state, currentActiveIndices, currentEpoch); nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); + } else { + currentShuffling = cachedCurrentShuffling + ? cachedCurrentShuffling + : shufflingCache.getSync(currentEpoch, currentDecisionRoot, { + state, + activeIndices: currentActiveIndices, + }); + + previousShuffling = cachedPreviousShuffling + ? cachedPreviousShuffling + : isGenesis + ? currentShuffling // TODO: (@matthewkeil) does this need to be added to the cache at previousEpoch and previousDecisionRoot? + : shufflingCache.getSync(previousEpoch, previousDecisionRoot, { + state, + activeIndices: previousActiveIndices, + }); + + nextShuffling = cachedNextShuffling + ? cachedNextShuffling + : shufflingCache.getSync(nextEpoch, nextDecisionRoot, { + state, + activeIndices: nextActiveIndices, + }); } const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER); @@ -640,6 +636,7 @@ export class EpochCache { this.nextShuffling = null; this.shufflingCache?.build(epochAfterUpcoming, this.nextDecisionRoot, state, this.nextActiveIndices); } else { + // Only for testing. shufflingCache should always be available in prod this.nextShuffling = computeEpochShuffling(state, this.nextActiveIndices, epochAfterUpcoming); } From 600acae90e33feacd20ce9c0f1184ec9ac6bf308 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 00:58:57 -0400 Subject: [PATCH 43/79] refactor: rename shuffling decision root functions --- .../src/api/impl/beacon/state/index.ts | 2 +- .../beacon-node/src/api/impl/validator/index.ts | 2 +- .../state-transition/src/cache/epochCache.ts | 16 ++++++++-------- .../state-transition/src/util/epochShuffling.ts | 10 +++++++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index 4dd028ed1978..b5b9e8c71555 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -202,7 +202,7 @@ export function getBeaconStateApi({ const epoch = filters.epoch ?? computeEpochAtSlot(state.slot); const startSlot = computeStartSlotAtEpoch(epoch); - const decisionRoot = stateCached.epochCtx.getDecisionRoot(epoch); + const decisionRoot = stateCached.epochCtx.getShufflingDecisionRoot(epoch); const shuffling = await chain.shufflingCache.get(epoch, decisionRoot); if (!shuffling) { throw new ApiError( diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index c033b3223ba4..400ff7a25422 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -989,7 +989,7 @@ export function getValidatorApi( // Check that all validatorIndex belong to the state before calling getCommitteeAssignments() const pubkeys = getPubkeysForIndices(state.validators, indices); - const decisionRoot = state.epochCtx.getDecisionRoot(epoch); + const decisionRoot = state.epochCtx.getShufflingDecisionRoot(epoch); const shuffling = await chain.shufflingCache.get(epoch, decisionRoot); if (!shuffling) { throw new ApiError( diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 36eff3bb796d..31cfb7fe7b24 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -28,7 +28,7 @@ import { import { computeEpochShuffling, EpochShuffling, - getShufflingDecisionBlock, + calculateShufflingDecisionRoot, IShufflingCache, } from "../util/epochShuffling.js"; import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../util/syncCommittee.js"; @@ -328,11 +328,11 @@ export class EpochCache { // BeaconChain could provide a shuffling cache to avoid re-computing shuffling every epoch // in that case, we don't need to compute shufflings again - const previousDecisionRoot = getShufflingDecisionBlock(config, state, previousEpoch); + const previousDecisionRoot = calculateShufflingDecisionRoot(config, state, previousEpoch); const cachedPreviousShuffling = shufflingCache?.getSync(previousEpoch, previousDecisionRoot); - const currentDecisionRoot = getShufflingDecisionBlock(config, state, currentEpoch); + const currentDecisionRoot = calculateShufflingDecisionRoot(config, state, currentEpoch); const cachedCurrentShuffling = shufflingCache?.getSync(currentEpoch, currentDecisionRoot); - const nextDecisionRoot = getShufflingDecisionBlock(config, state, nextEpoch); + const nextDecisionRoot = calculateShufflingDecisionRoot(config, state, nextEpoch); const cachedNextShuffling = shufflingCache?.getSync(nextEpoch, nextDecisionRoot); for (let i = 0; i < validatorCount; i++) { @@ -619,7 +619,7 @@ export class EpochCache { this.proposers = computeProposers(upcomingProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements); // calculate next values - this.nextDecisionRoot = getShufflingDecisionBlock(state.config, state, epochAfterUpcoming); + this.nextDecisionRoot = calculateShufflingDecisionRoot(state.config, state, epochAfterUpcoming); this.nextActiveIndices = new Array(nextActiveIndicesLength); if (nextActiveIndicesLength > epochTransitionCache.nextEpochShufflingActiveValidatorIndices.length) { @@ -878,7 +878,7 @@ export class EpochCache { throw new EpochCacheError({ code: EpochCacheErrorCode.NEXT_SHUFFLING_NOT_AVAILABLE, epoch: epoch, - decisionRoot: this.getDecisionRoot(this.nextEpoch), + decisionRoot: this.getShufflingDecisionRoot(this.nextEpoch), }); } throw new EpochCacheError({ @@ -891,7 +891,7 @@ export class EpochCache { return shuffling; } - getDecisionRoot(epoch: Epoch): RootHex { + getShufflingDecisionRoot(epoch: Epoch): RootHex { switch (epoch) { case this.epoch - 1: return this.previousDecisionRoot; @@ -917,7 +917,7 @@ export class EpochCache { case this.nextEpoch: if (!this.nextShuffling) { this.nextShuffling = - this.shufflingCache?.getSync(this.nextEpoch, this.getDecisionRoot(this.nextEpoch)) ?? null; + this.shufflingCache?.getSync(this.nextEpoch, this.getShufflingDecisionRoot(this.nextEpoch)) ?? null; } return this.nextShuffling; default: diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 094245ccc8f0..8cde5861c408 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -141,7 +141,7 @@ export function computeEpochShuffling( }; } -function getDecisionBlock(state: BeaconStateAllForks, epoch: Epoch): RootHex { +function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch): RootHex { const pivotSlot = computeStartSlotAtEpoch(epoch - 1) - 1; return toRootHex(getBlockRootAtSlot(state, pivotSlot)); } @@ -151,8 +151,12 @@ function getDecisionBlock(state: BeaconStateAllForks, epoch: Epoch): RootHex { * - Special case close to genesis block, return the genesis block root * - This is similar to forkchoice.getDependentRoot() function, otherwise we cannot get cached shuffing in attestation verification when syncing from genesis. */ -export function getShufflingDecisionBlock(config: BeaconConfig, state: BeaconStateAllForks, epoch: Epoch): RootHex { +export function calculateShufflingDecisionRoot( + config: BeaconConfig, + state: BeaconStateAllForks, + epoch: Epoch +): RootHex { return state.slot > GENESIS_SLOT - ? getDecisionBlock(state, epoch) + ? calculateDecisionRoot(state, epoch) : toHexString(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(config, state).blockHeader)); } From 8945e0381f29bd03f8873e03cb92697490031d98 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 01:15:21 -0400 Subject: [PATCH 44/79] refactor: remove unused comment --- packages/beacon-node/test/spec/presets/genesis.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/beacon-node/test/spec/presets/genesis.test.ts b/packages/beacon-node/test/spec/presets/genesis.test.ts index 0732d2bd7ec0..f03f2595a566 100644 --- a/packages/beacon-node/test/spec/presets/genesis.test.ts +++ b/packages/beacon-node/test/spec/presets/genesis.test.ts @@ -19,7 +19,6 @@ import {getConfig} from "../../utils/config.js"; import {RunnerType} from "../utils/types.js"; import {specTestIterator} from "../utils/specTestIterator.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; -// import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; // The aim of the genesis tests is to provide a baseline to test genesis-state initialization and test if the // proposed genesis-validity conditions are working. From 635d74ccdcdf5047eb6dc28f557f7f1108cf271d Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 01:32:51 -0400 Subject: [PATCH 45/79] feat: async add nextShuffling to EpochCache after its built --- packages/beacon-node/src/chain/chain.ts | 2 +- .../beacon-node/src/chain/shufflingCache.ts | 25 +++++++++++++------ .../state-transition/src/cache/epochCache.ts | 14 ++++++++++- .../src/util/epochShuffling.ts | 10 ++++++-- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index adfc567481d8..64fd2defcf34 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -239,7 +239,6 @@ export class BeaconChain implements IBeaconChain { this.beaconProposerCache = new BeaconProposerCache(opts); this.checkpointBalancesCache = new CheckpointBalancesCache(); - this.shufflingCache = new ShufflingCache(metrics, this.opts); // Restore state caches // anchorState may already by a CachedBeaconState. If so, don't create the cache again, since deserializing all @@ -255,6 +254,7 @@ export class BeaconChain implements IBeaconChain { index2pubkey: [], }); + this.shufflingCache = new ShufflingCache(metrics, logger, this.opts); cachedState.epochCtx.shufflingCache = this.shufflingCache; this.shufflingCache.set(cachedState.epochCtx.previousShuffling, cachedState.epochCtx.previousDecisionRoot); this.shufflingCache.set(cachedState.epochCtx.currentShuffling, cachedState.epochCtx.currentDecisionRoot); diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index ad5e8507a02e..bdf925150781 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -66,6 +66,7 @@ export class ShufflingCache implements IShufflingCache { constructor( readonly metrics: Metrics | null = null, + readonly logger: Logger | null = null, opts: ShufflingCacheOpts = {} ) { if (metrics) { @@ -170,19 +171,27 @@ export class ShufflingCache implements IShufflingCache { /** * Queue asynchronous build for an EpochShuffling */ - build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void { + build( + epoch: number, + decisionRoot: string, + state: BeaconStateAllForks, + activeIndices: ValidatorIndex[] + ): Promise { this.insertPromise(epoch, decisionRoot); /** * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations * on a NICE thread with a rust implementation */ - setTimeout(() => { - const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer(); - const shuffling = computeEpochShuffling(state, activeIndices, epoch); - timer?.(); - this.set(shuffling, decisionRoot); - // wait until after the first slot to help with attestation and block proposal performance - }, this.minTimeDelayToBuild); + return new Promise((resolve) => { + setTimeout(() => { + const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer(); + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + timer?.(); + this.set(shuffling, decisionRoot); + resolve(shuffling); + // wait until after the first slot to help with attestation and block proposal performance + }, this.minTimeDelayToBuild); + }); } /** diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 31cfb7fe7b24..c09327c2fe0c 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -634,7 +634,19 @@ export class EpochCache { if (this.shufflingCache) { this.nextShuffling = null; - this.shufflingCache?.build(epochAfterUpcoming, this.nextDecisionRoot, state, this.nextActiveIndices); + const decisionRoot = this.nextDecisionRoot; + this.shufflingCache + ?.build(epochAfterUpcoming, this.nextDecisionRoot, state, this.nextActiveIndices) + .then((shuffling) => { + this.nextShuffling = shuffling; + }) + .catch((err) => { + this.shufflingCache?.logger?.error( + "EPOCH_CONTEXT_SHUFFLING_BUILD_ERROR", + {epoch: epochAfterUpcoming, decisionRoot}, + err + ); + }); } else { // Only for testing. shufflingCache should always be available in prod this.nextShuffling = computeEpochShuffling(state, this.nextActiveIndices, epochAfterUpcoming); diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 8cde5861c408..da4547a17dec 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -1,6 +1,6 @@ import {toHexString} from "@chainsafe/ssz"; import {Epoch, RootHex, ssz, ValidatorIndex} from "@lodestar/types"; -import {GaugeExtra, intDiv, NoLabels, toRootHex} from "@lodestar/utils"; +import {GaugeExtra, intDiv, Logger, NoLabels, toRootHex} from "@lodestar/utils"; import { DOMAIN_BEACON_ATTESTER, GENESIS_SLOT, @@ -28,6 +28,7 @@ export interface PublicShufflingCacheMetrics { } export interface IShufflingCache { metrics: PublicShufflingCacheMetrics | null; + logger: Logger | null; /** * Gets a cached shuffling via the epoch and decision root. If the state and * activeIndices are passed and a shuffling is not available it will be built @@ -46,7 +47,12 @@ export interface IShufflingCache { /** * Queue asynchronous build for an EpochShuffling */ - build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void; + build( + epoch: Epoch, + decisionRoot: RootHex, + state: BeaconStateAllForks, + activeIndices: ValidatorIndex[] + ): Promise; /** * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will From a2ad1edfdd667529cd8d9e2f368faf231a693da2 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 01:49:44 -0400 Subject: [PATCH 46/79] feat: make ShufflingCache.set private --- packages/beacon-node/src/chain/chain.ts | 21 ++++++++++++------- .../beacon-node/src/chain/shufflingCache.ts | 13 +++++++++--- .../src/util/epochShuffling.ts | 6 ------ 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 64fd2defcf34..236e6a52b215 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -254,13 +254,20 @@ export class BeaconChain implements IBeaconChain { index2pubkey: [], }); - this.shufflingCache = new ShufflingCache(metrics, logger, this.opts); - cachedState.epochCtx.shufflingCache = this.shufflingCache; - this.shufflingCache.set(cachedState.epochCtx.previousShuffling, cachedState.epochCtx.previousDecisionRoot); - this.shufflingCache.set(cachedState.epochCtx.currentShuffling, cachedState.epochCtx.currentDecisionRoot); - if (cachedState.epochCtx.nextShuffling !== null) { - this.shufflingCache.set(cachedState.epochCtx.nextShuffling, cachedState.epochCtx.nextDecisionRoot); - } + this.shufflingCache = cachedState.epochCtx.shufflingCache = new ShufflingCache(metrics, logger, this.opts, [ + { + shuffling: cachedState.epochCtx.previousShuffling, + decisionRoot: cachedState.epochCtx.previousDecisionRoot, + }, + { + shuffling: cachedState.epochCtx.currentShuffling, + decisionRoot: cachedState.epochCtx.currentDecisionRoot, + }, + { + shuffling: cachedState.epochCtx.nextShuffling, + decisionRoot: cachedState.epochCtx.nextDecisionRoot, + }, + ]); // Persist single global instance of state caches this.pubkey2index = cachedState.epochCtx.pubkey2index; diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index bdf925150781..8b7de8cf7c07 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -6,7 +6,7 @@ import { computeEpochShuffling, } from "@lodestar/state-transition"; import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; -import {LodestarError, MapDef, pruneSetToMax} from "@lodestar/utils"; +import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; /** @@ -67,7 +67,8 @@ export class ShufflingCache implements IShufflingCache { constructor( readonly metrics: Metrics | null = null, readonly logger: Logger | null = null, - opts: ShufflingCacheOpts = {} + opts: ShufflingCacheOpts = {}, + precalculatedShufflings?: {shuffling: EpochShuffling | null; decisionRoot: RootHex}[] ) { if (metrics) { metrics.shufflingCache.size.addCollect(() => @@ -79,6 +80,12 @@ export class ShufflingCache implements IShufflingCache { this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS; this.minTimeDelayToBuild = opts.minTimeDelayToBuildShuffling ?? DEFAULT_MIN_TIME_DELAY_IN_MS; + + precalculatedShufflings?.map(({shuffling, decisionRoot}) => { + if (shuffling !== null) { + this.set(shuffling, decisionRoot); + } + }); } /** @@ -198,7 +205,7 @@ export class ShufflingCache implements IShufflingCache { * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will * resolve the promise with the built shuffling */ - set(shuffling: EpochShuffling, decisionRoot: string): void { + private set(shuffling: EpochShuffling, decisionRoot: string): void { const shufflingAtEpoch = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch); // if a pending shuffling promise exists, resolve it const cacheItem = shufflingAtEpoch.get(decisionRoot); diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index da4547a17dec..2973f325046f 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -53,12 +53,6 @@ export interface IShufflingCache { state: BeaconStateAllForks, activeIndices: ValidatorIndex[] ): Promise; - - /** - * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will - * resolve the promise with the built shuffling - */ - set(shuffling: EpochShuffling, decisionRoot: RootHex): void; } /** From db86883a913d6d420edb66204c897ccf70415a0f Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 01:55:54 -0400 Subject: [PATCH 47/79] feat: chance metrics to nextShufflingNotOnEpochCache instead of positive case --- packages/beacon-node/src/metrics/metrics/lodestar.ts | 6 +++--- packages/state-transition/src/cache/epochCache.ts | 4 ++-- packages/state-transition/src/util/epochShuffling.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index ca8df880e60c..21fb92efeb66 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1292,9 +1292,9 @@ export function createLodestarMetrics( name: "lodestar_shuffling_cache_promise_not_resolved_count", help: "Count of shuffling cache promises that were requested before the promise was resolved", }), - nextShufflingOnEpochCache: register.gauge({ - name: "lodestar_shuffling_cache_next_shuffling_on_epoch_cache", - help: "The next shuffling was already pulled to the epoch cache before the epoch transition", + nextShufflingNotOnEpochCache: register.gauge({ + name: "lodestar_shuffling_cache_next_shuffling_not_on_epoch_cache", + help: "The next shuffling was not on the epoch cache before the epoch transition", }), shufflingPromiseResolutionTime: register.histogram({ name: "lodestar_shuffling_cache_promise_resolution_time", diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index c09327c2fe0c..6425cd0cee50 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -599,10 +599,10 @@ export class EpochCache { // move next to current or calculate upcoming this.currentDecisionRoot = this.nextDecisionRoot; if (this.nextShuffling) { - // was already pulled by the api or another method on EpochCache + // was already pulled from the ShufflingCache to the EpochCache (should be in most cases) this.currentShuffling = this.nextShuffling; - this.shufflingCache?.metrics?.shufflingCache.nextShufflingOnEpochCache.inc(); } else { + this.shufflingCache?.metrics?.shufflingCache.nextShufflingNotOnEpochCache.inc(); this.currentShuffling = this.shufflingCache?.getSync(upcomingEpoch, this.nextDecisionRoot, { state, diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 2973f325046f..38330320024d 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -23,7 +23,7 @@ export interface ShufflingBuildProps { export interface PublicShufflingCacheMetrics { shufflingCache: { - nextShufflingOnEpochCache: GaugeExtra; + nextShufflingNotOnEpochCache: GaugeExtra; }; } export interface IShufflingCache { From 7114aa5249c861c2cf94249fabc77712f5f468a4 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 02:08:34 -0400 Subject: [PATCH 48/79] refactor: move diff to separate PR --- packages/utils/src/diff.ts | 232 ------------------------------------ packages/utils/src/index.ts | 1 - 2 files changed, 233 deletions(-) delete mode 100644 packages/utils/src/diff.ts diff --git a/packages/utils/src/diff.ts b/packages/utils/src/diff.ts deleted file mode 100644 index 204989016b46..000000000000 --- a/packages/utils/src/diff.ts +++ /dev/null @@ -1,232 +0,0 @@ -/* eslint-disable no-console */ -import fs from "node:fs"; - -const primitiveTypeof = ["number", "string", "bigint", "boolean"]; -export type BufferType = Uint8Array | Uint32Array; -export type PrimitiveType = number | string | bigint | boolean | BufferType; -export type DiffableCollection = Record; -export type Diffable = PrimitiveType | Array | DiffableCollection; - -export interface Diff { - objectPath: string; - errorMessage?: string; - val1: Diffable; - val2: Diffable; -} - -export function diffUint8Array(val1: Uint8Array, val2: PrimitiveType, objectPath: string): Diff[] { - if (!(val2 instanceof Uint8Array)) { - return [ - { - objectPath, - errorMessage: `val1${objectPath} is a Uint8Array, but val2${objectPath} is not`, - val1, - val2, - }, - ]; - } - const hex1 = Buffer.from(val1).toString("hex"); - const hex2 = Buffer.from(val2).toString("hex"); - if (hex1 !== hex2) { - return [ - { - objectPath, - val1: `0x${hex1}`, - val2: `0x${hex2}`, - }, - ]; - } - return []; -} - -export function diffUint32Array(val1: Uint32Array, val2: PrimitiveType, objectPath: string): Diff[] { - if (!(val2 instanceof Uint32Array)) { - return [ - { - objectPath, - errorMessage: `val1${objectPath} is a Uint32Array, but val2${objectPath} is not`, - val1, - val2, - }, - ]; - } - const diffs: Diff[] = []; - val1.forEach((value, index) => { - const value2 = val2[index]; - if (value !== value2) { - diffs.push({ - objectPath: `${objectPath}[${index}]`, - val1: `0x${value.toString(16).padStart(8, "0")}`, - val2: value2 ? `0x${val2[index].toString(16).padStart(8, "0")}` : "undefined", - }); - } - }); - return diffs; -} - -function diffPrimitiveValue(val1: PrimitiveType, val2: PrimitiveType, objectPath: string): Diff[] { - if (val1 instanceof Uint8Array) { - return diffUint8Array(val1, val2, objectPath); - } - if (val1 instanceof Uint32Array) { - return diffUint32Array(val1, val2, objectPath); - } - - const diff = {objectPath, val1, val2} as Diff; - const type1 = typeof val1; - if (!primitiveTypeof.includes(type1)) { - diff.errorMessage = `val1${objectPath} is not a supported type`; - } - const type2 = typeof val2; - if (!primitiveTypeof.includes(type2)) { - diff.errorMessage = `val2${objectPath} is not a supported type`; - } - if (type1 !== type2) { - diff.errorMessage = `val1${objectPath} is not the same type as val2${objectPath}`; - } - if (val1 !== val2) { - return [diff]; - } - return []; -} - -function isPrimitiveValue(val: unknown): val is PrimitiveType { - if (Array.isArray(val)) return false; - if (typeof val === "object") { - return val instanceof Uint8Array || val instanceof Uint32Array; - } - return true; -} - -function isDiffable(val: unknown): val is Diffable { - return !(typeof val === "function" || typeof val === "symbol" || typeof val === "undefined" || val === null); -} - -export function getDiffs(val1: Diffable, val2: Diffable, objectPath: string): Diff[] { - if (isPrimitiveValue(val1)) { - if (!isPrimitiveValue(val2)) { - return [ - { - objectPath, - errorMessage: `val1${objectPath} is a primitive value and val2${objectPath} is not`, - val1, - val2, - }, - ]; - } - return diffPrimitiveValue(val1, val2, objectPath); - } - - const isArray = Array.isArray(val1); - let errorMessage: string | undefined; - if (isArray && !Array.isArray(val2)) { - errorMessage = `val1${objectPath} is an array and val2${objectPath} is not`; - } else if (typeof val1 === "object" && typeof val2 !== "object") { - errorMessage = `val1${objectPath} is a nested object and val2${objectPath} is not`; - } - if (errorMessage) { - return [ - { - objectPath, - errorMessage, - val1, - val2, - }, - ]; - } - - const diffs: Diff[] = []; - for (const [index, value] of Object.entries(val1)) { - if (!isDiffable(value)) { - diffs.push({objectPath, val1, val2, errorMessage: `val1${objectPath} is not Diffable`}); - continue; - } - const value2 = (val2 as DiffableCollection)[index]; - if (!isDiffable(value2)) { - diffs.push({objectPath, val1, val2, errorMessage: `val2${objectPath} is not Diffable`}); - continue; - } - const innerPath = isArray ? `${objectPath}[${index}]` : `${objectPath}.${index}`; - diffs.push(...getDiffs(value, value2, innerPath)); - } - return diffs; -} - -/** - * Find the different values on complex, nested objects. Outputs the path through the object to - * each value that does not match from val1 and val2. Optionally can output the values that differ. - * - * For objects that differ greatly, can write to a file instead of the terminal for analysis - * - * ## Example - * ```ts - * const obj1 = { - * key1: { - * key2: [ - * { key3: 1 }, - * { key3: new Uint8Array([1, 2, 3]) } - * ] - * }, - * key4: new Uint32Array([1, 2, 3]), - * key5: 362436 - * }; - * - * const obj2 = { - * key1: { - * key2: [ - * { key3: 1 }, - * { key3: new Uint8Array([1, 2, 4]) } - * ] - * }, - * key4: new Uint32Array([1, 2, 4]) - * key5: true - * }; - * - * diffObjects(obj1, obj2, true); - * - * - * ``` - * - * ## Output - * ```sh - * val.key1.key2[1].key3 - * - 0x010203 - * - 0x010204 - * val.key4[2] - * - 0x00000003 - * - 0x00000004 - * val.key5 - * val1.key5 is not the same type as val2.key5 - * - 362436 - * - true - * ``` - */ -export function diff(val1: unknown, val2: unknown, outputValues = false, filename?: string): void { - if (!isDiffable(val1)) { - console.log("val1 is not Diffable"); - return; - } - if (!isDiffable(val2)) { - console.log("val2 is not Diffable"); - return; - } - const diffs = getDiffs(val1, val2, ""); - let output = ""; - if (diffs.length) { - diffs.forEach((diff) => { - let diffOutput = `value${diff.objectPath}`; - if (diff.errorMessage) { - diffOutput += `\n ${diff.errorMessage}`; - } - if (outputValues) { - diffOutput += `\n - ${diff.val1.toString()}\n - ${diff.val2.toString()}\n`; - } - output += `${diffOutput}\n`; - }); - if (filename) { - fs.writeFileSync(filename, output); - } else { - console.log(output); - } - } -} diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 5ba509047829..2057e50e07bc 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -3,7 +3,6 @@ export * from "./assert.js"; export * from "./base64.js"; export * from "./bytes.js"; export * from "./command.js"; -export * from "./diff.js"; export * from "./err.js"; export * from "./errors.js"; export * from "./format.js"; From 4555604675bded9c9f00ead9d75725036702733d Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 03:26:10 -0400 Subject: [PATCH 49/79] chore: fix tests using shufflingCache.set method --- .../test/unit/chain/shufflingCache.test.ts | 12 ++++++++---- .../test/utils/validationData/attestation.ts | 19 ++++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index 5bf8e5835b3d..62b02cbf2b12 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -12,8 +12,12 @@ describe("ShufflingCache", function () { let shufflingCache: ShufflingCache; beforeEach(() => { - shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - shufflingCache.set(state.epochCtx.currentShuffling, currentDecisionRoot); + shufflingCache = new ShufflingCache(null, null, {maxShufflingCacheEpochs: 1}, [ + { + shuffling: state.epochCtx.currentShuffling, + decisionRoot: currentDecisionRoot, + }, + ]); }); it("should get shuffling from cache", async function () { @@ -26,7 +30,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch, "0x00"); expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert shuffling at other epochs does prune the cache - shufflingCache.set(state.epochCtx.previousShuffling, state.epochCtx.previousDecisionRoot); + shufflingCache["set"](state.epochCtx.previousShuffling, state.epochCtx.previousDecisionRoot); // the current shuffling is not available anymore expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toBeNull(); }); @@ -37,7 +41,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(previousEpoch, previousDecisionRoot); const shufflingRequest0 = shufflingCache.get(previousEpoch, previousDecisionRoot); const shufflingRequest1 = shufflingCache.get(previousEpoch, previousDecisionRoot); - shufflingCache.set(state.epochCtx.previousShuffling, previousDecisionRoot); + shufflingCache["set"](state.epochCtx.previousShuffling, previousDecisionRoot); expect(await shufflingRequest0).toEqual(state.epochCtx.previousShuffling); expect(await shufflingRequest1).toEqual(state.epochCtx.previousShuffling); }); diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index a2bd76452b0c..22f551cbb663 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -76,11 +76,20 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { dataAvailabilityStatus: DataAvailabilityStatus.PreData, }; - const shufflingCache = new ShufflingCache(); - shufflingCache.set(state.epochCtx.previousShuffling, state.epochCtx.previousDecisionRoot); - shufflingCache.set(state.epochCtx.currentShuffling, state.epochCtx.currentDecisionRoot); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - shufflingCache.set(state.epochCtx.nextShuffling!, state.epochCtx.nextDecisionRoot); + const shufflingCache = new ShufflingCache(null, null, {}, [ + { + shuffling: state.epochCtx.previousShuffling, + decisionRoot: state.epochCtx.previousDecisionRoot, + }, + { + shuffling: state.epochCtx.currentShuffling, + decisionRoot: state.epochCtx.currentDecisionRoot, + }, + { + shuffling: state.epochCtx.nextShuffling, + decisionRoot: state.epochCtx.nextDecisionRoot, + }, + ]); const forkChoice = { getBlock: (root) => { From d49bfee2a0a50effc664106441f06bbbb9ef3e14 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 03:33:03 -0400 Subject: [PATCH 50/79] feat: remove minTimeDelayToBuild --- packages/beacon-node/src/chain/shufflingCache.ts | 10 +++------- packages/cli/src/options/beaconNodeOptions/chain.ts | 11 ----------- .../cli/test/unit/options/beaconNodeOptions.test.ts | 2 -- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 8b7de8cf7c07..450c200149e7 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -8,6 +8,7 @@ import { import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; +import {callInNextEventLoop} from "../util/eventLoop.js"; /** * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case: @@ -23,8 +24,6 @@ const MAX_EPOCHS = 4; */ const MAX_PROMISES = 2; -const DEFAULT_MIN_TIME_DELAY_IN_MS = 5; - enum CacheItemType { shuffling, promise, @@ -46,7 +45,6 @@ type CacheItem = ShufflingCacheItem | PromiseCacheItem; export type ShufflingCacheOpts = { maxShufflingCacheEpochs?: number; - minTimeDelayToBuildShuffling?: number; }; /** @@ -62,7 +60,6 @@ export class ShufflingCache implements IShufflingCache { ); private readonly maxEpochs: number; - private readonly minTimeDelayToBuild: number; constructor( readonly metrics: Metrics | null = null, @@ -79,7 +76,6 @@ export class ShufflingCache implements IShufflingCache { } this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS; - this.minTimeDelayToBuild = opts.minTimeDelayToBuildShuffling ?? DEFAULT_MIN_TIME_DELAY_IN_MS; precalculatedShufflings?.map(({shuffling, decisionRoot}) => { if (shuffling !== null) { @@ -190,14 +186,14 @@ export class ShufflingCache implements IShufflingCache { * on a NICE thread with a rust implementation */ return new Promise((resolve) => { - setTimeout(() => { + callInNextEventLoop(() => { const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer(); const shuffling = computeEpochShuffling(state, activeIndices, epoch); timer?.(); this.set(shuffling, decisionRoot); resolve(shuffling); // wait until after the first slot to help with attestation and block proposal performance - }, this.minTimeDelayToBuild); + }); }); } diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index c905f86ad6ac..aae97b6db68f 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -27,7 +27,6 @@ export type ChainArgs = { broadcastValidationStrictness?: string; "chain.minSameMessageSignatureSetsToBatch"?: number; "chain.maxShufflingCacheEpochs"?: number; - "chain.minTimeDelayToBuildShuffling"?: number; "chain.archiveBlobEpochs"?: number; "chain.nHistoricalStates"?: boolean; "chain.nHistoricalStatesFileDataStore"?: boolean; @@ -61,8 +60,6 @@ export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] { minSameMessageSignatureSetsToBatch: args["chain.minSameMessageSignatureSetsToBatch"] ?? defaultOptions.chain.minSameMessageSignatureSetsToBatch, maxShufflingCacheEpochs: args["chain.maxShufflingCacheEpochs"] ?? defaultOptions.chain.maxShufflingCacheEpochs, - minTimeDelayToBuildShuffling: - args["chain.minTimeDelayToBuildShuffling"] ?? defaultOptions.chain.minTimeDelayToBuildShuffling, archiveBlobEpochs: args["chain.archiveBlobEpochs"], nHistoricalStates: args["chain.nHistoricalStates"] ?? defaultOptions.chain.nHistoricalStates, nHistoricalStatesFileDataStore: @@ -238,14 +235,6 @@ Will double processing times. Use only for debugging purposes.", group: "chain", }, - "chain.minTimeDelayToBuildShuffling": { - hidden: true, - description: "Minimum amount of time to delay before building a shuffling after its queued", - type: "number", - default: defaultOptions.chain.minTimeDelayToBuildShuffling, - group: "chain", - }, - "chain.archiveBlobEpochs": { description: "Number of epochs to retain finalized blobs (minimum of MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS)", type: "number", diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index cdcd831ed15f..d74ae73b966f 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -37,7 +37,6 @@ describe("options / beaconNodeOptions", () => { "chain.trustedSetup": "", "chain.minSameMessageSignatureSetsToBatch": 32, "chain.maxShufflingCacheEpochs": 100, - "chain.minTimeDelayToBuildShuffling": 5, "chain.archiveBlobEpochs": 10000, "chain.nHistoricalStates": true, "chain.nHistoricalStatesFileDataStore": true, @@ -146,7 +145,6 @@ describe("options / beaconNodeOptions", () => { trustedSetup: "", minSameMessageSignatureSetsToBatch: 32, maxShufflingCacheEpochs: 100, - minTimeDelayToBuildShuffling: 5, archiveBlobEpochs: 10000, nHistoricalStates: true, nHistoricalStatesFileDataStore: true, From 923addbfc4285da62b6fc71dba2bbf72474f26d0 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 21 Aug 2024 23:57:45 -0400 Subject: [PATCH 51/79] feat: return promise from insertPromise and then through build --- packages/beacon-node/src/chain/chain.ts | 2 +- .../beacon-node/src/chain/shufflingCache.ts | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 236e6a52b215..062a90cb821c 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -878,7 +878,7 @@ export class BeaconChain implements IBeaconChain { ): Promise { // this is to prevent multiple calls to get shuffling for the same epoch and dependent root // any subsequent calls of the same epoch and dependent root will wait for this promise to resolve - this.shufflingCache.insertPromise(attEpoch, shufflingDependentRoot); + void this.shufflingCache.insertPromise(attEpoch, shufflingDependentRoot); const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); let state: CachedBeaconStateAllForks; diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 450c200149e7..3a45f45f069e 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -88,7 +88,7 @@ export class ShufflingCache implements IShufflingCache { * Insert a promise to make sure we don't regen state for the same shuffling. * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. */ - insertPromise(epoch: Epoch, decisionRoot: RootHex): void { + insertPromise(epoch: Epoch, decisionRoot: RootHex): Promise { const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) .flatMap((innerMap) => Array.from(innerMap.values())) .filter((item) => isPromiseCacheItem(item)).length; @@ -113,6 +113,7 @@ export class ShufflingCache implements IShufflingCache { }; this.itemsByDecisionRootByEpoch.getOrDefault(epoch).set(decisionRoot, cacheItem); this.metrics?.shufflingCache.insertPromiseCount.inc(); + return promise; } /** @@ -180,21 +181,20 @@ export class ShufflingCache implements IShufflingCache { state: BeaconStateAllForks, activeIndices: ValidatorIndex[] ): Promise { - this.insertPromise(epoch, decisionRoot); + const promise = this.insertPromise(epoch, decisionRoot); /** * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations * on a NICE thread with a rust implementation */ - return new Promise((resolve) => { - callInNextEventLoop(() => { - const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer(); - const shuffling = computeEpochShuffling(state, activeIndices, epoch); - timer?.(); - this.set(shuffling, decisionRoot); - resolve(shuffling); - // wait until after the first slot to help with attestation and block proposal performance - }); + callInNextEventLoop(() => { + const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer(); + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + timer?.(); + this.set(shuffling, decisionRoot); + // wait until after the first slot to help with attestation and block proposal performance }); + + return promise; } /** From 1613d8fe9496e8853cf155dc2a192ee8e66a0a59 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 00:09:13 -0400 Subject: [PATCH 52/79] fix: update metrics names and help field --- packages/beacon-node/src/chain/shufflingCache.ts | 2 +- packages/beacon-node/src/metrics/metrics/lodestar.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 3a45f45f069e..7fc630fed6e4 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -210,7 +210,7 @@ export class ShufflingCache implements IShufflingCache { cacheItem.resolveFn(shuffling); this.metrics?.shufflingCache.shufflingPromiseResolutionTime.observe(Date.now() - cacheItem.timeInserted); } else { - this.metrics?.shufflingCache.shufflingRecalculated.inc(); + this.metrics?.shufflingCache.shufflingBuiltMultipleTimes.inc(); } } // set the shuffling diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 21fb92efeb66..86898d6b240f 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1280,9 +1280,9 @@ export function createLodestarMetrics( name: "lodestar_gossip_attestation_shuffling_cache_miss_count", help: "Count of shuffling cache miss", }), - shufflingRecalculated: register.gauge({ + shufflingBuiltMultipleTimes: register.gauge({ name: "lodestar_shuffling_cache_recalculated_shuffling_count", - help: "Count of shuffling cache promises that were discarded and the shuffling was built synchronously", + help: "Count of shuffling that were build multiple times", }), shufflingPromiseNotResolvedAndThrownAway: register.gauge({ name: "lodestar_shuffling_cache_promise_not_resolved_and_thrown_away_count", @@ -1298,12 +1298,12 @@ export function createLodestarMetrics( }), shufflingPromiseResolutionTime: register.histogram({ name: "lodestar_shuffling_cache_promise_resolution_time", - help: "Count of shuffling cache promises that were requested before the promise was resolved", + help: "Time from promise insertion until promise resolution when shuffling was ready", buckets: [1, 10, 100, 1000], }), shufflingCalculationTime: register.histogram({ name: "lodestar_shuffling_cache_shuffling_calculation_time", - help: "Count of shuffling cache promises that were requested before the promise was resolved", + help: "Run time of shuffling calculation", buckets: [0.5, 0.75, 1, 1.25, 1.5], }), }, From 0a64e36e35cc74b0d2b81d0a0811692143fb5ed5 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 01:02:37 -0400 Subject: [PATCH 53/79] feat: move build of shuffling to beforeProcessEpoch --- packages/beacon-node/src/chain/chain.ts | 2 +- .../beacon-node/src/chain/shufflingCache.ts | 14 ++------ .../state-transition/src/cache/epochCache.ts | 36 +++++++++---------- .../src/cache/epochTransitionCache.ts | 33 +++++++++++++---- .../src/util/epochShuffling.ts | 14 ++++---- 5 files changed, 55 insertions(+), 44 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 062a90cb821c..236e6a52b215 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -878,7 +878,7 @@ export class BeaconChain implements IBeaconChain { ): Promise { // this is to prevent multiple calls to get shuffling for the same epoch and dependent root // any subsequent calls of the same epoch and dependent root will wait for this promise to resolve - void this.shufflingCache.insertPromise(attEpoch, shufflingDependentRoot); + this.shufflingCache.insertPromise(attEpoch, shufflingDependentRoot); const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); let state: CachedBeaconStateAllForks; diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 7fc630fed6e4..778eb047c669 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -88,7 +88,7 @@ export class ShufflingCache implements IShufflingCache { * Insert a promise to make sure we don't regen state for the same shuffling. * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. */ - insertPromise(epoch: Epoch, decisionRoot: RootHex): Promise { + insertPromise(epoch: Epoch, decisionRoot: RootHex): void { const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) .flatMap((innerMap) => Array.from(innerMap.values())) .filter((item) => isPromiseCacheItem(item)).length; @@ -113,7 +113,6 @@ export class ShufflingCache implements IShufflingCache { }; this.itemsByDecisionRootByEpoch.getOrDefault(epoch).set(decisionRoot, cacheItem); this.metrics?.shufflingCache.insertPromiseCount.inc(); - return promise; } /** @@ -175,13 +174,8 @@ export class ShufflingCache implements IShufflingCache { /** * Queue asynchronous build for an EpochShuffling */ - build( - epoch: number, - decisionRoot: string, - state: BeaconStateAllForks, - activeIndices: ValidatorIndex[] - ): Promise { - const promise = this.insertPromise(epoch, decisionRoot); + build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void { + this.insertPromise(epoch, decisionRoot); /** * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations * on a NICE thread with a rust implementation @@ -193,8 +187,6 @@ export class ShufflingCache implements IShufflingCache { this.set(shuffling, decisionRoot); // wait until after the first slot to help with attestation and block proposal performance }); - - return promise; } /** diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 6425cd0cee50..3be17c1a8f7c 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -582,14 +582,13 @@ export class EpochCache { afterProcessEpoch( state: CachedBeaconStateAllForks, epochTransitionCache: { - nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; - nextEpochShufflingActiveIndicesLength: number; + nextShufflingDecisionRoot: RootHex; + nextShufflingActiveIndices: ValidatorIndex[]; nextEpochTotalActiveBalanceByIncrement: number; } ): void { const upcomingEpoch = this.nextEpoch; const epochAfterUpcoming = upcomingEpoch + 1; - const nextActiveIndicesLength = epochTransitionCache.nextEpochShufflingActiveIndicesLength; // move current to previous this.previousShuffling = this.currentShuffling; @@ -618,32 +617,29 @@ export class EpochCache { // next epoch was moved to current epoch so use current here this.proposers = computeProposers(upcomingProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements); - // calculate next values - this.nextDecisionRoot = calculateShufflingDecisionRoot(state.config, state, epochAfterUpcoming); - this.nextActiveIndices = new Array(nextActiveIndicesLength); - - if (nextActiveIndicesLength > epochTransitionCache.nextEpochShufflingActiveValidatorIndices.length) { - throw new Error( - `Invalid activeValidatorCount: ${nextActiveIndicesLength} > ${epochTransitionCache.nextEpochShufflingActiveValidatorIndices.length}` - ); - } - // only the first `activeValidatorCount` elements are copied to `activeIndices` - for (let i = 0; i < nextActiveIndicesLength; i++) { - this.nextActiveIndices[i] = epochTransitionCache.nextEpochShufflingActiveValidatorIndices[i]; - } - + // handle next values + this.nextDecisionRoot = epochTransitionCache.nextShufflingDecisionRoot; + this.nextActiveIndices = epochTransitionCache.nextShufflingActiveIndices; if (this.shufflingCache) { this.nextShuffling = null; - const decisionRoot = this.nextDecisionRoot; + // This promise will resolve immediately after the synchronous code of the state-transition runs. Until + // the build is done on a worker thread it will be calculated immediately after the epoch transition + // completes. Once the work is done concurrently it should be ready by time this get runs so the promise + // will resolve directly on the next spin of the event loop because the epoch transition and shuffling take + // about the same time to calculate so theoretically its ready now. Do not await here though in case it + // is not ready yet as the transition must not be asynchronous. this.shufflingCache - ?.build(epochAfterUpcoming, this.nextDecisionRoot, state, this.nextActiveIndices) + .get(epochAfterUpcoming, this.nextDecisionRoot) .then((shuffling) => { + if (!shuffling) { + throw new Error("EpochShuffling not returned from get in afterProcessEpoch"); + } this.nextShuffling = shuffling; }) .catch((err) => { this.shufflingCache?.logger?.error( "EPOCH_CONTEXT_SHUFFLING_BUILD_ERROR", - {epoch: epochAfterUpcoming, decisionRoot}, + {epoch: epochAfterUpcoming, decisionRoot: epochTransitionCache.nextShufflingDecisionRoot}, err ); }); diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index e6f84de6c62e..ec7d7f706544 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,4 +1,4 @@ -import {Epoch, ValidatorIndex} from "@lodestar/types"; +import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; import {intDiv} from "@lodestar/utils"; import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; @@ -14,6 +14,7 @@ import { FLAG_CURR_HEAD_ATTESTER, } from "../util/attesterStatus.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from "../index.js"; +import {calculateShufflingDecisionRoot} from "../util/epochShuffling.js"; import {computeBaseRewardPerIncrement} from "../util/altair.js"; import {processPendingAttestations} from "../epoch/processPendingAttestations.js"; @@ -143,12 +144,12 @@ export interface EpochTransitionCache { * | beforeProcessEpoch | calculate during the validator loop| * | afterEpochTransitionCache | read it | */ - nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; + nextShufflingActiveIndices: ValidatorIndex[]; /** - * We do not use up to `nextEpochShufflingActiveValidatorIndices.length`, use this to control that + * Shuffling decision root that gets set on the EpochCache in afterProcessEpoch */ - nextEpochShufflingActiveIndicesLength: number; + nextShufflingDecisionRoot: RootHex; /** * Altair specific, this is total active balances for the next epoch. @@ -348,6 +349,26 @@ export function beforeProcessEpoch( } } + // Trigger async build of shuffling for epoch after next (nextShuffling post epoch transition) + const epochAfterUpcoming = state.epochCtx.nextEpoch + 1; + const nextShufflingDecisionRoot = calculateShufflingDecisionRoot(state.config, state, epochAfterUpcoming); + const nextShufflingActiveIndices = new Array(nextEpochShufflingActiveIndicesLength); + if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { + throw new Error( + `Invalid activeValidatorCount: ${nextEpochShufflingActiveIndicesLength} > ${nextEpochShufflingActiveValidatorIndices.length}` + ); + } + // only the first `activeValidatorCount` elements are copied to `activeIndices` + for (let i = 0; i < nextEpochShufflingActiveIndicesLength; i++) { + nextShufflingActiveIndices[i] = nextEpochShufflingActiveValidatorIndices[i]; + } + state.epochCtx.shufflingCache?.build( + epochAfterUpcoming, + nextShufflingDecisionRoot, + state, + nextShufflingActiveIndices + ); + if (totalActiveStakeByIncrement < 1) { totalActiveStakeByIncrement = 1; } else if (totalActiveStakeByIncrement >= Number.MAX_SAFE_INTEGER) { @@ -471,8 +492,8 @@ export function beforeProcessEpoch( indicesEligibleForActivationQueue, indicesEligibleForActivation, indicesToEject, - nextEpochShufflingActiveValidatorIndices, - nextEpochShufflingActiveIndicesLength, + nextShufflingDecisionRoot, + nextShufflingActiveIndices, // to be updated in processEffectiveBalanceUpdates nextEpochTotalActiveBalanceByIncrement: 0, isActivePrevEpoch, diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 38330320024d..5c4a3785f93d 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -44,15 +44,17 @@ export interface IShufflingCache { buildProps?: T ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null; + /** + * Gets a cached shuffling via the epoch and decision root. Returns a promise + * for the shuffling if it hs not calculated yet. Returns null if a build has + * not been queued nor a shuffling was calculated. + */ + get(epoch: Epoch, decisionRoot: RootHex): Promise; + /** * Queue asynchronous build for an EpochShuffling */ - build( - epoch: Epoch, - decisionRoot: RootHex, - state: BeaconStateAllForks, - activeIndices: ValidatorIndex[] - ): Promise; + build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void; } /** From ed850ee0ced72029b643db4e86d82600cfb7cf43 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 01:13:45 -0400 Subject: [PATCH 54/79] feat: allow calc of pivot slot before slot increment --- .../state-transition/src/cache/epochTransitionCache.ts | 2 +- packages/state-transition/src/util/epochShuffling.ts | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index ec7d7f706544..1d2f5bbd6390 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -351,7 +351,7 @@ export function beforeProcessEpoch( // Trigger async build of shuffling for epoch after next (nextShuffling post epoch transition) const epochAfterUpcoming = state.epochCtx.nextEpoch + 1; - const nextShufflingDecisionRoot = calculateShufflingDecisionRoot(state.config, state, epochAfterUpcoming); + const nextShufflingDecisionRoot = calculateShufflingDecisionRoot(state.config, state, epochAfterUpcoming, true); const nextShufflingActiveIndices = new Array(nextEpochShufflingActiveIndicesLength); if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { throw new Error( diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 5c4a3785f93d..ce4ad136c0b6 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -143,8 +143,9 @@ export function computeEpochShuffling( }; } -function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch): RootHex { - const pivotSlot = computeStartSlotAtEpoch(epoch - 1) - 1; +function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch, beforeSlotIncrement = false): RootHex { + const pivotEpoch = beforeSlotIncrement ? epoch : epoch - 1; + const pivotSlot = computeStartSlotAtEpoch(pivotEpoch) - 1; return toRootHex(getBlockRootAtSlot(state, pivotSlot)); } @@ -156,9 +157,10 @@ function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch): RootHe export function calculateShufflingDecisionRoot( config: BeaconConfig, state: BeaconStateAllForks, - epoch: Epoch + epoch: Epoch, + beforeSlotIncrement?: boolean ): RootHex { return state.slot > GENESIS_SLOT - ? calculateDecisionRoot(state, epoch) + ? calculateDecisionRoot(state, epoch, beforeSlotIncrement) : toHexString(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(config, state).blockHeader)); } From 5e65f7efa85fad169a6263d1bebc1b4f8d3701d4 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 01:27:09 -0400 Subject: [PATCH 55/79] fix: calc of pivot slot before slot increment --- packages/state-transition/src/util/epochShuffling.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index ce4ad136c0b6..2342cd327c41 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -144,8 +144,10 @@ export function computeEpochShuffling( } function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch, beforeSlotIncrement = false): RootHex { - const pivotEpoch = beforeSlotIncrement ? epoch : epoch - 1; - const pivotSlot = computeStartSlotAtEpoch(pivotEpoch) - 1; + let pivotSlot = computeStartSlotAtEpoch(epoch - 1); + if (!beforeSlotIncrement) { + pivotSlot--; + } return toRootHex(getBlockRootAtSlot(state, pivotSlot)); } From 2c1100c4642e4557dc94a27da6ce00ef4a5af7fc Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 02:37:21 -0400 Subject: [PATCH 56/79] Revert "fix: calc of pivot slot before slot increment" This reverts commit 5e65f7efa85fad169a6263d1bebc1b4f8d3701d4. --- packages/state-transition/src/util/epochShuffling.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 2342cd327c41..ce4ad136c0b6 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -144,10 +144,8 @@ export function computeEpochShuffling( } function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch, beforeSlotIncrement = false): RootHex { - let pivotSlot = computeStartSlotAtEpoch(epoch - 1); - if (!beforeSlotIncrement) { - pivotSlot--; - } + const pivotEpoch = beforeSlotIncrement ? epoch : epoch - 1; + const pivotSlot = computeStartSlotAtEpoch(pivotEpoch) - 1; return toRootHex(getBlockRootAtSlot(state, pivotSlot)); } From d5317eca05094ecbbf7c6f58cacac5791ea9da44 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 02:37:35 -0400 Subject: [PATCH 57/79] Revert "feat: allow calc of pivot slot before slot increment" This reverts commit ed850ee0ced72029b643db4e86d82600cfb7cf43. --- .../state-transition/src/cache/epochTransitionCache.ts | 2 +- packages/state-transition/src/util/epochShuffling.ts | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 1d2f5bbd6390..ec7d7f706544 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -351,7 +351,7 @@ export function beforeProcessEpoch( // Trigger async build of shuffling for epoch after next (nextShuffling post epoch transition) const epochAfterUpcoming = state.epochCtx.nextEpoch + 1; - const nextShufflingDecisionRoot = calculateShufflingDecisionRoot(state.config, state, epochAfterUpcoming, true); + const nextShufflingDecisionRoot = calculateShufflingDecisionRoot(state.config, state, epochAfterUpcoming); const nextShufflingActiveIndices = new Array(nextEpochShufflingActiveIndicesLength); if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { throw new Error( diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index ce4ad136c0b6..5c4a3785f93d 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -143,9 +143,8 @@ export function computeEpochShuffling( }; } -function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch, beforeSlotIncrement = false): RootHex { - const pivotEpoch = beforeSlotIncrement ? epoch : epoch - 1; - const pivotSlot = computeStartSlotAtEpoch(pivotEpoch) - 1; +function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch): RootHex { + const pivotSlot = computeStartSlotAtEpoch(epoch - 1) - 1; return toRootHex(getBlockRootAtSlot(state, pivotSlot)); } @@ -157,10 +156,9 @@ function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch, beforeS export function calculateShufflingDecisionRoot( config: BeaconConfig, state: BeaconStateAllForks, - epoch: Epoch, - beforeSlotIncrement?: boolean + epoch: Epoch ): RootHex { return state.slot > GENESIS_SLOT - ? calculateDecisionRoot(state, epoch, beforeSlotIncrement) + ? calculateDecisionRoot(state, epoch) : toHexString(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(config, state).blockHeader)); } From 82ac512f809c3e154fe2326d18883d1b3c30964e Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 02:39:45 -0400 Subject: [PATCH 58/79] feat: allow getting current block root for shuffling calculation --- packages/state-transition/src/util/blockRoot.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/state-transition/src/util/blockRoot.ts b/packages/state-transition/src/util/blockRoot.ts index 54d96885e675..5c518c2b4540 100644 --- a/packages/state-transition/src/util/blockRoot.ts +++ b/packages/state-transition/src/util/blockRoot.ts @@ -17,8 +17,8 @@ import {computeStartSlotAtEpoch} from "./epoch.js"; * Return the block root at a recent [[slot]]. */ export function getBlockRootAtSlot(state: BeaconStateAllForks, slot: Slot): Root { - if (slot >= state.slot) { - throw Error(`Can only get block root in the past currentSlot=${state.slot} slot=${slot}`); + if (slot > state.slot) { + throw Error(`Can only get current block root, or one in the past currentSlot=${state.slot} slot=${slot}`); } if (slot < state.slot - SLOTS_PER_HISTORICAL_ROOT) { throw Error(`Cannot get block root more than ${SLOTS_PER_HISTORICAL_ROOT} in the past`); From aa9ab9c69313e9cb68dabdfad0f4e5a39521c258 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 03:12:20 -0400 Subject: [PATCH 59/79] fix: get nextShufflingDecisionRoot directly from state.blockRoots --- .../src/cache/epochTransitionCache.ts | 11 ++++++++--- packages/state-transition/src/util/blockRoot.ts | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index ec7d7f706544..f2309908fa9b 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,6 +1,12 @@ import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; import {intDiv} from "@lodestar/utils"; -import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; +import { + EPOCHS_PER_SLASHINGS_VECTOR, + FAR_FUTURE_EPOCH, + ForkSeq, + MAX_EFFECTIVE_BALANCE, + SLOTS_PER_HISTORICAL_ROOT, +} from "@lodestar/params"; import { hasMarkers, @@ -14,7 +20,6 @@ import { FLAG_CURR_HEAD_ATTESTER, } from "../util/attesterStatus.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from "../index.js"; -import {calculateShufflingDecisionRoot} from "../util/epochShuffling.js"; import {computeBaseRewardPerIncrement} from "../util/altair.js"; import {processPendingAttestations} from "../epoch/processPendingAttestations.js"; @@ -351,7 +356,7 @@ export function beforeProcessEpoch( // Trigger async build of shuffling for epoch after next (nextShuffling post epoch transition) const epochAfterUpcoming = state.epochCtx.nextEpoch + 1; - const nextShufflingDecisionRoot = calculateShufflingDecisionRoot(state.config, state, epochAfterUpcoming); + const nextShufflingDecisionRoot = state.blockRoots.get(state.slot % SLOTS_PER_HISTORICAL_ROOT); const nextShufflingActiveIndices = new Array(nextEpochShufflingActiveIndicesLength); if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { throw new Error( diff --git a/packages/state-transition/src/util/blockRoot.ts b/packages/state-transition/src/util/blockRoot.ts index 5c518c2b4540..54d96885e675 100644 --- a/packages/state-transition/src/util/blockRoot.ts +++ b/packages/state-transition/src/util/blockRoot.ts @@ -17,8 +17,8 @@ import {computeStartSlotAtEpoch} from "./epoch.js"; * Return the block root at a recent [[slot]]. */ export function getBlockRootAtSlot(state: BeaconStateAllForks, slot: Slot): Root { - if (slot > state.slot) { - throw Error(`Can only get current block root, or one in the past currentSlot=${state.slot} slot=${slot}`); + if (slot >= state.slot) { + throw Error(`Can only get block root in the past currentSlot=${state.slot} slot=${slot}`); } if (slot < state.slot - SLOTS_PER_HISTORICAL_ROOT) { throw Error(`Cannot get block root more than ${SLOTS_PER_HISTORICAL_ROOT} in the past`); From ca8445ac31045683e19561a39b61bf0263615f4c Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 03:16:56 -0400 Subject: [PATCH 60/79] fix: convert toRootHex --- packages/state-transition/src/cache/epochTransitionCache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index f2309908fa9b..64785260b445 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,5 +1,5 @@ import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; -import {intDiv} from "@lodestar/utils"; +import {intDiv, toRootHex} from "@lodestar/utils"; import { EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, @@ -356,7 +356,7 @@ export function beforeProcessEpoch( // Trigger async build of shuffling for epoch after next (nextShuffling post epoch transition) const epochAfterUpcoming = state.epochCtx.nextEpoch + 1; - const nextShufflingDecisionRoot = state.blockRoots.get(state.slot % SLOTS_PER_HISTORICAL_ROOT); + const nextShufflingDecisionRoot = toRootHex(state.blockRoots.get(state.slot % SLOTS_PER_HISTORICAL_ROOT)); const nextShufflingActiveIndices = new Array(nextEpochShufflingActiveIndicesLength); if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { throw new Error( From 19ae447fbbb316f4545b213b0a9620df096a832f Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 22 Aug 2024 03:23:13 -0400 Subject: [PATCH 61/79] docs: add comment about pulling decisionRoot directly from state --- packages/state-transition/src/cache/epochTransitionCache.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 64785260b445..d6e02f57006b 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -356,6 +356,9 @@ export function beforeProcessEpoch( // Trigger async build of shuffling for epoch after next (nextShuffling post epoch transition) const epochAfterUpcoming = state.epochCtx.nextEpoch + 1; + // cannot call calculateShufflingDecisionRoot here because spec prevent getting current slot + // as a decision block. we are part way through the transition though and this was added in + // process slot beforeProcessEpoch happens so it available and valid const nextShufflingDecisionRoot = toRootHex(state.blockRoots.get(state.slot % SLOTS_PER_HISTORICAL_ROOT)); const nextShufflingActiveIndices = new Array(nextEpochShufflingActiveIndicesLength); if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { From ff6694d7834577fdd7cf964013572b350cf3e614 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 23 Aug 2024 17:37:22 -0400 Subject: [PATCH 62/79] feat: add back metrics for regen attestation cache hit/miss --- .../src/chain/validation/attestation.ts | 3 +++ .../beacon-node/src/metrics/metrics/lodestar.ts | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 0d574999e9a3..1116e87e1d25 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -586,9 +586,12 @@ export async function getShufflingForAttestationVerification( const shuffling = await chain.shufflingCache.get(attEpoch, shufflingDependentRoot); if (shuffling) { + // most of the time, we should get the shuffling from cache + chain.metrics?.gossipAttestation.shufflingCacheHit.inc({caller: regenCaller}); return shuffling; } + chain.metrics?.gossipAttestation.shufflingCacheMiss.inc({caller: regenCaller}); try { // for the 1st time of the same epoch and dependent root, it awaits for the regen state // from the 2nd time, it should use the same cached promise and it should reach the above code diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 86898d6b240f..ad76db8dcc43 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -630,6 +630,16 @@ export function createLodestarMetrics( labelNames: ["caller"], buckets: [0, 1, 2, 4, 8, 16, 32, 64], }), + shufflingCacheHit: register.gauge<{caller: RegenCaller}>({ + name: "lodestar_gossip_attestation_shuffling_cache_hit_count", + help: "Count of gossip attestation verification shuffling cache hit", + labelNames: ["caller"], + }), + shufflingCacheMiss: register.gauge<{caller: RegenCaller}>({ + name: "lodestar_gossip_attestation_shuffling_cache_miss_count", + help: "Count of gossip attestation verification shuffling cache miss", + labelNames: ["caller"], + }), shufflingCacheRegenHit: register.gauge<{caller: RegenCaller}>({ name: "lodestar_gossip_attestation_shuffling_cache_regen_hit_count", help: "Count of gossip attestation verification shuffling cache regen hit", @@ -1273,11 +1283,11 @@ export function createLodestarMetrics( help: "Total number of times insertPromise is called", }), hit: register.gauge({ - name: "lodestar_gossip_attestation_shuffling_cache_hit_count", + name: "lodestar_shuffling_cache_hit_count", help: "Count of shuffling cache hit", }), miss: register.gauge({ - name: "lodestar_gossip_attestation_shuffling_cache_miss_count", + name: "lodestar_shuffling_cache_miss_count", help: "Count of shuffling cache miss", }), shufflingBuiltMultipleTimes: register.gauge({ From 1a7ab441596f5e8e4cd646eb692695f62834ef02 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 28 Aug 2024 02:00:17 -0400 Subject: [PATCH 63/79] docs: fix docstring on shufflingCache.build --- packages/beacon-node/src/chain/shufflingCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 778eb047c669..8bbdb9176fcc 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -172,7 +172,7 @@ export class ShufflingCache implements IShufflingCache { } /** - * Queue asynchronous build for an EpochShuffling + * Queue asynchronous build for an EpochShuffling, triggered from state-transition */ build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void { this.insertPromise(epoch, decisionRoot); From 85a9ae127fc9f2f85993232afb28d879b2cf3042 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 28 Aug 2024 02:19:29 -0400 Subject: [PATCH 64/79] refactor: change validatorIndices to Uint32Array --- .../beacon-node/src/chain/shufflingCache.ts | 2 +- .../state-transition/src/cache/epochCache.ts | 21 +++++++++++-------- .../src/cache/epochTransitionCache.ts | 7 ++++--- .../src/util/epochShuffling.ts | 11 +++++----- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 8bbdb9176fcc..01be8d03b35b 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -174,7 +174,7 @@ export class ShufflingCache implements IShufflingCache { /** * Queue asynchronous build for an EpochShuffling, triggered from state-transition */ - build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void { + build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: Uint32Array): void { this.insertPromise(epoch, decisionRoot); /** * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 3be17c1a8f7c..755ba45cf4a1 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -149,7 +149,7 @@ export class EpochCache { * Cache nextActiveIndices so that in afterProcessEpoch the next shuffling can be build synchronously * in case it is not built or the ShufflingCache is not available */ - nextActiveIndices: ValidatorIndex[]; + nextActiveIndices: Uint32Array; /** * Effective balances, for altair processAttestations() */ @@ -241,7 +241,7 @@ export class EpochCache { previousShuffling: EpochShuffling; currentShuffling: EpochShuffling; nextShuffling: EpochShuffling | null; - nextActiveIndices: ValidatorIndex[]; + nextActiveIndices: Uint32Array; effectiveBalanceIncrements: EffectiveBalanceIncrements; totalSlashingsByIncrement: number; syncParticipantReward: number; @@ -322,9 +322,9 @@ export class EpochCache { const effectiveBalanceIncrements = getEffectiveBalanceIncrementsWithLen(validatorCount); const totalSlashingsByIncrement = getTotalSlashingsByIncrement(state); - const previousActiveIndices: ValidatorIndex[] = []; - const currentActiveIndices: ValidatorIndex[] = []; - const nextActiveIndices: ValidatorIndex[] = []; + const previousActiveIndicesAsNumberArray: ValidatorIndex[] = []; + const currentActiveIndicesAsNumberArray: ValidatorIndex[] = []; + const nextActiveIndicesAsNumberArray: ValidatorIndex[] = []; // BeaconChain could provide a shuffling cache to avoid re-computing shuffling every epoch // in that case, we don't need to compute shufflings again @@ -344,17 +344,17 @@ export class EpochCache { // we only need to track active indices for previous, current and next epoch if we have to compute shufflings // skip doing that if we already have cached shufflings if (cachedPreviousShuffling == null && isActiveValidator(validator, previousEpoch)) { - previousActiveIndices.push(i); + previousActiveIndicesAsNumberArray.push(i); } if (isActiveValidator(validator, currentEpoch)) { if (cachedCurrentShuffling == null) { - currentActiveIndices.push(i); + currentActiveIndicesAsNumberArray.push(i); } // We track totalActiveBalanceIncrements as ETH to fit total network balance in a JS number (53 bits) totalActiveBalanceIncrements += effectiveBalanceIncrements[i]; } if (cachedNextShuffling == null && isActiveValidator(validator, nextEpoch)) { - nextActiveIndices.push(i); + nextActiveIndicesAsNumberArray.push(i); } const {exitEpoch} = validator; @@ -376,6 +376,9 @@ export class EpochCache { throw Error("totalActiveBalanceIncrements >= Number.MAX_SAFE_INTEGER. MAX_EFFECTIVE_BALANCE is too low."); } + const previousActiveIndices = new Uint32Array(previousActiveIndicesAsNumberArray); + const currentActiveIndices = new Uint32Array(currentActiveIndicesAsNumberArray); + const nextActiveIndices = new Uint32Array(nextActiveIndicesAsNumberArray); let previousShuffling: EpochShuffling; let currentShuffling: EpochShuffling; let nextShuffling: EpochShuffling; @@ -583,7 +586,7 @@ export class EpochCache { state: CachedBeaconStateAllForks, epochTransitionCache: { nextShufflingDecisionRoot: RootHex; - nextShufflingActiveIndices: ValidatorIndex[]; + nextShufflingActiveIndices: Uint32Array; nextEpochTotalActiveBalanceByIncrement: number; } ): void { diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index d6e02f57006b..9d36436dd8b9 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -149,7 +149,7 @@ export interface EpochTransitionCache { * | beforeProcessEpoch | calculate during the validator loop| * | afterEpochTransitionCache | read it | */ - nextShufflingActiveIndices: ValidatorIndex[]; + nextShufflingActiveIndices: Uint32Array; /** * Shuffling decision root that gets set on the EpochCache in afterProcessEpoch @@ -360,7 +360,7 @@ export function beforeProcessEpoch( // as a decision block. we are part way through the transition though and this was added in // process slot beforeProcessEpoch happens so it available and valid const nextShufflingDecisionRoot = toRootHex(state.blockRoots.get(state.slot % SLOTS_PER_HISTORICAL_ROOT)); - const nextShufflingActiveIndices = new Array(nextEpochShufflingActiveIndicesLength); + const _nextShufflingActiveIndices = new Array(nextEpochShufflingActiveIndicesLength); if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { throw new Error( `Invalid activeValidatorCount: ${nextEpochShufflingActiveIndicesLength} > ${nextEpochShufflingActiveValidatorIndices.length}` @@ -368,8 +368,9 @@ export function beforeProcessEpoch( } // only the first `activeValidatorCount` elements are copied to `activeIndices` for (let i = 0; i < nextEpochShufflingActiveIndicesLength; i++) { - nextShufflingActiveIndices[i] = nextEpochShufflingActiveValidatorIndices[i]; + _nextShufflingActiveIndices[i] = nextEpochShufflingActiveValidatorIndices[i]; } + const nextShufflingActiveIndices = new Uint32Array(_nextShufflingActiveIndices); state.epochCtx.shufflingCache?.build( epochAfterUpcoming, nextShufflingDecisionRoot, diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 5c4a3785f93d..08af07e46fa7 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -18,7 +18,7 @@ import {computeAnchorCheckpoint} from "./computeAnchorCheckpoint.js"; export interface ShufflingBuildProps { state: BeaconStateAllForks; - activeIndices: ValidatorIndex[]; + activeIndices: Uint32Array; } export interface PublicShufflingCacheMetrics { @@ -54,7 +54,7 @@ export interface IShufflingCache { /** * Queue asynchronous build for an EpochShuffling */ - build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: ValidatorIndex[]): void; + build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: Uint32Array): void; } /** @@ -105,13 +105,12 @@ export function computeCommitteeCount(activeValidatorCount: number): number { export function computeEpochShuffling( state: BeaconStateAllForks, - activeIndices: ValidatorIndex[], + activeIndices: Uint32Array, epoch: Epoch ): EpochShuffling { - const _activeIndices = new Uint32Array(activeIndices); const activeValidatorCount = activeIndices.length; - const shuffling = _activeIndices.slice(); + const shuffling = activeIndices.slice(); const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); unshuffleList(shuffling, seed); @@ -136,7 +135,7 @@ export function computeEpochShuffling( return { epoch, - activeIndices: _activeIndices, + activeIndices, shuffling, committees, committeesPerSlot, From ee1256e4258c703efd7473347ffed853e37eb9f9 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 28 Aug 2024 02:46:33 -0400 Subject: [PATCH 65/79] refactor: remove comment and change variable name --- packages/state-transition/src/cache/epochCache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 755ba45cf4a1..73dfc3aa9da5 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -400,7 +400,7 @@ export class EpochCache { previousShuffling = cachedPreviousShuffling ? cachedPreviousShuffling : isGenesis - ? currentShuffling // TODO: (@matthewkeil) does this need to be added to the cache at previousEpoch and previousDecisionRoot? + ? currentShuffling : shufflingCache.getSync(previousEpoch, previousDecisionRoot, { state, activeIndices: previousActiveIndices, @@ -606,7 +606,7 @@ export class EpochCache { } else { this.shufflingCache?.metrics?.shufflingCache.nextShufflingNotOnEpochCache.inc(); this.currentShuffling = - this.shufflingCache?.getSync(upcomingEpoch, this.nextDecisionRoot, { + this.shufflingCache?.getSync(upcomingEpoch, this.currentDecisionRoot, { state, // have to use the "nextActiveIndices" that were saved in the last transition here to calculate // the upcoming shuffling if it is not already built (similar condition to the below computation) From eff10c4fc612d619d2803e6cb7ce839ce17fa5f2 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 28 Aug 2024 02:47:48 -0400 Subject: [PATCH 66/79] fix: use toRootHex instead of toHexString --- packages/state-transition/src/util/epochShuffling.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 08af07e46fa7..c26c62fd2079 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {Epoch, RootHex, ssz, ValidatorIndex} from "@lodestar/types"; import {GaugeExtra, intDiv, Logger, NoLabels, toRootHex} from "@lodestar/utils"; import { @@ -159,5 +158,5 @@ export function calculateShufflingDecisionRoot( ): RootHex { return state.slot > GENESIS_SLOT ? calculateDecisionRoot(state, epoch) - : toHexString(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(config, state).blockHeader)); + : toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(config, state).blockHeader)); } From 4d06deb8bfc01fdd040df2da6502ba5f0cb55450 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 28 Aug 2024 02:56:54 -0400 Subject: [PATCH 67/79] refactor: deduplicate moved function computeAnchorCheckpoint --- packages/beacon-node/src/chain/chain.ts | 2 +- .../beacon-node/src/chain/forkChoice/index.ts | 2 +- packages/beacon-node/src/chain/initState.ts | 38 +------------------ .../beacon-node/src/sync/backfill/backfill.ts | 3 +- .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../unit/chain/forkChoice/forkChoice.test.ts | 3 +- packages/state-transition/src/util/index.ts | 1 + 7 files changed, 9 insertions(+), 42 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 236e6a52b215..61b2fb61687b 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -13,6 +13,7 @@ import { PubkeyIndexMap, EpochShuffling, computeEndSlotAtEpoch, + computeAnchorCheckpoint, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import { @@ -60,7 +61,6 @@ import { import {IChainOptions} from "./options.js"; import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {initializeForkChoice} from "./forkChoice/index.js"; -import {computeAnchorCheckpoint} from "./initState.js"; import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool} from "./bls/index.js"; import { SeenAttesters, diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index d57b7f86cb98..346a4afe1e7f 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -14,10 +14,10 @@ import { getEffectiveBalanceIncrementsZeroInactive, isExecutionStateType, isMergeTransitionComplete, + computeAnchorCheckpoint, } from "@lodestar/state-transition"; import {Logger, toRootHex} from "@lodestar/utils"; -import {computeAnchorCheckpoint} from "../initState.js"; import {ChainEventEmitter} from "../emitter.js"; import {ChainEvent} from "../emitter.js"; import {GENESIS_SLOT} from "../../constants/index.js"; diff --git a/packages/beacon-node/src/chain/initState.ts b/packages/beacon-node/src/chain/initState.ts index 19f0f1cc959d..0707c99ce825 100644 --- a/packages/beacon-node/src/chain/initState.ts +++ b/packages/beacon-node/src/chain/initState.ts @@ -1,15 +1,13 @@ import { - blockToHeader, computeEpochAtSlot, BeaconStateAllForks, CachedBeaconStateAllForks, - computeCheckpointEpochAtStateSlot, computeStartSlotAtEpoch, } from "@lodestar/state-transition"; -import {SignedBeaconBlock, phase0, ssz} from "@lodestar/types"; +import {SignedBeaconBlock} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {Logger, toHex, toRootHex} from "@lodestar/utils"; -import {GENESIS_SLOT, ZERO_HASH} from "../constants/index.js"; +import {GENESIS_SLOT} from "../constants/index.js"; import {IBeaconDb} from "../db/index.js"; import {Eth1Provider} from "../eth1/index.js"; import {Metrics} from "../metrics/index.js"; @@ -202,35 +200,3 @@ export function initBeaconMetrics(metrics: Metrics, state: BeaconStateAllForks): metrics.currentJustifiedEpoch.set(state.currentJustifiedCheckpoint.epoch); metrics.finalizedEpoch.set(state.finalizedCheckpoint.epoch); } - -export function computeAnchorCheckpoint( - config: ChainForkConfig, - anchorState: BeaconStateAllForks -): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader} { - let blockHeader; - let root; - const blockTypes = config.getForkTypes(anchorState.latestBlockHeader.slot); - - if (anchorState.latestBlockHeader.slot === GENESIS_SLOT) { - const block = blockTypes.BeaconBlock.defaultValue(); - block.stateRoot = anchorState.hashTreeRoot(); - blockHeader = blockToHeader(config, block); - root = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader); - } else { - blockHeader = ssz.phase0.BeaconBlockHeader.clone(anchorState.latestBlockHeader); - if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { - blockHeader.stateRoot = anchorState.hashTreeRoot(); - } - root = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader); - } - - return { - checkpoint: { - root, - // the checkpoint epoch = computeEpochAtSlot(anchorState.slot) + 1 if slot is not at epoch boundary - // this is similar to a process_slots() call - epoch: computeCheckpointEpochAtStateSlot(anchorState.slot), - }, - blockHeader, - }; -} diff --git a/packages/beacon-node/src/sync/backfill/backfill.ts b/packages/beacon-node/src/sync/backfill/backfill.ts index 38258fde07fd..77d2836bdcc3 100644 --- a/packages/beacon-node/src/sync/backfill/backfill.ts +++ b/packages/beacon-node/src/sync/backfill/backfill.ts @@ -1,6 +1,6 @@ import {EventEmitter} from "events"; import {StrictEventEmitter} from "strict-event-emitter-types"; -import {BeaconStateAllForks, blockToHeader} from "@lodestar/state-transition"; +import {BeaconStateAllForks, blockToHeader, computeAnchorCheckpoint} from "@lodestar/state-transition"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {phase0, Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types"; import {ErrorAborted, Logger, sleep, toRootHex} from "@lodestar/utils"; @@ -15,7 +15,6 @@ import {PeerIdStr} from "../../util/peerId.js"; import {shuffleOne} from "../../util/shuffle.js"; import {Metrics} from "../../metrics/metrics"; import {byteArrayEquals} from "../../util/bytes.js"; -import {computeAnchorCheckpoint} from "../../chain/initState.js"; import {verifyBlockProposerSignature, verifyBlockSequence, BackfillBlockHeader, BackfillBlock} from "./verify.js"; import {BackfillSyncError, BackfillSyncErrorCode} from "./errors.js"; /** diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 60ff6ce48302..f12ec1e2efcf 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -2,6 +2,7 @@ import {itBench} from "@dapplion/benchmark"; import {BitArray, toHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAltair, + computeAnchorCheckpoint, computeEpochAtSlot, computeStartSlotAtEpoch, getBlockRootAtSlot, @@ -13,7 +14,6 @@ import {ssz} from "@lodestar/types"; // eslint-disable-next-line import/no-relative-packages import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; -import {computeAnchorCheckpoint} from "../../../../src/chain/initState.js"; const vc = 1_500_000; diff --git a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts index 6b96a0d1172f..611673086ce5 100644 --- a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts +++ b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts @@ -5,12 +5,13 @@ import {CheckpointWithHex, ExecutionStatus, ForkChoice, DataAvailabilityStatus} import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import { CachedBeaconStateAllForks, + computeAnchorCheckpoint, computeEpochAtSlot, getEffectiveBalanceIncrementsZeroed, } from "@lodestar/state-transition"; import {phase0, Slot, ssz, ValidatorIndex} from "@lodestar/types"; import {getTemporaryBlockHeader, processSlots} from "@lodestar/state-transition"; -import {ChainEventEmitter, computeAnchorCheckpoint, initializeForkChoice} from "../../../../src/chain/index.js"; +import {ChainEventEmitter, initializeForkChoice} from "../../../../src/chain/index.js"; import {generateSignedBlockAtSlot} from "../../../utils/typeGenerator.js"; import {createCachedBeaconStateTest} from "../../../utils/cachedBeaconState.js"; import {generateState} from "../../../utils/state.js"; diff --git a/packages/state-transition/src/util/index.ts b/packages/state-transition/src/util/index.ts index bef3ae2f0511..40e4a0d7705c 100644 --- a/packages/state-transition/src/util/index.ts +++ b/packages/state-transition/src/util/index.ts @@ -6,6 +6,7 @@ export * from "./balance.js"; export * from "./blindedBlock.js"; export * from "./calculateCommitteeAssignments.js"; export * from "./capella.js"; +export * from "./computeAnchorCheckpoint.js"; export * from "./execution.js"; export * from "./blockRoot.js"; export * from "./domain.js"; From 1ab811c781a51bd965ac9acaf64ecd2a522081e3 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 28 Aug 2024 03:05:36 -0400 Subject: [PATCH 68/79] fix: touch up metrics per PR comments --- packages/beacon-node/src/chain/shufflingCache.ts | 12 +++++++----- packages/beacon-node/src/metrics/metrics/lodestar.ts | 11 ++++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 01be8d03b35b..f4c9081e787b 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -36,7 +36,7 @@ type ShufflingCacheItem = { type PromiseCacheItem = { type: CacheItemType.promise; - timeInserted: number; + timeInsertedMs: number; promise: Promise; resolveFn: (shuffling: EpochShuffling) => void; }; @@ -107,7 +107,7 @@ export class ShufflingCache implements IShufflingCache { const cacheItem: PromiseCacheItem = { type: CacheItemType.promise, - timeInserted: Date.now(), + timeInsertedMs: Date.now(), promise, resolveFn, }; @@ -163,7 +163,7 @@ export class ShufflingCache implements IShufflingCache { let shuffling: EpochShuffling | null = null; if (buildProps) { - const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer(); + const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "getSync"}); shuffling = computeEpochShuffling(buildProps.state, buildProps.activeIndices, epoch); timer?.(); this.set(shuffling, decisionRoot); @@ -181,7 +181,7 @@ export class ShufflingCache implements IShufflingCache { * on a NICE thread with a rust implementation */ callInNextEventLoop(() => { - const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer(); + const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "build"}); const shuffling = computeEpochShuffling(state, activeIndices, epoch); timer?.(); this.set(shuffling, decisionRoot); @@ -200,7 +200,9 @@ export class ShufflingCache implements IShufflingCache { if (cacheItem) { if (isPromiseCacheItem(cacheItem)) { cacheItem.resolveFn(shuffling); - this.metrics?.shufflingCache.shufflingPromiseResolutionTime.observe(Date.now() - cacheItem.timeInserted); + this.metrics?.shufflingCache.shufflingPromiseResolutionTime.observe( + (Date.now() - cacheItem.timeInsertedMs) / 1000 + ); } else { this.metrics?.shufflingCache.shufflingBuiltMultipleTimes.inc(); } diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index ad76db8dcc43..5c810e38776a 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1307,14 +1307,15 @@ export function createLodestarMetrics( help: "The next shuffling was not on the epoch cache before the epoch transition", }), shufflingPromiseResolutionTime: register.histogram({ - name: "lodestar_shuffling_cache_promise_resolution_time", - help: "Time from promise insertion until promise resolution when shuffling was ready", - buckets: [1, 10, 100, 1000], + name: "lodestar_shuffling_cache_promise_resolution_time_seconds", + help: "Time from promise insertion until promise resolution when shuffling was ready in seconds", + buckets: [0.5, 1, 1.5, 2], }), - shufflingCalculationTime: register.histogram({ - name: "lodestar_shuffling_cache_shuffling_calculation_time", + shufflingCalculationTime: register.histogram<{source: "build" | "getSync"}>({ + name: "lodestar_shuffling_cache_shuffling_calculation_time_seconds", help: "Run time of shuffling calculation", buckets: [0.5, 0.75, 1, 1.25, 1.5], + labelNames: ["source"], }), }, From fd8196e2d75e0ee6a47b2235aaab12b6ad2b129d Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 3 Sep 2024 04:57:33 -0400 Subject: [PATCH 69/79] fix: merge conflict --- .../test/perf/util/shufflings.test.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/state-transition/test/perf/util/shufflings.test.ts b/packages/state-transition/test/perf/util/shufflings.test.ts index 56034e5458b0..41767c184349 100644 --- a/packages/state-transition/test/perf/util/shufflings.test.ts +++ b/packages/state-transition/test/perf/util/shufflings.test.ts @@ -27,9 +27,9 @@ describe("epoch shufflings", () => { itBench({ id: `computeProposers - vc ${numValidators}`, fn: () => { - const epochSeed = getSeed(state, state.epochCtx.nextShuffling.epoch, DOMAIN_BEACON_PROPOSER); + const epochSeed = getSeed(state, state.epochCtx.epoch, DOMAIN_BEACON_PROPOSER); const fork = state.config.getForkSeq(state.slot); - computeProposers(fork, epochSeed, state.epochCtx.nextShuffling, state.epochCtx.effectiveBalanceIncrements); + computeProposers(fork, epochSeed, state.epochCtx.currentShuffling, state.epochCtx.effectiveBalanceIncrements); }, }); @@ -45,12 +45,7 @@ describe("epoch shufflings", () => { id: `getNextSyncCommittee - vc ${numValidators}`, fn: () => { const fork = state.config.getForkSeq(state.slot); - getNextSyncCommittee( - fork, - state, - state.epochCtx.nextActiveIndices, - state.epochCtx.effectiveBalanceIncrements - ); + getNextSyncCommittee(fork, state, state.epochCtx.nextActiveIndices, state.epochCtx.effectiveBalanceIncrements); }, }); }); From 4f777c8ea12fd0a0e041996ec6b3eb457eb0aced Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 3 Sep 2024 04:59:24 -0400 Subject: [PATCH 70/79] chore: lint --- packages/beacon-node/src/chain/shufflingCache.ts | 2 +- packages/state-transition/src/cache/epochTransitionCache.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index f4c9081e787b..ae6f9ad05271 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -5,7 +5,7 @@ import { ShufflingBuildProps, computeEpochShuffling, } from "@lodestar/state-transition"; -import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; +import {Epoch, RootHex} from "@lodestar/types"; import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; import {callInNextEventLoop} from "../util/eventLoop.js"; diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 47b948cf5d6e..86623f5754bd 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -4,7 +4,6 @@ import { EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, - MAX_EFFECTIVE_BALANCE, SLOTS_PER_HISTORICAL_ROOT, MIN_ACTIVATION_BALANCE, } from "@lodestar/params"; From 222908c18bfe9e9237718c73287b0dcee463108a Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 5 Sep 2024 22:44:30 -0400 Subject: [PATCH 71/79] refactor: add scope around activeIndices to GC arrays --- .../state-transition/src/cache/epochCache.ts | 61 ++++++++++--------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index c467adfa8bb5..0398038d18de 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -422,42 +422,45 @@ export class EpochCache { throw Error("totalActiveBalanceIncrements >= Number.MAX_SAFE_INTEGER. MAX_EFFECTIVE_BALANCE is too low."); } - const previousActiveIndices = new Uint32Array(previousActiveIndicesAsNumberArray); - const currentActiveIndices = new Uint32Array(currentActiveIndicesAsNumberArray); const nextActiveIndices = new Uint32Array(nextActiveIndicesAsNumberArray); let previousShuffling: EpochShuffling; let currentShuffling: EpochShuffling; let nextShuffling: EpochShuffling; - if (!shufflingCache) { - // Only for testing. shufflingCache should always be available in prod - previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); - currentShuffling = isGenesis - ? previousShuffling - : computeEpochShuffling(state, currentActiveIndices, currentEpoch); - nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); - } else { - currentShuffling = cachedCurrentShuffling - ? cachedCurrentShuffling - : shufflingCache.getSync(currentEpoch, currentDecisionRoot, { - state, - activeIndices: currentActiveIndices, - }); - - previousShuffling = cachedPreviousShuffling - ? cachedPreviousShuffling - : isGenesis - ? currentShuffling - : shufflingCache.getSync(previousEpoch, previousDecisionRoot, { + + { + const previousActiveIndices = new Uint32Array(previousActiveIndicesAsNumberArray); + const currentActiveIndices = new Uint32Array(currentActiveIndicesAsNumberArray); + if (!shufflingCache) { + // Only for testing. shufflingCache should always be available in prod + previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); + currentShuffling = isGenesis + ? previousShuffling + : computeEpochShuffling(state, currentActiveIndices, currentEpoch); + nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); + } else { + currentShuffling = cachedCurrentShuffling + ? cachedCurrentShuffling + : shufflingCache.getSync(currentEpoch, currentDecisionRoot, { state, - activeIndices: previousActiveIndices, + activeIndices: currentActiveIndices, }); - nextShuffling = cachedNextShuffling - ? cachedNextShuffling - : shufflingCache.getSync(nextEpoch, nextDecisionRoot, { - state, - activeIndices: nextActiveIndices, - }); + previousShuffling = cachedPreviousShuffling + ? cachedPreviousShuffling + : isGenesis + ? currentShuffling + : shufflingCache.getSync(previousEpoch, previousDecisionRoot, { + state, + activeIndices: previousActiveIndices, + }); + + nextShuffling = cachedNextShuffling + ? cachedNextShuffling + : shufflingCache.getSync(nextEpoch, nextDecisionRoot, { + state, + activeIndices: nextActiveIndices, + }); + } } const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER); From ffbbfea5ecbc9de60ff983ec761a527b6ddbab3c Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 5 Sep 2024 22:45:11 -0400 Subject: [PATCH 72/79] feat: directly use Uint32Array instead of transcribing number array to Uint32Array --- packages/state-transition/src/cache/epochTransitionCache.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 86623f5754bd..d32c7ad03ca8 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -372,7 +372,7 @@ export function beforeProcessEpoch( // as a decision block. we are part way through the transition though and this was added in // process slot beforeProcessEpoch happens so it available and valid const nextShufflingDecisionRoot = toRootHex(state.blockRoots.get(state.slot % SLOTS_PER_HISTORICAL_ROOT)); - const _nextShufflingActiveIndices = new Array(nextEpochShufflingActiveIndicesLength); + const nextShufflingActiveIndices = new Uint32Array(nextEpochShufflingActiveIndicesLength); if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { throw new Error( `Invalid activeValidatorCount: ${nextEpochShufflingActiveIndicesLength} > ${nextEpochShufflingActiveValidatorIndices.length}` @@ -380,9 +380,8 @@ export function beforeProcessEpoch( } // only the first `activeValidatorCount` elements are copied to `activeIndices` for (let i = 0; i < nextEpochShufflingActiveIndicesLength; i++) { - _nextShufflingActiveIndices[i] = nextEpochShufflingActiveValidatorIndices[i]; + nextShufflingActiveIndices[i] = nextEpochShufflingActiveValidatorIndices[i]; } - const nextShufflingActiveIndices = new Uint32Array(_nextShufflingActiveIndices); state.epochCtx.shufflingCache?.build( epochAfterUpcoming, nextShufflingDecisionRoot, From 7f6d42096fdcb3500b5d189013b0b08086621cbd Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 6 Sep 2024 00:06:45 -0400 Subject: [PATCH 73/79] refactor: activeIndices per tuyen comment --- .../state-transition/src/cache/epochCache.ts | 64 ++++++++++--------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 0398038d18de..5eb49e04718d 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -427,40 +427,42 @@ export class EpochCache { let currentShuffling: EpochShuffling; let nextShuffling: EpochShuffling; - { - const previousActiveIndices = new Uint32Array(previousActiveIndicesAsNumberArray); - const currentActiveIndices = new Uint32Array(currentActiveIndicesAsNumberArray); - if (!shufflingCache) { - // Only for testing. shufflingCache should always be available in prod - previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch); - currentShuffling = isGenesis - ? previousShuffling - : computeEpochShuffling(state, currentActiveIndices, currentEpoch); - nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); - } else { - currentShuffling = cachedCurrentShuffling - ? cachedCurrentShuffling - : shufflingCache.getSync(currentEpoch, currentDecisionRoot, { - state, - activeIndices: currentActiveIndices, - }); + if (!shufflingCache) { + // Only for testing. shufflingCache should always be available in prod + previousShuffling = computeEpochShuffling( + state, + new Uint32Array(previousActiveIndicesAsNumberArray), + previousEpoch + ); - previousShuffling = cachedPreviousShuffling - ? cachedPreviousShuffling - : isGenesis - ? currentShuffling - : shufflingCache.getSync(previousEpoch, previousDecisionRoot, { - state, - activeIndices: previousActiveIndices, - }); - - nextShuffling = cachedNextShuffling - ? cachedNextShuffling - : shufflingCache.getSync(nextEpoch, nextDecisionRoot, { + currentShuffling = isGenesis + ? previousShuffling + : computeEpochShuffling(state, new Uint32Array(currentActiveIndicesAsNumberArray), currentEpoch); + + nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); + } else { + currentShuffling = cachedCurrentShuffling + ? cachedCurrentShuffling + : shufflingCache.getSync(currentEpoch, currentDecisionRoot, { + state, + activeIndices: new Uint32Array(currentActiveIndicesAsNumberArray), + }); + + previousShuffling = cachedPreviousShuffling + ? cachedPreviousShuffling + : isGenesis + ? currentShuffling + : shufflingCache.getSync(previousEpoch, previousDecisionRoot, { state, - activeIndices: nextActiveIndices, + activeIndices: new Uint32Array(previousActiveIndicesAsNumberArray), }); - } + + nextShuffling = cachedNextShuffling + ? cachedNextShuffling + : shufflingCache.getSync(nextEpoch, nextDecisionRoot, { + state, + activeIndices: nextActiveIndices, + }); } const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER); From afc03f2218d9b988c7951aaf05dc81d08e002a4c Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Mon, 9 Sep 2024 19:31:02 -0400 Subject: [PATCH 74/79] refactor: rename to epochAfterNext --- .../state-transition/src/cache/epochTransitionCache.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index d32c7ad03ca8..27b781e8a6a1 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -367,7 +367,7 @@ export function beforeProcessEpoch( } // Trigger async build of shuffling for epoch after next (nextShuffling post epoch transition) - const epochAfterUpcoming = state.epochCtx.nextEpoch + 1; + const epochAfterNext = state.epochCtx.nextEpoch + 1; // cannot call calculateShufflingDecisionRoot here because spec prevent getting current slot // as a decision block. we are part way through the transition though and this was added in // process slot beforeProcessEpoch happens so it available and valid @@ -382,12 +382,7 @@ export function beforeProcessEpoch( for (let i = 0; i < nextEpochShufflingActiveIndicesLength; i++) { nextShufflingActiveIndices[i] = nextEpochShufflingActiveValidatorIndices[i]; } - state.epochCtx.shufflingCache?.build( - epochAfterUpcoming, - nextShufflingDecisionRoot, - state, - nextShufflingActiveIndices - ); + state.epochCtx.shufflingCache?.build(epochAfterNext, nextShufflingDecisionRoot, state, nextShufflingActiveIndices); if (totalActiveStakeByIncrement < 1) { totalActiveStakeByIncrement = 1; From d1d0e2255b327ea5a0056ca69593930ae8b86ed0 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Mon, 16 Sep 2024 14:08:30 +0700 Subject: [PATCH 75/79] chore: review PR --- .../beacon-node/src/chain/shufflingCache.ts | 1 - .../state-transition/src/stateTransition.ts | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index ae6f9ad05271..6c42228b5356 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -185,7 +185,6 @@ export class ShufflingCache implements IShufflingCache { const shuffling = computeEpochShuffling(state, activeIndices, epoch); timer?.(); this.set(shuffling, decisionRoot); - // wait until after the first slot to help with attestation and block proposal performance }); } diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index 40e87c8d07d2..52c20a9c9833 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -166,6 +166,25 @@ export function processSlots( /** * All processSlot() logic but separate so stateTransition() can recycle the caches + * + * Epoch transition will be processed at the last slot of an epoch. Note that compute_shuffling() is going + * to be executed in parallel (either by napi-rs or worker thread) with processEpoch() like below: + * + * state-transition + * ╔══════════════════════════════════════════════════════════════════════════════════╗ + * ║ beforeProcessEpoch processEpoch afterPRocessEpoch ║ + * ║ |-------------------------|--------------------|-------------------------------|║ + * ║ | | | ║ + * ╚═══════════════════════|═══════════════════════════════|══════════════════════════╝ + * | | + * build() get() + * | | + * ╔═══════════════════════V═══════════════════════════════V═══════════════════════════╗ + * ║ | | ║ + * ║ |-------------------------------| ║ + * ║ compute_shuffling() ║ + * ╚═══════════════════════════════════════════════════════════════════════════════════╝ + * beacon-node ShufflingCache */ function processSlotsWithTransientCache( postState: CachedBeaconStateAllForks, From c5c57721cd598ddfcbf0f467b4dbcc4e15702214 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 19 Sep 2024 14:56:41 +0700 Subject: [PATCH 76/79] feat: update no shuffling ApiError to 500 status --- packages/beacon-node/src/api/impl/beacon/state/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index b5b9e8c71555..77e2fd5aab46 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -206,7 +206,7 @@ export function getBeaconStateApi({ const shuffling = await chain.shufflingCache.get(epoch, decisionRoot); if (!shuffling) { throw new ApiError( - 400, + 500, `No shuffling found to calculate committees for epoch: ${epoch} and decisionRoot: ${decisionRoot}` ); } From 88d3105b665c01964c44bf3dca5a9b2b3e7d15ef Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 19 Sep 2024 15:05:03 +0700 Subject: [PATCH 77/79] fix: add back unnecessary eslint directive. to be remove under separate PR --- packages/light-client/test/unit/webEsmBundle.browser.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/light-client/test/unit/webEsmBundle.browser.test.ts b/packages/light-client/test/unit/webEsmBundle.browser.test.ts index 05afe1fba8e7..defc421d7071 100644 --- a/packages/light-client/test/unit/webEsmBundle.browser.test.ts +++ b/packages/light-client/test/unit/webEsmBundle.browser.test.ts @@ -1,4 +1,4 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-member-access*/ +/* eslint-disable @typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-call */ import {expect, describe, it, vi, beforeAll} from "vitest"; import {sleep} from "@lodestar/utils"; import {Lightclient, LightclientEvent, utils, transport} from "../../dist/lightclient.min.mjs"; From b857d6e24019a097707cbdaa0e70980fb936cba3 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 20 Sep 2024 14:15:44 +0700 Subject: [PATCH 78/79] feat: update no shuffling ApiError to 500 status --- packages/beacon-node/src/api/impl/validator/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index cc52c3101edc..d2ce6672c89f 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1000,7 +1000,7 @@ export function getValidatorApi( const shuffling = await chain.shufflingCache.get(epoch, decisionRoot); if (!shuffling) { throw new ApiError( - 400, + 500, `No shuffling found to calculate committee assignments for epoch: ${epoch} and decisionRoot: ${decisionRoot}` ); } From af5c98954b0343a6e073b2a2b9bea01d0aaec292 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Fri, 20 Sep 2024 14:21:02 +0700 Subject: [PATCH 79/79] docs: add comment about upcomingEpoch --- packages/state-transition/src/cache/epochCache.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 5eb49e04718d..63731aa33dea 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -652,6 +652,11 @@ export class EpochCache { nextEpochTotalActiveBalanceByIncrement: number; } ): void { + // Because the slot was incremented before entering this function the "next epoch" is actually the "current epoch" + // in this context but that is not actually true because the state transition happens in the last 4 seconds of the + // epoch. For the context of this function "upcoming epoch" is used to denote the epoch that will begin after this + // function returns. The epoch that is "next" once the state transition is complete is referred to as the + // epochAfterUpcoming for the same reason to help minimize confusion. const upcomingEpoch = this.nextEpoch; const epochAfterUpcoming = upcomingEpoch + 1;