Track block source

dashboards/lodestar_general.json

@@ -8639,7 +8639,7 @@
                 "uid": "${DS_PROMETHEUS}"
               },
               "exemplar": false,
-              "expr": "delta(lodestar_gossip_block_elapsed_time_till_become_head_sum[$__rate_interval])\n/\ndelta(lodestar_gossip_block_elapsed_time_till_become_head_count[$__rate_interval])",
+              "expr": "delta(lodestar_block_elapsed_time_till_become_head_sum{src=\"BLOCK_SOURCE_GOSSIP\"}[$__rate_interval])\n/\ndelta(lodestar_block_elapsed_time_till_become_head_count[$__rate_interval])",
               "hide": false,
               "interval": "",
               "legendFormat": "till become head",

packages/lodestar/src/api/impl/beacon/blocks/index.ts

@@ -5,6 +5,7 @@ import {computeTimeAtSlot} from "@chainsafe/lodestar-beacon-state-transition";
 import {SLOTS_PER_HISTORICAL_ROOT} from "@chainsafe/lodestar-params";
 import {sleep} from "@chainsafe/lodestar-utils";
 import {fromHexString, toHexString} from "@chainsafe/ssz";
+import {BlockSource} from "../../../../chain/blocks/types";
 import {BlockError, BlockErrorCode} from "../../../../chain/errors";
 import {OpSource} from "../../../../metrics/validatorMonitor";
 import {NetworkEvent} from "../../../../network";
@@ -157,7 +158,7 @@ export function getBeaconBlockApi({
         // specification is very clear that this is the desired behaviour.
         network.gossip.publishBeaconBlock(signedBlock),
 
-        chain.processBlock(signedBlock).catch((e) => {
+        chain.processBlock(signedBlock, {source: BlockSource.API}).catch((e) => {
           if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) {
             network.events.emit(NetworkEvent.unknownBlockParent, signedBlock, network.peerId.toB58String());
           }

packages/lodestar/src/chain/blocks/importBlock.ts

@@ -201,7 +201,7 @@ export async function importBlock(chain: ImportBlockModules, fullyVerifiedBlock:
   const newHead = chain.forkChoice.getHead();
   if (newHead.blockRoot !== oldHead.blockRoot) {
     // new head
-    pendingEvents.push(ChainEvent.forkChoiceHead, newHead);
+    pendingEvents.push(ChainEvent.forkChoiceHead, newHead, fullyVerifiedBlock.source);
     chain.metrics?.forkChoiceChangedHead.inc();
 
     const distance = chain.forkChoice.getCommonAncestorDistance(oldHead, newHead);

packages/lodestar/src/chain/blocks/types.ts

@@ -23,6 +23,10 @@ export type FullyVerifiedBlockFlags = {
    * If the execution payload couldnt be verified because of EL syncing status, used in optimistic sync or for merge block
    */
   executionStatus?: ExecutionStatus;
+  /**
+   * If from RangeSync module, we won't attest to this block so it's okay to ignore a SYNCING message from execution layer
+   */
+  source: BlockSource;
 };
 
 export type PartiallyVerifiedBlockFlags = FullyVerifiedBlockFlags & {
@@ -34,16 +38,22 @@ export type PartiallyVerifiedBlockFlags = FullyVerifiedBlockFlags & {
    * Metadata: `true` if all the signatures including the proposer signature have been verified
    */
   validSignatures?: boolean;
-  /**
-   * From RangeSync module, we won't attest to this block so it's okay to ignore a SYNCING message from execution layer
-   */
-  fromRangeSync?: boolean;
   /**
    * Verify signatures on main thread or not.
    */
   blsVerifyOnMainThread?: boolean;
 };
 
+/**
+ * An enum of how a block comes to this node.
+ */
+export enum BlockSource {
+  API = "BLOCK_SOURCE_API",
+  GOSSIP = "BLOCK_SOURCE_GOSSIP",
+  UNKNOWN_BLOCK_SYNC = "BLOCK_SOURCE_UNKNOWN_BLOCK_SYNC",
+  RANGE_SYNC = "BLOCK_SOURCE_RANGE_SYNC",
+}
+
 /**
  * A wrapper around a `SignedBeaconBlock` that indicates that this block is fully verified and ready to import
  */

packages/lodestar/src/chain/blocks/verifyBlock.ts

@@ -50,6 +50,7 @@ export async function verifyBlock(
     parentBlock,
     skipImportingAttestations: partiallyVerifiedBlock.skipImportingAttestations,
     executionStatus,
+    source: partiallyVerifiedBlock.source,
   };
 }
 

packages/lodestar/src/chain/chain.ts

@@ -257,11 +257,11 @@ export class BeaconChain implements IBeaconChain {
     return await this.db.block.get(fromHexString(block.blockRoot));
   }
 
-  async processBlock(block: allForks.SignedBeaconBlock, flags?: PartiallyVerifiedBlockFlags): Promise<void> {
+  async processBlock(block: allForks.SignedBeaconBlock, flags: PartiallyVerifiedBlockFlags): Promise<void> {
     return await this.blockProcessor.processBlockJob({...flags, block});
   }
 
-  async processChainSegment(blocks: allForks.SignedBeaconBlock[], flags?: PartiallyVerifiedBlockFlags): Promise<void> {
+  async processChainSegment(blocks: allForks.SignedBeaconBlock[], flags: PartiallyVerifiedBlockFlags): Promise<void> {
     return await this.blockProcessor.processChainSegment(blocks.map((block) => ({...flags, block})));
   }
 

packages/lodestar/src/chain/emitter.ts

@@ -6,6 +6,7 @@ import {phase0, Epoch, Slot, allForks} from "@chainsafe/lodestar-types";
 import {CheckpointWithHex, IProtoBlock} from "@chainsafe/lodestar-fork-choice";
 import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition";
 import {AttestationError, BlockError} from "./errors";
+import {BlockSource} from "./blocks/types";
 
 /**
  * Important chain events that occur during normal chain operation.
@@ -115,7 +116,7 @@ export interface IChainEvents {
   [ChainEvent.clockSlot]: (slot: Slot) => void;
   [ChainEvent.clockEpoch]: (epoch: Epoch) => void;
 
-  [ChainEvent.forkChoiceHead]: (head: IProtoBlock) => void;
+  [ChainEvent.forkChoiceHead]: (head: IProtoBlock, source: BlockSource) => void;
   [ChainEvent.forkChoiceReorg]: (head: IProtoBlock, oldHead: IProtoBlock, depth: number) => void;
   [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithHex) => void;
   [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithHex) => void;

packages/lodestar/src/chain/eventHandlers.ts

@@ -9,6 +9,7 @@ import {ChainEvent, IChainEvents} from "./emitter";
 import {BeaconChain} from "./chain";
 import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "./reprocess";
 import {toCheckpointHex} from "./stateCache";
+import {BlockSource} from "./blocks/types";
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type AnyCallback = () => Promise<void>;
@@ -148,7 +149,7 @@ export async function onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWit
   }
 }
 
-export function onForkChoiceHead(this: BeaconChain, head: IProtoBlock): void {
+export function onForkChoiceHead(this: BeaconChain, head: IProtoBlock, source: BlockSource): void {
   const delaySec = this.clock.secFromSlot(head.slot);
   this.logger.verbose("New chain head", {
     headSlot: head.slot,
@@ -158,7 +159,7 @@ export function onForkChoiceHead(this: BeaconChain, head: IProtoBlock): void {
   this.syncContributionAndProofPool.prune(head.slot);
   this.seenContributionAndProof.prune(head.slot);
   this.metrics?.headSlot.set(head.slot);
-  this.metrics?.gossipBlock.elapsedTimeTillBecomeHead.observe(delaySec);
+  this.metrics?.blockElapsedTimeTillBecomeHead.observe({src: source}, delaySec);
 }
 
 export function onForkChoiceReorg(this: BeaconChain, head: IProtoBlock, oldHead: IProtoBlock, depth: number): void {

packages/lodestar/src/metrics/metrics/lodestar.ts

@@ -514,11 +514,6 @@ export function createLodestarMetrics(
 
     // Gossip block
     gossipBlock: {
-      elapsedTimeTillBecomeHead: register.histogram({
-        name: "lodestar_gossip_block_elapsed_time_till_become_head",
-        help: "Time elappsed between block slot time and the time block becomes head",
-        buckets: [0.1, 1, 10],
-      }),
       elapsedTimeTillReceived: register.histogram({
         name: "lodestar_gossip_block_elappsed_time_till_received",
         help: "Time elappsed between block slot time and the time block received via gossip",
@@ -875,5 +870,12 @@ export function createLodestarMetrics(
         labelNames: ["type"],
       }),
     },
+
+    blockElapsedTimeTillBecomeHead: register.histogram<"src">({
+      name: "lodestar_block_elapsed_time_till_become_head",
+      help: "Time elappsed between block slot time and the time block becomes head",
+      buckets: [0.5, 1, 2, 4, 6, 12],
+      labelNames: ["src"],
+    }),
   };
 }

packages/lodestar/src/network/gossip/handlers/index.ts

@@ -29,6 +29,7 @@ import {
 import {INetwork} from "../../interface";
 import {NetworkEvent} from "../../events";
 import {PeerAction} from "../../peers";
+import {BlockSource} from "../../../chain/blocks/types";
 
 /**
  * Gossip handler options as part of network options
@@ -119,7 +120,11 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
       // otherwise we can't utilize bls thread pool capacity and Gossip Job Wait Time can't be kept low consistently.
       // See https://github.com/ChainSafe/lodestar/issues/3792
       chain
-        .processBlock(signedBlock, {validProposerSignature: true, blsVerifyOnMainThread: true})
+        .processBlock(signedBlock, {
+          validProposerSignature: true,
+          blsVerifyOnMainThread: true,
+          source: BlockSource.GOSSIP,
+        })
         .then(() => {
           // Returns the delay between the start of `block.slot` and `current time`
           const delaySec = chain.clock.secFromSlot(slot);

packages/lodestar/src/sync/range/range.ts

@@ -12,6 +12,7 @@ import {RangeSyncType, getRangeSyncType, rangeSyncTypes} from "../utils/remoteSy
 import {updateChains} from "./utils";
 import {ChainTarget, SyncChainFns, SyncChain, SyncChainOpts, SyncChainDebugState} from "./chain";
 import {PartiallyVerifiedBlockFlags} from "../../chain/blocks";
+import {BlockSource} from "../../chain/blocks/types";
 
 export enum RangeSyncEvent {
   completedChain = "RangeSync-completedChain",
@@ -200,7 +201,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       // Ignore WOULD_REVERT_FINALIZED_SLOT error, continue with the next block in chain segment
       ignoreIfFinalized: true,
       // We won't attest to this block so it's okay to ignore a SYNCING message from execution layer
-      fromRangeSync: true,
+      source: BlockSource.RANGE_SYNC,
       // when this runs, syncing is the most important thing and gossip is not likely to run
       // so we can utilize worker threads to verify signatures
       blsVerifyOnMainThread: false,

packages/lodestar/src/sync/unknownBlock.ts

@@ -14,6 +14,7 @@ import {pruneSetToMax} from "../util/map";
 import {PendingBlock, PendingBlockStatus} from "./interface";
 import {getDescendantBlocks, getAllDescendantBlocks, getLowestPendingUnknownParents} from "./utils/pendingBlocksTree";
 import {SyncOptions} from "./options";
+import {BlockSource} from "../chain/blocks/types";
 
 const MAX_ATTEMPTS_PER_BLOCK = 5;
 const MAX_KNOWN_BAD_BLOCKS = 500;
@@ -171,7 +172,11 @@ export class UnknownBlockSync {
     // otherwise we can't utilize bls thread pool capacity and Gossip Job Wait Time can't be kept low consistently.
     // See https://github.com/ChainSafe/lodestar/issues/3792
     const res = await wrapError(
-      this.chain.processBlock(pendingBlock.signedBlock, {ignoreIfKnown: true, blsVerifyOnMainThread: true})
+      this.chain.processBlock(pendingBlock.signedBlock, {
+        ignoreIfKnown: true,
+        blsVerifyOnMainThread: true,
+        source: BlockSource.UNKNOWN_BLOCK_SYNC,
+      })
     );
     pendingBlock.status = PendingBlockStatus.pending;
 

packages/lodestar/test/unit/api/impl/beacon/blocks/publishBlock.test.ts

@@ -8,6 +8,7 @@ import {generateEmptySignedBlock} from "../../../../../utils/block";
 import {allForks} from "@chainsafe/lodestar-types";
 import {BeaconSync} from "../../../../../../src/sync";
 import {setupApiImplTestServer, ApiImplTestModules} from "../../index.test";
+import {BlockSource} from "../../../../../../src/chain/blocks/types";
 
 use(chaiAsPromised);
 
@@ -43,7 +44,7 @@ describe("api - beacon - publishBlock", function () {
 
     syncStub.isSynced.returns(true);
     await expect(blockApi.publishBlock(block)).to.be.fulfilled;
-    expect(chainStub.processBlock.calledOnceWith(block)).to.be.true;
+    expect(chainStub.processBlock.calledOnceWith(block, {source: BlockSource.API})).to.be.true;
     expect(gossipStub.publishBeaconBlock.calledOnceWith(block)).to.be.true;
   });
 });

packages/lodestar/test/unit/api/impl/events/events.test.ts

@@ -9,6 +9,7 @@ import {generateProtoBlock, generateEmptySignedBlock, generateSignedBlock} from
 import {generateAttestation, generateEmptySignedVoluntaryExit} from "../../../../utils/attestation";
 import {generateCachedState} from "../../../../utils/state";
 import {StateContextCache} from "../../../../../src/chain/stateCache";
+import {BlockSource} from "../../../../../src/chain/blocks/types";
 
 describe("Events api impl", function () {
   describe("beacon event stream", function () {
@@ -44,7 +45,7 @@ describe("Events api impl", function () {
       const headBlock = generateProtoBlock();
       stateCacheStub.get.withArgs(headBlock.stateRoot).returns(generateCachedState({slot: 1000}));
       chainEventEmmitter.emit(ChainEvent.forkChoiceReorg, headBlock, headBlock, 2);
-      chainEventEmmitter.emit(ChainEvent.forkChoiceHead, headBlock);
+      chainEventEmmitter.emit(ChainEvent.forkChoiceHead, headBlock, BlockSource.GOSSIP);
 
       expect(events).to.have.length(1, "Wrong num of received events");
       expect(events[0].type).to.equal(routes.events.EventType.head);
@@ -56,7 +57,7 @@ describe("Events api impl", function () {
 
       const headBlock = generateProtoBlock();
       stateCacheStub.get.withArgs(headBlock.stateRoot).returns(generateCachedState({slot: 1000}));
-      chainEventEmmitter.emit(ChainEvent.forkChoiceHead, headBlock);
+      chainEventEmmitter.emit(ChainEvent.forkChoiceHead, headBlock, BlockSource.GOSSIP);
 
       expect(events).to.have.length(1, "Wrong num of received events");
       expect(events[0].type).to.equal(routes.events.EventType.head);

packages/lodestar/test/unit/chain/blocks/verifyBlock.test.ts

@@ -8,6 +8,7 @@ import {LocalClock} from "../../../../src/chain/clock";
 import {BlockErrorCode} from "../../../../src/chain/errors";
 import {allForks, ssz} from "@chainsafe/lodestar-types";
 import {expectThrowsLodestarError} from "../../../utils/errors";
+import {BlockSource} from "../../../../src/chain/blocks/types";
 
 describe("chain / blocks / verifyBlock", function () {
   let forkChoice: SinonStubbedInstance<ForkChoice>;
@@ -30,29 +31,41 @@ describe("chain / blocks / verifyBlock", function () {
 
   it("PARENT_UNKNOWN", function () {
     forkChoice.getBlockHex.returns(null);
-    expectThrowsLodestarError(() => verifyBlockSanityChecks(modules, {block}), BlockErrorCode.PARENT_UNKNOWN);
+    expectThrowsLodestarError(
+      () => verifyBlockSanityChecks(modules, {block, source: BlockSource.GOSSIP}),
+      BlockErrorCode.PARENT_UNKNOWN
+    );
   });
 
   it("GENESIS_BLOCK", function () {
     block.message.slot = 0;
-    expectThrowsLodestarError(() => verifyBlockSanityChecks(modules, {block}), BlockErrorCode.GENESIS_BLOCK);
+    expectThrowsLodestarError(
+      () => verifyBlockSanityChecks(modules, {block, source: BlockSource.GOSSIP}),
+      BlockErrorCode.GENESIS_BLOCK
+    );
   });
 
   it("ALREADY_KNOWN", function () {
     forkChoice.hasBlockHex.returns(true);
-    expectThrowsLodestarError(() => verifyBlockSanityChecks(modules, {block}), BlockErrorCode.ALREADY_KNOWN);
+    expectThrowsLodestarError(
+      () => verifyBlockSanityChecks(modules, {block, source: BlockSource.GOSSIP}),
+      BlockErrorCode.ALREADY_KNOWN
+    );
   });
 
   it("WOULD_REVERT_FINALIZED_SLOT", function () {
     forkChoice.getFinalizedCheckpoint.returns({epoch: 5, root: Buffer.alloc(32), rootHex: ""});
     expectThrowsLodestarError(
-      () => verifyBlockSanityChecks(modules, {block}),
+      () => verifyBlockSanityChecks(modules, {block, source: BlockSource.GOSSIP}),
       BlockErrorCode.WOULD_REVERT_FINALIZED_SLOT
     );
   });
 
   it("FUTURE_SLOT", function () {
     block.message.slot = currentSlot + 1;
-    expectThrowsLodestarError(() => verifyBlockSanityChecks(modules, {block}), BlockErrorCode.FUTURE_SLOT);
+    expectThrowsLodestarError(
+      () => verifyBlockSanityChecks(modules, {block, source: BlockSource.GOSSIP}),
+      BlockErrorCode.FUTURE_SLOT
+    );
   });
 });