feat: support payload reorgs in chain reorg events

packages/api/src/beacon/routes/events.ts

@@ -223,7 +223,9 @@ export type EventData = {
   [EventType.chainReorg]: {
     slot: Slot;
     depth: UintNum64;
+    oldHeadHash: RootHex;
     oldHeadBlock: RootHex;
+    newHeadHash: RootHex;
     newHeadBlock: RootHex;
     oldHeadState: RootHex;
     newHeadState: RootHex;
@@ -410,7 +412,9 @@ export function getTypeByEvent(config: ChainForkConfig): {[K in EventType]: Type
       {
         slot: ssz.Slot,
         depth: ssz.UintNum64,
+        oldHeadHash: stringType,
         oldHeadBlock: stringType,
+        newHeadHash: stringType,
         newHeadBlock: stringType,
         oldHeadState: stringType,
         newHeadState: stringType,

packages/api/test/unit/beacon/oapiSpec.test.ts

@@ -80,6 +80,8 @@ runTestCheckAgainstSpec(openApiJson, definitions, testDatas, ignoredOperations,
 
 const ignoredTopics: string[] = [
   "execution_payload_bid",
+  // TODO: unskip once the spec release includes the execution hashes from beacon-APIs#585
+  "chain_reorg",
   // TODO: unskip once the spec release adds `current_slot` to the fast_confirmation event
   // (tracked in https://github.com/ethereum/beacon-APIs/pull/598)
   "fast_confirmation",

packages/api/test/unit/beacon/testData/events.ts

@@ -166,7 +166,9 @@ export const eventTestData: EventData = {
   [EventType.chainReorg]: {
     slot: 200,
     depth: 50,
+    oldHeadHash: "0x03b2a1c9d8e7f6054a3b2c1d0e9f8a7b6c5d4e3f2a1b0c9d8e7f60543b2a1c9d",
     oldHeadBlock: "0x9a2fefd2fdb57f74993c7780ea5b9030d2897b615b89f808011ca5aebed54eaf",
+    newHeadHash: "0xe4d3c2b1a0f9e8d7c6b5a49382716054f3e2d1c0b9a8f7e6d5c4b3a291807162",
     newHeadBlock: "0x76262e91970d375a19bfe8a867288d7b9cde43c8635f598d93d39d041706fc76",
     oldHeadState: "0x9a2fefd2fdb57f74993c7780ea5b9030d2897b615b89f808011ca5aebed54eaf",
     newHeadState: "0x600e852a08c1200654ddf11025f1ceacb3c2e74bdd5c630cde0838b2591b69f9",

packages/beacon-node/src/chain/blocks/importBlock.ts

@@ -1,7 +1,6 @@
 import {BitArray} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {
-  AncestorStatus,
   ExecutionStatus,
   ForkChoiceError,
   ForkChoiceErrorCode,
@@ -42,7 +41,7 @@ import {callInNextEventLoop} from "../../util/eventLoop.js";
 import {isOptimisticBlock} from "../../util/forkChoice.js";
 import {isQueueErrorAborted} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
-import {ChainEvent, ReorgEventData} from "../emitter.js";
+import {ChainEvent} from "../emitter.js";
 import {ForkchoiceCaller} from "../forkChoice/index.js";
 import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
 import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
@@ -341,28 +340,6 @@ export async function importBlock(
 
     this.metrics?.forkChoice.changedHead.inc();
 
-    const ancestorResult = this.forkChoice.getCommonAncestorDepth(oldHead, newHead);
-    if (ancestorResult.code === AncestorStatus.CommonAncestor) {
-      // CommonAncestor = chain reorg, old head and new head not direct descendants
-
-      const forkChoiceReorgEventData: ReorgEventData = {
-        depth: ancestorResult.depth,
-        epoch: computeEpochAtSlot(newHead.slot),
-        slot: newHead.slot,
-        newHeadBlock: newHead.blockRoot,
-        oldHeadBlock: oldHead.blockRoot,
-        newHeadState: newHead.stateRoot,
-        oldHeadState: oldHead.stateRoot,
-        executionOptimistic: isOptimisticBlock(newHead),
-      };
-
-      this.emitter.emit(routes.events.EventType.chainReorg, forkChoiceReorgEventData);
-      this.logger.verbose("Chain reorg", forkChoiceReorgEventData);
-
-      this.metrics?.forkChoice.reorg.inc();
-      this.metrics?.forkChoice.reorgDistance.observe(ancestorResult.depth);
-    }
-
     // Lightclient server support (only after altair)
     // - Persist state witness
     // - Use block's syncAggregate

packages/beacon-node/src/chain/chain.ts

@@ -5,6 +5,7 @@ import {Type} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {BeaconConfig} from "@lodestar/config";
 import {
+  AncestorStatus,
   CheckpointWithHex,
   EpochDifference,
   ForkChoiceStateGetter,
@@ -58,7 +59,7 @@ import {
 } from "@lodestar/types";
 import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
-import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
+import {GENESIS_EPOCH, ZERO_HASH, ZERO_HASH_HEX} from "../constants/index.js";
 import {IBeaconDb} from "../db/index.js";
 import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.js";
 import {decodeArchivedEnvelope} from "../db/repositories/index.js";
@@ -90,7 +91,7 @@ import {persistPayloadEnvelopeInput} from "./blocks/writePayloadEnvelopeInputToD
 import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js";
 import {BuilderCircuitBreaker} from "./builderCircuitBreaker.js";
 import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
-import {ChainEvent, ChainEventEmitter} from "./emitter.js";
+import {ChainEvent, ChainEventEmitter, ReorgEventData} from "./emitter.js";
 import {ForkchoiceCaller, initializeForkChoice} from "./forkChoice/index.js";
 import {GetBlobsTracker} from "./GetBlobsTracker.js";
 import {CommonBlockBody, FindHeadFnName, IBeaconChain, ProposerPreparationData, StateGetOpts} from "./interface.js";
@@ -1419,7 +1420,7 @@ export class BeaconChain implements IBeaconChain {
   }
 
   /**
-   * Emit `head` and `head_v2` when the fork choice head differs between two observations. Every caller that
+   * Emit head and reorg events when the fork choice head differs between two observations. Every caller that
    * can move the head, `updateAndGetHead()` and `updateTime()`, must capture the head before and pass both.
    */
   private emitHeadEvents(prevHead: ProtoBlock, head: ProtoBlock): void {
@@ -1468,6 +1469,49 @@ export class BeaconChain implements IBeaconChain {
         e as Error
       );
     }
+
+    try {
+      const ancestorResult = this.forkChoice.getCommonAncestorDepth(prevHead, head);
+      // EMPTY -> FULL extends the execution chain. Dropping a previously selected FULL payload reorgs it.
+      const payloadReorg =
+        ancestorResult.code === AncestorStatus.Descendant &&
+        prevHead.executionPayloadBlockHash !== null &&
+        prevHead.executionPayloadBlockHash !== head.executionPayloadBlockHash &&
+        (head.slot < prevHead.slot ||
+          (prevHead.payloadStatus === PayloadStatus.FULL &&
+            !this.forkChoice.isDescendant(
+              prevHead.blockRoot,
+              prevHead.payloadStatus,
+              head.blockRoot,
+              head.payloadStatus
+            )));
+
+      if (ancestorResult.code === AncestorStatus.CommonAncestor || payloadReorg) {
+        const depth =
+          ancestorResult.code === AncestorStatus.CommonAncestor
+            ? ancestorResult.depth
+            : Math.max(prevHead.slot - head.slot, 0);
+        const forkChoiceReorgEventData: ReorgEventData = {
+          slot: head.slot,
+          depth,
+          oldHeadHash: prevHead.executionPayloadBlockHash ?? ZERO_HASH_HEX,
+          oldHeadBlock: prevHead.blockRoot,
+          newHeadHash: head.executionPayloadBlockHash ?? ZERO_HASH_HEX,
+          newHeadBlock: head.blockRoot,
+          oldHeadState: prevHead.stateRoot,
+          newHeadState: head.stateRoot,
+          epoch: computeEpochAtSlot(head.slot),
+          executionOptimistic: isOptimisticBlock(head),
+        };
+
+        this.emitter.emit(routes.events.EventType.chainReorg, forkChoiceReorgEventData);
+        this.logger.verbose("Chain reorg", forkChoiceReorgEventData);
+        this.metrics?.forkChoice.reorg.inc();
+        this.metrics?.forkChoice.reorgDistance.observe(depth);
+      }
+    } catch (e) {
+      this.logger.debug("Error emitting chain_reorg event", {slot: head.slot, root: head.blockRoot}, e as Error);
+    }
   }
 
   predictProposerHead(): ProtoBlock {

packages/beacon-node/test/unit/chain/chainReorgEvent.test.ts

@@ -0,0 +1,160 @@
+import {assert, describe, expect, it, vi} from "vitest";
+import {routes} from "@lodestar/api";
+import {createChainForkConfig} from "@lodestar/config";
+import {
+  ExecutionStatus,
+  ForkChoice,
+  ForkChoiceStore,
+  PayloadStatus,
+  ProtoArray,
+  ProtoBlock,
+} from "@lodestar/fork-choice";
+import {DataAvailabilityStatus, getEffectiveBalanceIncrementsZeroed} from "@lodestar/state-transition";
+import {fromHex, toRootHex} from "@lodestar/utils";
+import {BeaconChain} from "../../../src/chain/chain.js";
+import {ChainEventEmitter, ReorgEventData} from "../../../src/chain/emitter.js";
+import {ForkchoiceCaller} from "../../../src/chain/forkChoice/index.js";
+import {getMockedLogger} from "../../mocks/loggerMock.js";
+import {generateProtoBlock} from "../../utils/typeGenerator.js";
+
+vi.unmock("@lodestar/fork-choice");
+
+function setup() {
+  const config = createChainForkConfig({GLOAS_FORK_EPOCH: 0});
+  const root = (value: number): string => toRootHex(Buffer.alloc(32, value));
+  const anchor = generateProtoBlock({
+    blockRoot: root(1),
+    executionPayloadBlockHash: root(2),
+    executionPayloadNumber: 0,
+    executionPayloadGasLimit: 30_000_000,
+    executionStatus: ExecutionStatus.Valid,
+  });
+  const protoArray = ProtoArray.initialize(anchor, 0);
+  const checkpoint = {epoch: 0, root: fromHex(anchor.blockRoot)};
+  const balances = getEffectiveBalanceIncrementsZeroed(0);
+  const store = new ForkChoiceStore(
+    0,
+    checkpoint,
+    checkpoint,
+    balances,
+    () => balances,
+    () => null
+  );
+  const forkChoice = new ForkChoice(config, store, protoArray, 0, null);
+
+  function addBlock(slot: number, parent: ProtoBlock) {
+    const parentHash = parent.executionPayloadBlockHash;
+    assert(parentHash !== null);
+    const block = generateProtoBlock({
+      slot,
+      blockRoot: root(10 + slot),
+      stateRoot: root(20 + slot),
+      parentRoot: parent.blockRoot,
+      parentBlockHash: parentHash,
+      executionPayloadBlockHash: parentHash,
+      executionPayloadNumber: slot,
+      executionPayloadGasLimit: 30_000_000,
+      executionStatus: ExecutionStatus.Valid,
+      payloadStatus: PayloadStatus.PENDING,
+    });
+    protoArray.onBlock(block, slot, null);
+    protoArray.onExecutionPayload(
+      block.blockRoot,
+      slot,
+      root(30 + slot),
+      slot,
+      30_000_000,
+      null,
+      ExecutionStatus.Valid,
+      DataAvailabilityStatus.Available
+    );
+    const empty = protoArray.getNode(block.blockRoot, PayloadStatus.EMPTY);
+    const full = protoArray.getNode(block.blockRoot, PayloadStatus.FULL);
+    assert(empty && full);
+    return {empty, full};
+  }
+
+  const parent = addBlock(1, anchor);
+  const childOnFull = addBlock(2, parent.full);
+  const childOnEmpty = addBlock(3, parent.empty);
+  const grandchildOnEmpty = addBlock(4, childOnEmpty.full);
+  const blocks = {parent, childOnFull, childOnEmpty, grandchildOnEmpty};
+  const emitter = new ChainEventEmitter();
+  const events: ReorgEventData[] = [];
+  emitter.on(routes.events.EventType.chainReorg, (data) => events.push(data));
+  const chain = Object.assign(Object.create(BeaconChain.prototype) as BeaconChain, {
+    config,
+    forkChoice,
+    emitter,
+    logger: getMockedLogger(),
+    metrics: null,
+  });
+  vi.spyOn(forkChoice, "getDependentRoot").mockReturnValue(anchor.blockRoot);
+
+  function updateHead(previous: ProtoBlock, head: ProtoBlock) {
+    vi.spyOn(forkChoice, "getHead").mockReturnValue(previous);
+    vi.spyOn(forkChoice, "updateAndGetHead").mockReturnValue({head});
+    chain.recomputeForkChoiceHead(ForkchoiceCaller.importBlock);
+  }
+
+  return {blocks, forkChoice, events, updateHead};
+}
+
+describe("BeaconChain chain_reorg event", () => {
+  it.each<
+    [
+      name: string,
+      oldBlock: keyof ReturnType<typeof setup>["blocks"],
+      oldStatus: "empty" | "full",
+      newBlock: keyof ReturnType<typeof setup>["blocks"],
+      newStatus: "empty" | "full",
+      depth: number | null,
+    ]
+  >([
+    ["same-root payload removal", "parent", "full", "parent", "empty", 0],
+    ["child excludes old payload", "parent", "full", "childOnEmpty", "empty", 0],
+    ["child replaces old payload", "parent", "full", "childOnEmpty", "full", 0],
+    ["later descendant excludes old payload", "parent", "full", "grandchildOnEmpty", "full", 0],
+    ["payload arrival", "parent", "empty", "parent", "full", null],
+    ["child extends empty head", "parent", "empty", "childOnEmpty", "full", null],
+    ["child includes previously unselected payload", "parent", "empty", "childOnFull", "full", null],
+    ["normal payload extension", "parent", "full", "childOnFull", "full", null],
+    ["beacon extension without new payload", "parent", "full", "childOnFull", "empty", null],
+    ["unchanged head", "parent", "full", "parent", "full", null],
+    ["beacon reorg", "childOnFull", "full", "childOnEmpty", "full", 2],
+    ["rollback from empty head drops earlier payload", "childOnFull", "empty", "parent", "empty", 1],
+  ])("%s", (_name, oldBlock, oldStatus, newBlock, newStatus, depth) => {
+    const {blocks, events, updateHead} = setup();
+    const previous = blocks[oldBlock][oldStatus];
+    const head = blocks[newBlock][newStatus];
+    updateHead(previous, head);
+
+    expect(events).toEqual(
+      depth === null
+        ? []
+        : [
+            {
+              slot: head.slot,
+              depth,
+              oldHeadHash: previous.executionPayloadBlockHash,
+              oldHeadBlock: previous.blockRoot,
+              newHeadHash: head.executionPayloadBlockHash,
+              newHeadBlock: head.blockRoot,
+              oldHeadState: previous.stateRoot,
+              newHeadState: head.stateRoot,
+              epoch: 0,
+              executionOptimistic: false,
+            },
+          ]
+    );
+  });
+
+  it("emits even if duty dependent roots are unavailable", () => {
+    const {blocks, forkChoice, events, updateHead} = setup();
+    vi.mocked(forkChoice.getDependentRoot).mockImplementation(() => {
+      throw Error("No block for root");
+    });
+    updateHead(blocks.parent.full, blocks.parent.empty);
+    expect(events).toHaveLength(1);
+  });
+});