Builder circuit breaker

CHANGELOG.md

@@ -20,5 +20,6 @@ For information on changes in released versions of Teku, see the [releases page]
 ### Additions and Improvements
 - Added Gas Limit APIs (GET/POST/DELETE)
 - Skip finding the PoW block that first satisfies the minimum genesis time condition when the genesis state is already known. Fixes an incompatibility with Nethermind's backwards sync for historic blocks.
+- Circuit breaker logic added when interacting with Builder endpoint
 
 ### Bug Fixes

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactory.java

@@ -167,16 +167,9 @@ public Consumer<BeaconBlockBodyBuilder> createSelector(
                             spec.atSlot(blockSlotState.getSlot()).getSchemaDefinitions())
                         .getExecutionPayloadHeaderSchema()
                         .getHeaderOfDefaultPayload(),
-                (executionPayloadContext) -> {
-                  final boolean transitionNotFinalized =
-                      executionPayloadContext
-                          .getForkChoiceState()
-                          .getFinalizedExecutionBlockHash()
-                          .isZero();
-
-                  return executionLayerChannel.builderGetHeader(
-                      executionPayloadContext, blockSlotState, transitionNotFinalized);
-                }));
+                (executionPayloadContext) ->
+                    executionLayerChannel.builderGetHeader(
+                        executionPayloadContext, blockSlotState)));
         return;
       }
 

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockFactoryTest.java

@@ -17,7 +17,6 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
@@ -305,7 +304,7 @@ private BeaconBlock assertBlockCreated(
                 Optional.of(dataStructureUtil.randomPayloadExecutionContext(false))));
     when(executionLayer.engineGetPayload(any(), any()))
         .thenReturn(SafeFuture.completedFuture(executionPayload));
-    when(executionLayer.builderGetHeader(any(), any(), anyBoolean()))
+    when(executionLayer.builderGetHeader(any(), any()))
         .thenReturn(SafeFuture.completedFuture(executionPayloadHeader));
 
     final BLSSignature randaoReveal = dataStructureUtil.randomSignature();

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTest.java

@@ -377,7 +377,7 @@ void shouldIncludeExecutionPayloadHeaderIfBlindedBlockRequested() {
 
     when(forkChoiceNotifier.getPayloadId(any(), any()))
         .thenReturn(SafeFuture.completedFuture(Optional.of(executionPayloadContext)));
-    when(executionLayer.builderGetHeader(executionPayloadContext, blockSlotState, false))
+    when(executionLayer.builderGetHeader(executionPayloadContext, blockSlotState))
         .thenReturn(SafeFuture.completedFuture(randomExecutionPayloadHeader));
 
     factory
@@ -410,29 +410,6 @@ void shouldIncludeExecutionPayloadIfNoBlindedBlockRequested() {
     assertThat(bodyBuilder.executionPayload).isEqualTo(randomExecutionPayload);
   }
 
-  @Test
-  void shouldIncludeExecutionPayloadIfBlindedBlockRequestedButPreMerge() {
-    final UInt64 slot = UInt64.ONE;
-    final BeaconState blockSlotState = dataStructureUtil.randomBeaconStatePreMerge(slot);
-
-    final ExecutionPayloadContext executionPayloadContext =
-        dataStructureUtil.randomPayloadExecutionContext(Bytes32.ZERO, false);
-    final ExecutionPayloadHeader randomExecutionPayloadHeader =
-        dataStructureUtil.randomExecutionPayloadHeader();
-
-    when(forkChoiceNotifier.getPayloadId(any(), any()))
-        .thenReturn(SafeFuture.completedFuture(Optional.of(executionPayloadContext)));
-    when(executionLayer.builderGetHeader(executionPayloadContext, blockSlotState, true))
-        .thenReturn(SafeFuture.completedFuture(randomExecutionPayloadHeader));
-
-    factory
-        .createSelector(
-            parentRoot, blockSlotState, dataStructureUtil.randomSignature(), Optional.empty())
-        .accept(blindedBodyBuilder);
-
-    assertThat(blindedBodyBuilder.executionPayloadHeader).isEqualTo(randomExecutionPayloadHeader);
-  }
-
   @Test
   void shouldUnblindSignedBlindedBeaconBlock() {
     final ExecutionPayload randomExecutionPayload = dataStructureUtil.randomExecutionPayload();

ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/BuilderCircuitBreaker.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright ConsenSys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.ethereum.executionlayer;
+
+import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
+
+public interface BuilderCircuitBreaker {
+  BuilderCircuitBreaker NOOP = (state) -> false;
+
+  boolean isEngaged(final BeaconState state);
+}

ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/BuilderCircuitBreakerImpl.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright ConsenSys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.ethereum.executionlayer;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashSet;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.tuweni.bytes.Bytes32;
+import tech.pegasys.teku.infrastructure.ssz.collections.SszBytes32Vector;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.spec.Spec;
+import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
+
+public class BuilderCircuitBreakerImpl implements BuilderCircuitBreaker {
+  private static final Logger LOG = LogManager.getLogger();
+  private final Spec spec;
+  private final int faultInspectionWindow;
+  private final int minimumUniqueBlockRootsInWindow;
+
+  public BuilderCircuitBreakerImpl(
+      final Spec spec, final int faultInspectionWindow, final int allowedFaults) {
+    checkArgument(
+        faultInspectionWindow > allowedFaults,
+        "FaultInspectionWindow must be greater than AllowedFaults");
+    this.spec = spec;
+    this.faultInspectionWindow = faultInspectionWindow;
+    this.minimumUniqueBlockRootsInWindow = faultInspectionWindow - allowedFaults;
+  }
+
+  @Override
+  public boolean isEngaged(final BeaconState state) {
+
+    final int uniqueBlockRootsCount = getLatestUniqueBlockRootsCount(state);
+    if (uniqueBlockRootsCount < minimumUniqueBlockRootsInWindow) {
+      LOG.debug(
+          "Builder circuit breaker engaged: slot: {}, uniqueBlockRootsCount: {}, window: {},  minimumUniqueBlockRootsInWindow: {}",
+          state.getSlot(),
+          uniqueBlockRootsCount,
+          faultInspectionWindow,
+          minimumUniqueBlockRootsInWindow);
+      return true;
+    }
+
+    LOG.debug("Builder circuit breaker has not engaged.");
+
+    return false;
+  }
+
+  @VisibleForTesting
+  int getLatestUniqueBlockRootsCount(final BeaconState state) throws IllegalArgumentException {
+    final int slotsPerHistoricalRoot =
+        spec.atSlot(state.getSlot()).getConfig().getSlotsPerHistoricalRoot();
+    checkArgument(
+        faultInspectionWindow <= slotsPerHistoricalRoot,
+        "faultInspectionWindow (%s) cannot exceed slotsPerHistoricalRoot config (%s)",
+        faultInspectionWindow,
+        slotsPerHistoricalRoot);
+
+    final HashSet<Bytes32> uniqueBlockRoots = new HashSet<>();
+    final SszBytes32Vector blockRoots = state.getBlockRoots();
+
+    // state slot is the slot we are building for
+    // thus our fault window will be (inclusive)
+    // FROM (state_slot-1)-(faultInspectionWindow-1) TO state_slot-1
+
+    // of which:
+    // state_slot-1 -> will be represented by getLatestBlockHeader
+    // FROM (state_slot-1)-(faultInspectionWindow-1) TO state_slot-2 -> to be found in blockRoots
+
+    // (state_slot-1)-(faultInspectionWindow-1) = state_slot-faultInspectionWindow
+    final UInt64 firstSlotOfInspectionWindow = state.getSlot().minusMinZero(faultInspectionWindow);
+    final UInt64 lastSlotOfInspectionWindow = state.getSlot().minusMinZero(1);
+
+    // if getLatestBlockHeader is outside the fault window,
+    // we have definitely missed all blocks so count will be 0
+    if (state.getLatestBlockHeader().getSlot().isLessThan(firstSlotOfInspectionWindow)) {
+      return 0;
+    }
+
+    UInt64 currentSlot = firstSlotOfInspectionWindow;
+    while (currentSlot.isLessThan(lastSlotOfInspectionWindow)) {
+      final int currentBlockRootIndex = currentSlot.mod(slotsPerHistoricalRoot).intValue();
+      uniqueBlockRoots.add(blockRoots.getElement(currentBlockRootIndex));
+      currentSlot = currentSlot.increment();
+    }
+
+    int uniqueBlockRootsCount = uniqueBlockRoots.size();
+
+    // let's count the latest block header only if it is from the last slot
+    if (state.getLatestBlockHeader().getSlot().equals(lastSlotOfInspectionWindow)) {
+      uniqueBlockRootsCount++;
+    }
+
+    return uniqueBlockRootsCount;
+  }
+}

ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java

@@ -50,6 +50,7 @@
 import tech.pegasys.teku.ethereum.executionclient.web3j.Web3JClient;
 import tech.pegasys.teku.ethereum.executionclient.web3j.Web3JExecutionEngineClient;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
+import tech.pegasys.teku.infrastructure.exceptions.ExceptionUtil;
 import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException;
 import tech.pegasys.teku.infrastructure.logging.EventLogger;
 import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory;
@@ -96,6 +97,7 @@ public class ExecutionLayerManagerImpl implements ExecutionLayerManager {
   private final Spec spec;
   private final EventLogger eventLogger;
   private final BuilderBidValidator builderBidValidator;
+  private final BuilderCircuitBreaker builderCircuitBreaker;
   private final LabelledMetric<Counter> executionPayloadSourceCounter;
 
   public static ExecutionLayerManagerImpl create(
@@ -104,7 +106,8 @@ public static ExecutionLayerManagerImpl create(
       final Optional<BuilderClient> builderClient,
       final Spec spec,
       final MetricsSystem metricsSystem,
-      final BuilderBidValidator builderBidValidator) {
+      final BuilderBidValidator builderBidValidator,
+      final BuilderCircuitBreaker builderCircuitBreaker) {
 
     final LabelledMetric<Counter> executionPayloadSourceCounter =
         metricsSystem.createLabelledCounter(
@@ -131,6 +134,7 @@ public static ExecutionLayerManagerImpl create(
         spec,
         eventLogger,
         builderBidValidator,
+        builderCircuitBreaker,
         executionPayloadSourceCounter);
   }
 
@@ -170,13 +174,15 @@ private ExecutionLayerManagerImpl(
       final Spec spec,
       final EventLogger eventLogger,
       final BuilderBidValidator builderBidValidator,
+      final BuilderCircuitBreaker builderCircuitBreaker,
       final LabelledMetric<Counter> executionPayloadSourceCounter) {
     this.executionEngineClient = executionEngineClient;
     this.builderClient = builderClient;
     this.latestBuilderAvailability = new AtomicBoolean(builderClient.isPresent());
     this.spec = spec;
     this.eventLogger = eventLogger;
     this.builderBidValidator = builderBidValidator;
+    this.builderCircuitBreaker = builderCircuitBreaker;
     this.executionPayloadSourceCounter = executionPayloadSourceCounter;
   }
 
@@ -335,9 +341,7 @@ public SafeFuture<Void> builderRegisterValidators(
 
   @Override
   public SafeFuture<ExecutionPayloadHeader> builderGetHeader(
-      final ExecutionPayloadContext executionPayloadContext,
-      final BeaconState state,
-      final boolean transitionNotFinalized) {
+      final ExecutionPayloadContext executionPayloadContext, final BeaconState state) {
     final UInt64 slot = state.getSlot();
 
     final SafeFuture<ExecutionPayload> localExecutionPayload =
@@ -350,8 +354,10 @@ public SafeFuture<ExecutionPayloadHeader> builderGetHeader(
     final FallbackReason fallbackReason;
     if (builderClient.isEmpty() && validatorRegistration.isEmpty()) {
       fallbackReason = FallbackReason.NOT_NEEDED;
-    } else if (transitionNotFinalized) {
+    } else if (isTransitionNotFinalized(executionPayloadContext)) {
       fallbackReason = FallbackReason.TRANSITION_NOT_FINALIZED;
+    } else if (isCircuitBreakerEngaged(state)) {
+      fallbackReason = FallbackReason.CIRCUIT_BREAKER_ENGAGED;
     } else if (builderClient.isEmpty()) {
       fallbackReason = FallbackReason.BUILDER_NOT_CONFIGURED;
     } else if (!isBuilderAvailable()) {
@@ -519,6 +525,26 @@ private void updateBuilderAvailability() {
             throwable -> markBuilderAsNotAvailable(getMessageOrSimpleName(throwable)));
   }
 
+  private boolean isTransitionNotFinalized(final ExecutionPayloadContext executionPayloadContext) {
+    return executionPayloadContext.getForkChoiceState().getFinalizedExecutionBlockHash().isZero();
+  }
+
+  private boolean isCircuitBreakerEngaged(final BeaconState state) {
+    try {
+      return builderCircuitBreaker.isEngaged(state);
+    } catch (Exception e) {
+      if (ExceptionUtil.hasCause(e, InterruptedException.class)) {
+        LOG.debug("Shutting down");
+      } else {
+        LOG.error(
+            "Builder circuit breaker engagement failure at slot {}. Acting like it has been engaged.",
+            state.getSlot(),
+            e);
+      }
+      return true;
+    }
+  }
+
   private void markBuilderAsNotAvailable(final String errorMessage) {
     latestBuilderAvailability.set(false);
     eventLogger.builderIsOffline(errorMessage);
@@ -589,6 +615,7 @@ protected enum FallbackReason {
     NOT_NEEDED("not_needed"),
     VALIDATOR_NOT_REGISTERED("validator_not_registered"),
     TRANSITION_NOT_FINALIZED("transition_not_finalized"),
+    CIRCUIT_BREAKER_ENGAGED("circuit_breaker_engaged"),
     BUILDER_NOT_AVAILABLE("builder_not_available"),
     BUILDER_NOT_CONFIGURED("builder_not_configured"),
     BUILDER_HEADER_NOT_AVAILABLE("builder_header_not_available"),

ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerStub.java

@@ -14,25 +14,43 @@
 package tech.pegasys.teku.ethereum.executionlayer;
 
 import java.util.Optional;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.tuweni.bytes.Bytes32;
+import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.time.TimeProvider;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
+import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext;
+import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadHeader;
+import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 import tech.pegasys.teku.spec.executionlayer.ExecutionLayerChannelStub;
 
 public class ExecutionLayerManagerStub extends ExecutionLayerChannelStub
     implements ExecutionLayerManager {
+  private static final Logger LOG = LogManager.getLogger();
+
+  private final BuilderCircuitBreaker builderCircuitBreaker;
 
   public ExecutionLayerManagerStub(
       Spec spec,
       TimeProvider timeProvider,
       boolean enableTransitionEmulation,
-      final Optional<Bytes32> terminalBlockHashInTTDMode) {
+      final Optional<Bytes32> terminalBlockHashInTTDMode,
+      final BuilderCircuitBreaker builderCircuitBreaker) {
     super(spec, timeProvider, enableTransitionEmulation, terminalBlockHashInTTDMode);
+    this.builderCircuitBreaker = builderCircuitBreaker;
   }
 
   @Override
   public void onSlot(UInt64 slot) {
     // NOOP
   }
+
+  @Override
+  public SafeFuture<ExecutionPayloadHeader> builderGetHeader(
+      ExecutionPayloadContext executionPayloadContext, BeaconState state) {
+    LOG.info("Builder Circuit Breaker isEngaged: " + builderCircuitBreaker.isEngaged(state));
+    return super.builderGetHeader(executionPayloadContext, state);
+  }
 }