From 65e2b6c69dd91a49765f2f86d4cc70ed2509fed5 Mon Sep 17 00:00:00 2001 From: Chris Hager Date: Mon, 3 Apr 2023 10:41:29 +0200 Subject: [PATCH 1/7] publish block first --- README.md | 1 - beaconclient/multi_beacon_client.go | 9 ++++++++- services/api/service.go | 25 +++++++++---------------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3ca0c49d..15033171 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,6 @@ redis-cli DEL boost-relay/sepolia:validators-registration boost-relay/sepolia:va #### Feature Flags * `DISABLE_PAYLOAD_DATABASE_STORAGE` - builder API - disable storing execution payloads in the database (i.e. when using memcached as data availability redundancy) -* `DISABLE_BLOCK_PUBLISHING` - disable publishing blocks to the beacon node at the end of getPayload * `DISABLE_LOWPRIO_BUILDERS` - reject block submissions by low-prio builders * `FORCE_GET_HEADER_204` - force 204 as getHeader response * `DISABLE_SSE_PAYLOAD_ATTRIBUTES` - instead of using SSE events, poll withdrawals and randao (requires custom Prysm fork) diff --git a/beaconclient/multi_beacon_client.go b/beaconclient/multi_beacon_client.go index 64be02f0..7f6c98e4 100644 --- a/beaconclient/multi_beacon_client.go +++ b/beaconclient/multi_beacon_client.go @@ -17,6 +17,7 @@ var ( ErrBeaconNodeSyncing = errors.New("beacon node is syncing or unavailable") ErrBeaconNodesUnavailable = errors.New("all beacon nodes responded with error") ErrWithdrawalsBeforeCapella = errors.New("withdrawals are not supported before capella") + ErrBeaconBlock202 = errors.New("beacon block failed validation but was still broadcast (202)") ) // IMultiBeaconClient is the interface for the MultiBeaconClient, which can manage several beacon client instances under the hood @@ -223,7 +224,13 @@ func (c *MultiBeaconClient) PublishBlock(block *common.SignedBeaconBlock) (code log.Debug("publishing block") if code, err = client.PublishBlock(block); err != nil { - log.WithField("statusCode", code).WithError(err).Warn("failed to publish block") + log.WithField("statusCode", code).WithError(err).Error("failed to publish block") + continue + } else if code == 202 { + // Should the block fail full validation, a separate success response code (202) is used to indicate that the block was successfully broadcast but failed integration. + // https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/publishBlock + log.WithField("statusCode", code).WithError(err).Error("block failed validation but was still broadcast") + err = ErrBeaconBlock202 continue } diff --git a/services/api/service.go b/services/api/service.go index ec807993..c1df6ea0 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -154,7 +154,6 @@ type RelayAPI struct { // Feature flags ffForceGetHeader204 bool - ffDisableBlockPublishing bool ffDisableLowPrioBuilders bool ffDisablePayloadDBStorage bool // disable storing the execution payloads in the database ffDisableSSEPayloadAttributes bool // instead of SSE, fall back to previous polling withdrawals+prevRandao from our custom Prysm fork @@ -238,11 +237,6 @@ func NewRelayAPI(opts RelayAPIOpts) (api *RelayAPI, err error) { api.ffForceGetHeader204 = true } - if os.Getenv("DISABLE_BLOCK_PUBLISHING") == "1" { - api.log.Warn("env: DISABLE_BLOCK_PUBLISHING - disabling publishing blocks on getPayload") - api.ffDisableBlockPublishing = true - } - if os.Getenv("DISABLE_LOWPRIO_BUILDERS") == "1" { api.log.Warn("env: DISABLE_LOWPRIO_BUILDERS - allowing only high-level builders") api.ffDisableLowPrioBuilders = true @@ -981,6 +975,15 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) } } + // Publish the signed beacon block via beacon-node + signedBeaconBlock := SignedBlindedBeaconBlockToBeaconBlock(payload, getPayloadResp) + code, err := api.beaconClient.PublishBlock(signedBeaconBlock) // errors are logged inside + if err != nil { + log.WithError(err).WithField("code", code).Error("failed to publish block") + api.RespondError(w, http.StatusBadRequest, "failed to publish block") + return + } + api.RespondOK(w, getPayloadResp) log = log.WithFields(logrus.Fields{ "numTx": getPayloadResp.NumTx(), @@ -1014,16 +1017,6 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) log.WithError(err).Error("failed to increment builder-stats after getPayload") } }() - - // Publish the signed beacon block via beacon-node - go func() { - if api.ffDisableBlockPublishing { - log.Info("publishing the block is disabled") - return - } - signedBeaconBlock := SignedBlindedBeaconBlockToBeaconBlock(payload, getPayloadResp) - _, _ = api.beaconClient.PublishBlock(signedBeaconBlock) // errors are logged inside - }() } // -------------------- From 7d5ebc275209b2b19640e926e7501e5dac7c5c91 Mon Sep 17 00:00:00 2001 From: Mateusz Morusiewicz <11313015+Ruteri@users.noreply.github.com> Date: Mon, 3 Apr 2023 11:34:58 +0200 Subject: [PATCH 2/7] Add concurrent release of the execution payload --- beaconclient/multi_beacon_client.go | 43 ++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/beaconclient/multi_beacon_client.go b/beaconclient/multi_beacon_client.go index 7f6c98e4..daf1e83e 100644 --- a/beaconclient/multi_beacon_client.go +++ b/beaconclient/multi_beacon_client.go @@ -211,6 +211,12 @@ func (c *MultiBeaconClient) beaconInstancesByLastResponse() []IBeaconInstance { return instances } +type publishResp struct { + index int + code int + err error +} + // PublishBlock publishes the signed beacon block via https://ethereum.github.io/beacon-APIs/#/ValidatorRequiredApi/publishBlock func (c *MultiBeaconClient) PublishBlock(block *common.SignedBeaconBlock) (code int, err error) { log := c.log.WithFields(logrus.Fields{ @@ -219,29 +225,46 @@ func (c *MultiBeaconClient) PublishBlock(block *common.SignedBeaconBlock) (code }) clients := c.beaconInstancesByLastResponse() + + // The chan will be cleaner up automatically once the function exists even if it was still being written to + resChans := make(chan publishResp, len(clients)) + for i, client := range clients { log := log.WithField("uri", client.GetURI()) log.Debug("publishing block") + go func(i int) { + code, err := client.PublishBlock(block) + resChans <- publishResp{ + index: i, + code: code, + err: err, + } + }(i) + } - if code, err = client.PublishBlock(block); err != nil { - log.WithField("statusCode", code).WithError(err).Error("failed to publish block") + var lastErrPublishResp publishResp + for i := 0; i < len(clients); i++ { + res := <-resChans + if res.err != nil { + log.WithField("statusCode", res.code).WithError(res.err).Error("failed to publish block") + lastErrPublishResp = res continue - } else if code == 202 { + } else if res.code == 202 { // Should the block fail full validation, a separate success response code (202) is used to indicate that the block was successfully broadcast but failed integration. // https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/publishBlock - log.WithField("statusCode", code).WithError(err).Error("block failed validation but was still broadcast") - err = ErrBeaconBlock202 + log.WithField("statusCode", res.code).WithError(res.err).Error("block failed validation but was still broadcast") + lastErrPublishResp = res continue } - c.bestBeaconIndex.Store(int64(i)) + c.bestBeaconIndex.Store(int64(res.index)) - log.WithField("statusCode", code).Info("published block") - return code, nil + log.WithField("statusCode", res.code).Info("published block") + return res.code, nil } - log.WithField("statusCode", code).WithError(err).Error("failed to publish block on any CL node") - return code, err + log.WithField("statusCode", lastErrPublishResp.code).WithError(lastErrPublishResp.err).Error("failed to publish block on any CL node") + return lastErrPublishResp.code, lastErrPublishResp.err } // GetGenesis returns the genesis info - https://ethereum.github.io/beacon-APIs/#/Beacon/getGenesis From 88673f5ccc6bc69f81f094b52b4e4a5a120a0609 Mon Sep 17 00:00:00 2001 From: Mateusz Morusiewicz <11313015+Ruteri@users.noreply.github.com> Date: Mon, 3 Apr 2023 11:41:54 +0200 Subject: [PATCH 3/7] Fix capture --- beaconclient/multi_beacon_client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beaconclient/multi_beacon_client.go b/beaconclient/multi_beacon_client.go index daf1e83e..71cd8634 100644 --- a/beaconclient/multi_beacon_client.go +++ b/beaconclient/multi_beacon_client.go @@ -232,14 +232,14 @@ func (c *MultiBeaconClient) PublishBlock(block *common.SignedBeaconBlock) (code for i, client := range clients { log := log.WithField("uri", client.GetURI()) log.Debug("publishing block") - go func(i int) { + go func(index int, client IBeaconInstance) { code, err := client.PublishBlock(block) resChans <- publishResp{ - index: i, + index: index, code: code, err: err, } - }(i) + }(i, client) } var lastErrPublishResp publishResp From 3d3d2fd9a081e2d05e8f111290e808b7e1a5944c Mon Sep 17 00:00:00 2001 From: Mateusz Morusiewicz <11313015+Ruteri@users.noreply.github.com> Date: Mon, 3 Apr 2023 11:48:01 +0200 Subject: [PATCH 4/7] Update logging --- beaconclient/multi_beacon_client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beaconclient/multi_beacon_client.go b/beaconclient/multi_beacon_client.go index 71cd8634..61ecfb97 100644 --- a/beaconclient/multi_beacon_client.go +++ b/beaconclient/multi_beacon_client.go @@ -246,24 +246,24 @@ func (c *MultiBeaconClient) PublishBlock(block *common.SignedBeaconBlock) (code for i := 0; i < len(clients); i++ { res := <-resChans if res.err != nil { - log.WithField("statusCode", res.code).WithError(res.err).Error("failed to publish block") + log.WithField("beaconIndex", res.index).WithField("statusCode", res.code).WithError(res.err).Error("failed to publish block") lastErrPublishResp = res continue } else if res.code == 202 { // Should the block fail full validation, a separate success response code (202) is used to indicate that the block was successfully broadcast but failed integration. // https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/publishBlock - log.WithField("statusCode", res.code).WithError(res.err).Error("block failed validation but was still broadcast") + log.WithField("beaconIndex", res.index).WithField("statusCode", res.code).WithError(res.err).Error("block failed validation but was still broadcast") lastErrPublishResp = res continue } c.bestBeaconIndex.Store(int64(res.index)) - log.WithField("statusCode", res.code).Info("published block") + log.WithField("beaconIndex", res.index).WithField("statusCode", res.code).Info("published block") return res.code, nil } - log.WithField("statusCode", lastErrPublishResp.code).WithError(lastErrPublishResp.err).Error("failed to publish block on any CL node") + log.Error("failed to publish block on any CL node") return lastErrPublishResp.code, lastErrPublishResp.err } From f30ac0f53b2067271a4810ced6489c609d548935 Mon Sep 17 00:00:00 2001 From: Mateusz Morusiewicz <11313015+Ruteri@users.noreply.github.com> Date: Mon, 3 Apr 2023 11:50:27 +0200 Subject: [PATCH 5/7] Log the whole URI --- beaconclient/multi_beacon_client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beaconclient/multi_beacon_client.go b/beaconclient/multi_beacon_client.go index 61ecfb97..2c89b2f6 100644 --- a/beaconclient/multi_beacon_client.go +++ b/beaconclient/multi_beacon_client.go @@ -246,20 +246,20 @@ func (c *MultiBeaconClient) PublishBlock(block *common.SignedBeaconBlock) (code for i := 0; i < len(clients); i++ { res := <-resChans if res.err != nil { - log.WithField("beaconIndex", res.index).WithField("statusCode", res.code).WithError(res.err).Error("failed to publish block") + log.WithField("beacon", clients[res.index].GetURI()).WithField("statusCode", res.code).WithError(res.err).Error("failed to publish block") lastErrPublishResp = res continue } else if res.code == 202 { // Should the block fail full validation, a separate success response code (202) is used to indicate that the block was successfully broadcast but failed integration. // https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/publishBlock - log.WithField("beaconIndex", res.index).WithField("statusCode", res.code).WithError(res.err).Error("block failed validation but was still broadcast") + log.WithField("beacon", clients[res.index].GetURI()).WithField("statusCode", res.code).WithError(res.err).Error("block failed validation but was still broadcast") lastErrPublishResp = res continue } c.bestBeaconIndex.Store(int64(res.index)) - log.WithField("beaconIndex", res.index).WithField("statusCode", res.code).Info("published block") + log.WithField("beacon", clients[res.index].GetURI()).WithField("statusCode", res.code).Info("published block") return res.code, nil } From 0fb456514b7f73079ad6be9c234cef138ae344a2 Mon Sep 17 00:00:00 2001 From: Chris Hager Date: Mon, 3 Apr 2023 11:54:58 +0200 Subject: [PATCH 6/7] add 1s delay on returning getPaylaod --- services/api/service.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/api/service.go b/services/api/service.go index c1df6ea0..79eac9e1 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -72,6 +72,7 @@ var ( numActiveValidatorProcessors = cli.GetEnvInt("NUM_ACTIVE_VALIDATOR_PROCESSORS", 10) numValidatorRegProcessors = cli.GetEnvInt("NUM_VALIDATOR_REG_PROCESSORS", 10) timeoutGetPayloadRetryMs = cli.GetEnvInt("GETPAYLOAD_RETRY_TIMEOUT_MS", 100) + timeoutGetPayloadResponseMs = cli.GetEnvInt("GETPAYLOAD_RESPONSE_TIMEOUT_MS", 1000) apiReadTimeoutMs = cli.GetEnvInt("API_TIMEOUT_READ_MS", 1500) apiReadHeaderTimeoutMs = cli.GetEnvInt("API_TIMEOUT_READHEADER_MS", 600) @@ -984,6 +985,9 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) return } + // give the beacon network some time to propagate the block + time.Sleep(time.Duration(timeoutGetPayloadResponseMs) * time.Millisecond) + api.RespondOK(w, getPayloadResp) log = log.WithFields(logrus.Fields{ "numTx": getPayloadResp.NumTx(), From b6eb1e59c991d0a96485324fa56cfdc5132f0af4 Mon Sep 17 00:00:00 2001 From: Chris Hager Date: Mon, 3 Apr 2023 12:01:48 +0200 Subject: [PATCH 7/7] fix naming --- services/api/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/api/service.go b/services/api/service.go index 79eac9e1..9ab343e6 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -72,7 +72,7 @@ var ( numActiveValidatorProcessors = cli.GetEnvInt("NUM_ACTIVE_VALIDATOR_PROCESSORS", 10) numValidatorRegProcessors = cli.GetEnvInt("NUM_VALIDATOR_REG_PROCESSORS", 10) timeoutGetPayloadRetryMs = cli.GetEnvInt("GETPAYLOAD_RETRY_TIMEOUT_MS", 100) - timeoutGetPayloadResponseMs = cli.GetEnvInt("GETPAYLOAD_RESPONSE_TIMEOUT_MS", 1000) + getPayloadResponseDelayMs = cli.GetEnvInt("GETPAYLOAD_DELAY_MS", 1000) apiReadTimeoutMs = cli.GetEnvInt("API_TIMEOUT_READ_MS", 1500) apiReadHeaderTimeoutMs = cli.GetEnvInt("API_TIMEOUT_READHEADER_MS", 600) @@ -986,7 +986,7 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) } // give the beacon network some time to propagate the block - time.Sleep(time.Duration(timeoutGetPayloadResponseMs) * time.Millisecond) + time.Sleep(time.Duration(getPayloadResponseDelayMs) * time.Millisecond) api.RespondOK(w, getPayloadResp) log = log.WithFields(logrus.Fields{