Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: request result simulation #236

Draft
wants to merge 50 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
8b8ab9d
add db schema and start reimplementing validator cache with persistence
pk910 Dec 17, 2024
182988e
continue working on validator set persistence
pk910 Dec 18, 2024
e45411c
continue implementation
pk910 Dec 19, 2024
0196c0d
Merge branch 'master' into pk910/validators-in-db
pk910 Dec 19, 2024
fc6c884
continue implementation
pk910 Jan 6, 2025
b86abd1
Merge branch 'master' into pk910/validators-in-db
pk910 Jan 23, 2025
a82df4e
continue implementation
pk910 Jan 23, 2025
a1f880b
cleanup
pk910 Jan 23, 2025
0848cb2
fix performance for filtered slot requests during long unfinality
pk910 Jan 24, 2025
1904aa3
Merge branch 'pk910/unfinality-performance-fixes' into pk910/validato…
pk910 Jan 24, 2025
3d7bc51
Merge branch 'pk910/unfinality-performance-fixes' into pk910/validato…
pk910 Jan 24, 2025
f52ce81
update validator activity page
pk910 Jan 24, 2025
7ab3bea
add log output for validator persistence
pk910 Jan 24, 2025
81b975d
fix validator persistence db queue
pk910 Jan 24, 2025
39be739
fix validator name filter
pk910 Jan 25, 2025
439a108
fix log message when persisting validators
pk910 Jan 25, 2025
1451203
Merge branch 'master' into pk910/validators-in-db
pk910 Jan 27, 2025
75a99a0
fix duty precalculation when validator set changed since last epoch
pk910 Feb 1, 2025
3822c35
show validator set restoration time in logs on startup
pk910 Feb 1, 2025
71177b4
dispose blocks to explicitly free memory for blocks even when still r…
pk910 Feb 2, 2025
669b899
add optional leveldb cache for validator pubkeys & activity
pk910 Feb 3, 2025
a845bb4
improve performance for activity cache
pk910 Feb 3, 2025
2d17c68
remove activity cache in leveldb (performance issues)
pk910 Feb 3, 2025
956e630
fix cache debug page
pk910 Feb 3, 2025
a8b8ba9
Merge branch 'master' into pk910/validators-in-db
pk910 Feb 7, 2025
3344068
add getter for validator status aggregation
pk910 Feb 7, 2025
898c4f9
fix validator status filter
pk910 Feb 7, 2025
5c4732d
make fork detection more resilent for non-permanent execution (dev se…
pk910 Feb 7, 2025
c196633
fix validators filter
pk910 Feb 7, 2025
ef7fffc
small fix for validator status aggregation
pk910 Feb 7, 2025
74e10b9
fix validator status sql query
pk910 Feb 7, 2025
d2d32db
code cleanup & performance improvements
pk910 Feb 7, 2025
4a02d56
commit
pk910 Feb 7, 2025
390137d
Merge branch 'pk910/validators-in-db' into pk910/request-simulation
pk910 Feb 7, 2025
c238cba
commit
pk910 Feb 8, 2025
39c3761
show simulated request results on UI
pk910 Feb 8, 2025
fad9941
fix result simulation in synchronizer
pk910 Feb 8, 2025
2b482e6
fix panic in EpochStats.precomputeFromParentState
pk910 Feb 8, 2025
faf6a86
Merge branch 'pk910/validators-in-db' into pk910/request-simulation
pk910 Feb 8, 2025
040084f
cache block operation results & clean up code
pk910 Feb 8, 2025
8adbcf0
fix table size for consolidations/withdrawals list
pk910 Feb 8, 2025
f30b64a
init controls on lazy loaded content
pk910 Feb 8, 2025
04164a5
show withdrawal request status on validator details page
pk910 Feb 8, 2025
c251d31
fix show additional withdrawal requests link on validator details page
pk910 Feb 8, 2025
e037a17
avoid incomplete spec warnings for non-scheduled forks
pk910 Feb 8, 2025
d20a4d3
fix state pullution from state simulator
pk910 Feb 9, 2025
aa2c4e2
avoid reusing cached simulation results for orphaned blocks on same slot
pk910 Feb 9, 2025
2468ba9
fix operation result error codes
pk910 Feb 9, 2025
60fb061
improve error messages
pk910 Feb 9, 2025
ac53118
fix schema upgrade for pgsql
pk910 Feb 13, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 45 additions & 38 deletions clients/consensus/chainspec.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,44 +20,51 @@ type ForkVersion struct {

// https://github.com/ethereum/consensus-specs/blob/dev/configs/mainnet.yaml
type ChainSpec struct {
PresetBase string `yaml:"PRESET_BASE"`
ConfigName string `yaml:"CONFIG_NAME" check-if:"false"`
MinGenesisTime time.Time `yaml:"MIN_GENESIS_TIME"`
GenesisForkVersion phase0.Version `yaml:"GENESIS_FORK_VERSION"`
AltairForkVersion phase0.Version `yaml:"ALTAIR_FORK_VERSION"`
AltairForkEpoch *uint64 `yaml:"ALTAIR_FORK_EPOCH"`
BellatrixForkVersion phase0.Version `yaml:"BELLATRIX_FORK_VERSION"`
BellatrixForkEpoch *uint64 `yaml:"BELLATRIX_FORK_EPOCH"`
CapellaForkVersion phase0.Version `yaml:"CAPELLA_FORK_VERSION"`
CapellaForkEpoch *uint64 `yaml:"CAPELLA_FORK_EPOCH"`
DenebForkVersion phase0.Version `yaml:"DENEB_FORK_VERSION"`
DenebForkEpoch *uint64 `yaml:"DENEB_FORK_EPOCH"`
ElectraForkVersion phase0.Version `yaml:"ELECTRA_FORK_VERSION" check-if-fork:"ElectraForkEpoch"`
ElectraForkEpoch *uint64 `yaml:"ELECTRA_FORK_EPOCH"`
Eip7594ForkVersion phase0.Version `yaml:"EIP7594_FORK_VERSION" check-if-fork:"Eip7594ForkEpoch"`
Eip7594ForkEpoch *uint64 `yaml:"EIP7594_FORK_EPOCH"`
SecondsPerSlot time.Duration `yaml:"SECONDS_PER_SLOT"`
SlotsPerEpoch uint64 `yaml:"SLOTS_PER_EPOCH"`
EpochsPerHistoricalVector uint64 `yaml:"EPOCHS_PER_HISTORICAL_VECTOR"`
EpochsPerSlashingVector uint64 `yaml:"EPOCHS_PER_SLASHINGS_VECTOR"`
EpochsPerSyncCommitteePeriod uint64 `yaml:"EPOCHS_PER_SYNC_COMMITTEE_PERIOD"`
MinSeedLookahead uint64 `yaml:"MIN_SEED_LOOKAHEAD"`
ShuffleRoundCount uint64 `yaml:"SHUFFLE_ROUND_COUNT"`
MaxEffectiveBalance uint64 `yaml:"MAX_EFFECTIVE_BALANCE"`
MaxEffectiveBalanceElectra uint64 `yaml:"MAX_EFFECTIVE_BALANCE_ELECTRA" check-if-fork:"ElectraForkEpoch"`
TargetCommitteeSize uint64 `yaml:"TARGET_COMMITTEE_SIZE"`
MaxCommitteesPerSlot uint64 `yaml:"MAX_COMMITTEES_PER_SLOT"`
MinPerEpochChurnLimit uint64 `yaml:"MIN_PER_EPOCH_CHURN_LIMIT"`
ChurnLimitQuotient uint64 `yaml:"CHURN_LIMIT_QUOTIENT"`
DomainBeaconProposer phase0.DomainType `yaml:"DOMAIN_BEACON_PROPOSER"`
DomainBeaconAttester phase0.DomainType `yaml:"DOMAIN_BEACON_ATTESTER"`
DomainSyncCommittee phase0.DomainType `yaml:"DOMAIN_SYNC_COMMITTEE"`
SyncCommitteeSize uint64 `yaml:"SYNC_COMMITTEE_SIZE"`
DepositContractAddress []byte `yaml:"DEPOSIT_CONTRACT_ADDRESS"`
MaxConsolidationRequestsPerPayload uint64 `yaml:"MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD" check-if-fork:"ElectraForkEpoch"`
MaxWithdrawalRequestsPerPayload uint64 `yaml:"MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD" check-if-fork:"ElectraForkEpoch"`
DepositChainId uint64 `yaml:"DEPOSIT_CHAIN_ID"`
MinActivationBalance uint64 `yaml:"MIN_ACTIVATION_BALANCE"`
PresetBase string `yaml:"PRESET_BASE"`
ConfigName string `yaml:"CONFIG_NAME" check-if:"false"`
MinGenesisTime time.Time `yaml:"MIN_GENESIS_TIME"`
GenesisForkVersion phase0.Version `yaml:"GENESIS_FORK_VERSION"`
AltairForkVersion phase0.Version `yaml:"ALTAIR_FORK_VERSION"`
AltairForkEpoch *uint64 `yaml:"ALTAIR_FORK_EPOCH"`
BellatrixForkVersion phase0.Version `yaml:"BELLATRIX_FORK_VERSION"`
BellatrixForkEpoch *uint64 `yaml:"BELLATRIX_FORK_EPOCH"`
CapellaForkVersion phase0.Version `yaml:"CAPELLA_FORK_VERSION"`
CapellaForkEpoch *uint64 `yaml:"CAPELLA_FORK_EPOCH"`
DenebForkVersion phase0.Version `yaml:"DENEB_FORK_VERSION"`
DenebForkEpoch *uint64 `yaml:"DENEB_FORK_EPOCH"`
ElectraForkVersion phase0.Version `yaml:"ELECTRA_FORK_VERSION" check-if-fork:"ElectraForkEpoch"`
ElectraForkEpoch *uint64 `yaml:"ELECTRA_FORK_EPOCH" check-if-fork:"ElectraForkEpoch"`
Eip7594ForkVersion phase0.Version `yaml:"EIP7594_FORK_VERSION" check-if-fork:"Eip7594ForkEpoch"`
Eip7594ForkEpoch *uint64 `yaml:"EIP7594_FORK_EPOCH" check-if-fork:"Eip7594ForkEpoch"`
SecondsPerSlot time.Duration `yaml:"SECONDS_PER_SLOT"`
SlotsPerEpoch uint64 `yaml:"SLOTS_PER_EPOCH"`
EpochsPerHistoricalVector uint64 `yaml:"EPOCHS_PER_HISTORICAL_VECTOR"`
EpochsPerSlashingVector uint64 `yaml:"EPOCHS_PER_SLASHINGS_VECTOR"`
EpochsPerSyncCommitteePeriod uint64 `yaml:"EPOCHS_PER_SYNC_COMMITTEE_PERIOD"`
MinSeedLookahead uint64 `yaml:"MIN_SEED_LOOKAHEAD"`
ShuffleRoundCount uint64 `yaml:"SHUFFLE_ROUND_COUNT"`
MaxEffectiveBalance uint64 `yaml:"MAX_EFFECTIVE_BALANCE"`
MaxEffectiveBalanceElectra uint64 `yaml:"MAX_EFFECTIVE_BALANCE_ELECTRA" check-if-fork:"ElectraForkEpoch"`
TargetCommitteeSize uint64 `yaml:"TARGET_COMMITTEE_SIZE"`
MaxCommitteesPerSlot uint64 `yaml:"MAX_COMMITTEES_PER_SLOT"`
MinPerEpochChurnLimit uint64 `yaml:"MIN_PER_EPOCH_CHURN_LIMIT"`
ChurnLimitQuotient uint64 `yaml:"CHURN_LIMIT_QUOTIENT"`
DomainBeaconProposer phase0.DomainType `yaml:"DOMAIN_BEACON_PROPOSER"`
DomainBeaconAttester phase0.DomainType `yaml:"DOMAIN_BEACON_ATTESTER"`
DomainSyncCommittee phase0.DomainType `yaml:"DOMAIN_SYNC_COMMITTEE"`
SyncCommitteeSize uint64 `yaml:"SYNC_COMMITTEE_SIZE"`
DepositContractAddress []byte `yaml:"DEPOSIT_CONTRACT_ADDRESS"`
MaxConsolidationRequestsPerPayload uint64 `yaml:"MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD" check-if-fork:"ElectraForkEpoch"`
MaxWithdrawalRequestsPerPayload uint64 `yaml:"MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD" check-if-fork:"ElectraForkEpoch"`
DepositChainId uint64 `yaml:"DEPOSIT_CHAIN_ID"`
MinActivationBalance uint64 `yaml:"MIN_ACTIVATION_BALANCE"`
MaxPendingPartialsPerWithdrawalsSweep uint64 `yaml:"MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP" check-if-fork:"ElectraForkEpoch"`
PendingPartialWithdrawalsLimit uint64 `yaml:"PENDING_PARTIAL_WITHDRAWALS_LIMIT" check-if-fork:"ElectraForkEpoch"`
PendingConsolidationsLimit uint64 `yaml:"PENDING_CONSOLIDATIONS_LIMIT" check-if-fork:"ElectraForkEpoch"`
MinPerEpochChurnLimitElectra uint64 `yaml:"MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA" check-if-fork:"ElectraForkEpoch"`
MaxPerEpochActivationExitChurnLimit uint64 `yaml:"MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT" check-if-fork:"ElectraForkEpoch"`
EffectiveBalanceIncrement uint64 `yaml:"EFFECTIVE_BALANCE_INCREMENT"`
ShardCommitteePeriod uint64 `yaml:"SHARD_COMMITTEE_PERIOD"`

// EIP7594: PeerDAS
NumberOfColumns *uint64 `yaml:"NUMBER_OF_COLUMNS" check-if-fork:"Eip7594ForkEpoch"`
Expand Down
2 changes: 1 addition & 1 deletion db/consolidation_request_txs.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func InsertConsolidationRequestTxs(consolidationTxs []*dbtypes.ConsolidationRequ
argIdx += fieldCount
}
fmt.Fprint(&sql, EngineQuery(map[dbtypes.DBEngineType]string{
dbtypes.DBEnginePgsql: " ON CONFLICT (block_root, block_index) DO UPDATE SET source_index = excluded.source_index, target_index = excluded.target_index, fork_id = excluded.fork_id",
dbtypes.DBEnginePgsql: " ON CONFLICT (block_root, block_index) DO UPDATE SET source_index = excluded.source_index, target_index = excluded.target_index, dequeue_block = excluded.dequeue_block, fork_id = excluded.fork_id",
dbtypes.DBEngineSqlite: "",
}))

Expand Down
12 changes: 7 additions & 5 deletions db/consolidation_requests.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ func InsertConsolidationRequests(consolidations []*dbtypes.ConsolidationRequest,
dbtypes.DBEnginePgsql: "INSERT INTO consolidation_requests ",
dbtypes.DBEngineSqlite: "INSERT OR REPLACE INTO consolidation_requests ",
}),
"(slot_number, slot_root, slot_index, orphaned, fork_id, source_address, source_index, source_pubkey, target_index, target_pubkey, tx_hash, block_number)",
"(slot_number, slot_root, slot_index, orphaned, fork_id, source_address, source_index, source_pubkey, target_index, target_pubkey, tx_hash, block_number, result)",
" VALUES ",
)
argIdx := 0
fieldCount := 12
fieldCount := 13

args := make([]interface{}, len(consolidations)*fieldCount)
for i, consolidation := range consolidations {
Expand Down Expand Up @@ -47,10 +47,11 @@ func InsertConsolidationRequests(consolidations []*dbtypes.ConsolidationRequest,
args[argIdx+9] = consolidation.TargetPubkey[:]
args[argIdx+10] = consolidation.TxHash[:]
args[argIdx+11] = consolidation.BlockNumber
args[argIdx+12] = consolidation.Result
argIdx += fieldCount
}
fmt.Fprint(&sql, EngineQuery(map[dbtypes.DBEngineType]string{
dbtypes.DBEnginePgsql: " ON CONFLICT (slot_root, slot_index) DO UPDATE SET orphaned = excluded.orphaned, fork_id = excluded.fork_id",
dbtypes.DBEnginePgsql: " ON CONFLICT (slot_root, slot_index) DO UPDATE SET orphaned = excluded.orphaned, fork_id = excluded.fork_id, result = excluded.result",
dbtypes.DBEngineSqlite: "",
}))
_, err := tx.Exec(sql.String(), args...)
Expand All @@ -66,7 +67,7 @@ func GetConsolidationRequestsFiltered(offset uint64, limit uint32, canonicalFork
fmt.Fprint(&sql, `
WITH cte AS (
SELECT
slot_number, slot_root, slot_index, orphaned, fork_id, source_address, source_index, source_pubkey, target_index, target_pubkey, tx_hash, block_number
slot_number, slot_root, slot_index, orphaned, fork_id, source_address, source_index, source_pubkey, target_index, target_pubkey, tx_hash, block_number, result
FROM consolidation_requests
`)

Expand Down Expand Up @@ -173,7 +174,8 @@ func GetConsolidationRequestsFiltered(offset uint64, limit uint32, canonicalFork
0 AS target_index,
null AS target_pubkey,
null AS tx_hash,
0 AS block_number
0 AS block_number,
0 AS result
FROM cte
UNION ALL SELECT * FROM (
SELECT * FROM cte
Expand Down
24 changes: 24 additions & 0 deletions db/schema/pgsql/20241216124511_validator-set.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
-- +goose Up
-- +goose StatementBegin

CREATE TABLE IF NOT EXISTS public."validators" (
validator_index BIGINT NOT NULL,
pubkey bytea NOT NULL,
withdrawal_credentials bytea NOT NULL,
effective_balance BIGINT NOT NULL,
slashed BOOLEAN NOT NULL,
activation_eligibility_epoch BIGINT NOT NULL,
activation_epoch BIGINT NOT NULL,
exit_epoch BIGINT NOT NULL,
withdrawable_epoch BIGINT NOT NULL,
CONSTRAINT validators_pkey PRIMARY KEY (validator_index)
);

CREATE INDEX IF NOT EXISTS "validators_pubkey_idx"
ON public."validators" ("pubkey");

-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
SELECT 'NOT SUPPORTED';
-- +goose StatementEnd
14 changes: 14 additions & 0 deletions db/schema/pgsql/20250207221132_request-results.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-- +goose Up
-- +goose StatementBegin

ALTER TABLE public."consolidation_requests"
ADD "result" smallint NOT NULL DEFAULT 0;

ALTER TABLE public."withdrawal_requests"
ADD "result" smallint NOT NULL DEFAULT 0;

-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
SELECT 'NOT SUPPORTED';
-- +goose StatementEnd
24 changes: 24 additions & 0 deletions db/schema/sqlite/20241216124511_validator-set.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
-- +goose Up
-- +goose StatementBegin

CREATE TABLE IF NOT EXISTS "validators" (
validator_index BIGINT NOT NULL,
pubkey BLOB NOT NULL,
withdrawal_credentials BLOB NOT NULL,
effective_balance BIGINT NOT NULL,
slashed BOOLEAN NOT NULL,
activation_eligibility_epoch BIGINT NOT NULL,
activation_epoch BIGINT NOT NULL,
exit_epoch BIGINT NOT NULL,
withdrawable_epoch BIGINT NOT NULL,
PRIMARY KEY (validator_index)
);

CREATE INDEX IF NOT EXISTS "validators_pubkey_idx"
ON "validators" ("pubkey");

-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
SELECT 'NOT SUPPORTED';
-- +goose StatementEnd
14 changes: 14 additions & 0 deletions db/schema/sqlite/20250207221132_request-results.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-- +goose Up
-- +goose StatementBegin

ALTER TABLE "consolidation_requests"
ADD "result" TINYINT NOT NULL DEFAULT 0;

ALTER TABLE "withdrawal_requests"
ADD "result" TINYINT NOT NULL DEFAULT 0;

-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
SELECT 'NOT SUPPORTED';
-- +goose StatementEnd
Loading