From 20acbf141908c98a68b3747205f04ed00fb1cbe7 Mon Sep 17 00:00:00 2001 From: Daniel Liu Date: Fri, 13 Dec 2024 14:00:11 +0800 Subject: [PATCH] core, metrics, p2p: expose various counter metrics for grafana (#19692) --- core/blockchain.go | 23 +++++++++++++++++++++++ core/headerchain.go | 6 ++++++ go.mod | 1 + go.sum | 5 +++++ metrics/cpu.go | 36 ++++++++++++++++++++++++++++++++++++ metrics/cpu_syscall.go | 35 +++++++++++++++++++++++++++++++++++ metrics/cpu_windows.go | 23 +++++++++++++++++++++++ metrics/metrics.go | 23 +++++++++++++++++++---- p2p/server.go | 4 +--- 9 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 metrics/cpu.go create mode 100644 metrics/cpu_syscall.go create mode 100644 metrics/cpu_windows.go diff --git a/core/blockchain.go b/core/blockchain.go index b90e461cc6ae..d11a421978ca 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -56,6 +56,10 @@ import ( ) var ( + headBlockGauge = metrics.NewRegisteredGauge("chain/head/block", nil) + headHeaderGauge = metrics.NewRegisteredGauge("chain/head/header", nil) + headFastBlockGauge = metrics.NewRegisteredGauge("chain/head/receipt", nil) + accountReadTimer = metrics.NewRegisteredTimer("chain/account/reads", nil) accountHashTimer = metrics.NewRegisteredTimer("chain/account/hashes", nil) accountUpdateTimer = metrics.NewRegisteredTimer("chain/account/updates", nil) @@ -355,6 +359,7 @@ func (bc *BlockChain) loadLastState() error { } // Everything seems to be fine, set as the head block bc.currentBlock.Store(currentBlock) + headBlockGauge.Update(int64(currentBlock.NumberU64())) // Restore the last known head header currentHeader := currentBlock.Header() @@ -374,9 +379,12 @@ func (bc *BlockChain) loadLastState() error { // Restore the last known head fast block bc.currentFastBlock.Store(currentBlock) + headFastBlockGauge.Update(int64(currentBlock.NumberU64())) + if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) { if block := bc.GetBlockByHash(head); block != nil { bc.currentFastBlock.Store(block) + headFastBlockGauge.Update(int64(block.NumberU64())) } } @@ -422,23 +430,28 @@ func (bc *BlockChain) SetHead(head uint64) error { // Rewind the block chain, ensuring we don't end up with a stateless head block if currentBlock := bc.CurrentBlock(); currentBlock != nil && currentHeader.Number.Uint64() < currentBlock.NumberU64() { bc.currentBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())) + headBlockGauge.Update(int64(currentHeader.Number.Uint64())) } if currentBlock := bc.CurrentBlock(); currentBlock != nil { if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil { // Rewound state missing, rolled back to before pivot, reset to genesis bc.currentBlock.Store(bc.genesisBlock) + headBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) } } // Rewind the fast block in a simpleton way to the target head if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock != nil && currentHeader.Number.Uint64() < currentFastBlock.NumberU64() { bc.currentFastBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())) + headFastBlockGauge.Update(int64(currentHeader.Number.Uint64())) } // If either blocks reached nil, reset to the genesis state if currentBlock := bc.CurrentBlock(); currentBlock == nil { bc.currentBlock.Store(bc.genesisBlock) + headBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) } if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock == nil { bc.currentFastBlock.Store(bc.genesisBlock) + headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) } currentBlock := bc.CurrentBlock() currentFastBlock := bc.CurrentFastBlock() @@ -463,6 +476,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { // If all checks out, manually set the head block bc.mu.Lock() bc.currentBlock.Store(block) + headBlockGauge.Update(int64(block.NumberU64())) bc.mu.Unlock() log.Info("Committed new head block", "number", block.Number(), "hash", hash) @@ -593,9 +607,12 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { bc.genesisBlock = genesis bc.insert(bc.genesisBlock, false) bc.currentBlock.Store(bc.genesisBlock) + headBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) + bc.hc.SetGenesis(bc.genesisBlock.Header()) bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) bc.currentFastBlock.Store(bc.genesisBlock) + headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) return nil } @@ -695,7 +712,9 @@ func (bc *BlockChain) insert(block *types.Block, writeBlock bool) { if writeBlock { rawdb.WriteBlock(bc.db, block) } + bc.currentBlock.Store(block) + headBlockGauge.Update(int64(block.NumberU64())) // save cache BlockSigners if bc.chainConfig.XDPoS != nil && !bc.chainConfig.IsTIPSigning(block.Number()) { @@ -713,6 +732,7 @@ func (bc *BlockChain) insert(block *types.Block, writeBlock bool) { log.Crit("Failed to insert head fast block hash", "err", err) } bc.currentFastBlock.Store(block) + headFastBlockGauge.Update(int64(block.NumberU64())) } } @@ -1050,10 +1070,12 @@ func (bc *BlockChain) Rollback(chain []common.Hash) { newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1) bc.currentFastBlock.Store(newFastBlock) WriteHeadFastBlockHash(bc.db, newFastBlock.Hash()) + headFastBlockGauge.Update(int64(newFastBlock.NumberU64())) } if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash { newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1) bc.currentBlock.Store(newBlock) + headBlockGauge.Update(int64(newBlock.NumberU64())) rawdb.WriteHeadBlockHash(bc.db, newBlock.Hash()) } } @@ -1136,6 +1158,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ log.Crit("Failed to update head fast block hash", "err", err) } bc.currentFastBlock.Store(head) + headFastBlockGauge.Update(int64(head.NumberU64())) } } bc.mu.Unlock() diff --git a/core/headerchain.go b/core/headerchain.go index b3cdc10f68d5..51a876d2ecac 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -101,6 +101,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c } } hc.currentHeaderHash = hc.CurrentHeader().Hash() + headHeaderGauge.Update(hc.CurrentHeader().Number.Int64()) return hc, nil } @@ -176,8 +177,10 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er if err := WriteHeadHeaderHash(hc.chainDb, hash); err != nil { log.Crit("Failed to insert head header hash", "err", err) } + hc.currentHeaderHash = hash hc.currentHeader.Store(types.CopyHeader(header)) + headHeaderGauge.Update(header.Number.Int64()) status = CanonStatTy } else { @@ -392,8 +395,10 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { if err := WriteHeadHeaderHash(hc.chainDb, head.Hash()); err != nil { log.Crit("Failed to insert head header hash", "err", err) } + hc.currentHeader.Store(head) hc.currentHeaderHash = head.Hash() + headHeaderGauge.Update(head.Number.Int64()) } // DeleteCallback is a callback function that is called by SetHead before @@ -432,6 +437,7 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { hc.currentHeader.Store(hc.genesisHeader) } hc.currentHeaderHash = hc.CurrentHeader().Hash() + headHeaderGauge.Update(hc.CurrentHeader().Number.Int64()) if err := WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash); err != nil { log.Crit("Failed to reset head header hash", "err", err) diff --git a/go.mod b/go.mod index d849d578c82b..109e053747f8 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/crate-crypto/go-kzg-4844 v0.7.0 github.com/deckarep/golang-set v1.8.0 github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 + github.com/elastic/gosigar v0.14.3 github.com/ethereum/c-kzg-4844 v0.4.0 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-isatty v0.0.17 diff --git a/go.sum b/go.sum index 55a922e8cc24..b826a51c6e9e 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 h1:Y9vTBSsV4hSwPSj4bac github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= +github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= @@ -166,6 +168,7 @@ github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= @@ -192,6 +195,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -245,6 +249,7 @@ gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUk gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/metrics/cpu.go b/metrics/cpu.go new file mode 100644 index 000000000000..3278d81616a3 --- /dev/null +++ b/metrics/cpu.go @@ -0,0 +1,36 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package metrics + +import "github.com/elastic/gosigar" + +// CPUStats is the system and process CPU stats. +type CPUStats struct { + GlobalTime int64 // Time spent by the CPU working on all processes + GlobalWait int64 // Time spent by waiting on disk for all processes + LocalTime int64 // Time spent by the CPU working on this process +} + +// ReadCPUStats retrieves the current CPU stats. +func ReadCPUStats(stats *CPUStats) { + global := gosigar.Cpu{} + global.Get() + + stats.GlobalTime = int64(global.User + global.Nice + global.Sys) + stats.GlobalWait = int64(global.Wait) + stats.LocalTime = getProcessCPUTime() +} diff --git a/metrics/cpu_syscall.go b/metrics/cpu_syscall.go new file mode 100644 index 000000000000..c111d89e7fcf --- /dev/null +++ b/metrics/cpu_syscall.go @@ -0,0 +1,35 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build !windows + +package metrics + +import ( + "syscall" + + "github.com/XinFinOrg/XDPoSChain/log" +) + +// getProcessCPUTime retrieves the process' CPU time since program startup. +func getProcessCPUTime() int64 { + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + log.Warn("Failed to retrieve CPU time", "err", err) + return 0 + } + return int64(usage.Utime.Sec+usage.Stime.Sec)*100 + int64(usage.Utime.Usec+usage.Stime.Usec)/10000 //nolint:unconvert +} diff --git a/metrics/cpu_windows.go b/metrics/cpu_windows.go new file mode 100644 index 000000000000..fb29a52a82c1 --- /dev/null +++ b/metrics/cpu_windows.go @@ -0,0 +1,23 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package metrics + +// getProcessCPUTime returns 0 on Windows as there is no system call to resolve +// the actual process' CPU time. +func getProcessCPUTime() int64 { + return 0 +} diff --git a/metrics/metrics.go b/metrics/metrics.go index bd23e4b24747..74de03a9dcc7 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -61,18 +61,27 @@ func CollectProcessMetrics(refresh time.Duration) { if !Enabled { return } + refreshFreq := int64(refresh / time.Second) + // Create the various data collectors + cpuStats := make([]*CPUStats, 2) memstats := make([]*runtime.MemStats, 2) diskstats := make([]*DiskStats, 2) for i := 0; i < len(memstats); i++ { + cpuStats[i] = new(CPUStats) memstats[i] = new(runtime.MemStats) diskstats[i] = new(DiskStats) } // Define the various metrics to collect + cpuSysLoad := GetOrRegisterGauge("system/cpu/sysload", DefaultRegistry) + cpuSysWait := GetOrRegisterGauge("system/cpu/syswait", DefaultRegistry) + cpuProcLoad := GetOrRegisterGauge("system/cpu/procload", DefaultRegistry) + + memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry) memAllocs := GetOrRegisterMeter("system/memory/allocs", DefaultRegistry) memFrees := GetOrRegisterMeter("system/memory/frees", DefaultRegistry) - memInuse := GetOrRegisterMeter("system/memory/inuse", DefaultRegistry) - memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry) + memHeld := GetOrRegisterGauge("system/memory/held", DefaultRegistry) + memUsed := GetOrRegisterGauge("system/memory/used", DefaultRegistry) var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter var diskReadBytesCounter, diskWriteBytesCounter Counter @@ -91,11 +100,17 @@ func CollectProcessMetrics(refresh time.Duration) { location1 := i % 2 location2 := (i - 1) % 2 + ReadCPUStats(cpuStats[location1]) + cpuSysLoad.Update((cpuStats[location1].GlobalTime - cpuStats[location2].GlobalTime) / refreshFreq) + cpuSysWait.Update((cpuStats[location1].GlobalWait - cpuStats[location2].GlobalWait) / refreshFreq) + cpuProcLoad.Update((cpuStats[location1].LocalTime - cpuStats[location2].LocalTime) / refreshFreq) + runtime.ReadMemStats(memstats[location1]) + memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs)) memAllocs.Mark(int64(memstats[location1].Mallocs - memstats[location2].Mallocs)) memFrees.Mark(int64(memstats[location1].Frees - memstats[location2].Frees)) - memInuse.Mark(int64(memstats[location1].Alloc - memstats[location2].Alloc)) - memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs)) + memHeld.Update(int64(memstats[location1].HeapSys - memstats[location1].HeapReleased)) + memUsed.Update(int64(memstats[location1].Alloc)) if ReadDiskStats(diskstats[location1]) == nil { diskReads.Mark(diskstats[location1].ReadCount - diskstats[location2].ReadCount) diff --git a/p2p/server.go b/p2p/server.go index fabbb9cdbfcd..51579ff7c30c 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -650,9 +650,7 @@ running: // This channel is used by RemoveTrustedPeer to remove an enode // from the trusted node set. srv.log.Trace("Removing trusted node", "node", n) - if _, ok := trusted[n.ID]; ok { - delete(trusted, n.ID) - } + delete(trusted, n.ID) // Unmark any already-connected peer as trusted if p, ok := peers[n.ID]; ok { p.rw.set(trustedConn, false)