From 4da6661d4abd1dee13e7b27137b89269f998824d Mon Sep 17 00:00:00 2001 From: dvovk Date: Sat, 13 Jul 2024 17:12:12 +0200 Subject: [PATCH 1/5] updated blkex --- erigon-lib/diagnostics/block_execution.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/erigon-lib/diagnostics/block_execution.go b/erigon-lib/diagnostics/block_execution.go index eb1c3dd4949..f62064cdc37 100644 --- a/erigon-lib/diagnostics/block_execution.go +++ b/erigon-lib/diagnostics/block_execution.go @@ -37,9 +37,7 @@ func (d *DiagnosticClient) runBlockExecutionListener(rootCtx context.Context) { case <-rootCtx.Done(): return case info := <-ch: - d.mu.Lock() - d.syncStats.BlockExecution = info - d.mu.Unlock() + d.SetBlockExecutionStats(info) if d.syncStats.SyncFinished { return @@ -48,3 +46,13 @@ func (d *DiagnosticClient) runBlockExecutionListener(rootCtx context.Context) { } }() } + +func (d *DiagnosticClient) SetBlockExecutionStats(stats BlockExecutionStatistics) { + d.mu.Lock() + defer d.mu.Unlock() + d.setBlockExecutionStats(stats) +} + +func (d *DiagnosticClient) setBlockExecutionStats(stats BlockExecutionStatistics) { + d.syncStats.BlockExecution = stats +} From 540f823a02722320ba0451c391c0d7a771188dc7 Mon Sep 17 00:00:00 2001 From: dvovk Date: Sat, 13 Jul 2024 17:29:33 +0200 Subject: [PATCH 2/5] test --- erigon-lib/diagnostics/block_execution.go | 46 ++++++++++++++----- erigon-lib/diagnostics/bodies.go | 56 ++++++++++++++++++----- erigon-lib/diagnostics/entities.go | 15 +----- 3 files changed, 79 insertions(+), 38 deletions(-) diff --git a/erigon-lib/diagnostics/block_execution.go b/erigon-lib/diagnostics/block_execution.go index f62064cdc37..13d6290f700 100644 --- a/erigon-lib/diagnostics/block_execution.go +++ b/erigon-lib/diagnostics/block_execution.go @@ -18,10 +18,43 @@ package diagnostics import ( "context" + "sync" "github.com/ledgerwatch/erigon-lib/log/v3" ) +type BlockEexcStatsData struct { + data BlockExecutionStatistics + mu sync.Mutex +} + +type BlockExecutionStatistics struct { + From uint64 `json:"from"` + To uint64 `json:"to"` + BlockNumber uint64 `json:"blockNumber"` + BlkPerSec float64 `json:"blkPerSec"` + TxPerSec float64 `json:"txPerSec"` + MgasPerSec float64 `json:"mgasPerSec"` + GasState float64 `json:"gasState"` + Batch uint64 `json:"batch"` + Alloc uint64 `json:"alloc"` + Sys uint64 `json:"sys"` + TimeElapsed float64 `json:"timeElapsed"` +} + +func (b *BlockEexcStatsData) SetData(d BlockExecutionStatistics) { + b.mu.Lock() + defer b.mu.Unlock() + b.data = d +} + +func (b *BlockEexcStatsData) Data() (d BlockExecutionStatistics) { + b.mu.Lock() + d = b.data + b.mu.Unlock() + return +} + func (d *DiagnosticClient) setupBlockExecutionDiagnostics(rootCtx context.Context) { d.runBlockExecutionListener(rootCtx) } @@ -37,8 +70,7 @@ func (d *DiagnosticClient) runBlockExecutionListener(rootCtx context.Context) { case <-rootCtx.Done(): return case info := <-ch: - d.SetBlockExecutionStats(info) - + d.syncStats.BlockExecution.SetData(info) if d.syncStats.SyncFinished { return } @@ -46,13 +78,3 @@ func (d *DiagnosticClient) runBlockExecutionListener(rootCtx context.Context) { } }() } - -func (d *DiagnosticClient) SetBlockExecutionStats(stats BlockExecutionStatistics) { - d.mu.Lock() - defer d.mu.Unlock() - d.setBlockExecutionStats(stats) -} - -func (d *DiagnosticClient) setBlockExecutionStats(stats BlockExecutionStatistics) { - d.syncStats.BlockExecution = stats -} diff --git a/erigon-lib/diagnostics/bodies.go b/erigon-lib/diagnostics/bodies.go index d52ebcac4d7..def8cfefae2 100644 --- a/erigon-lib/diagnostics/bodies.go +++ b/erigon-lib/diagnostics/bodies.go @@ -42,9 +42,7 @@ func (d *DiagnosticClient) runBodiesBlockDownloadListener(rootCtx context.Contex case <-rootCtx.Done(): return case info := <-ch: - d.bodiesMutex.Lock() - d.bodies.BlockDownload = info - d.bodiesMutex.Unlock() + d.setBodiesDownloadBlockUpdate(info) } } @@ -62,9 +60,7 @@ func (d *DiagnosticClient) runBodiesBlockWriteListener(rootCtx context.Context) case <-rootCtx.Done(): return case info := <-ch: - d.bodiesMutex.Lock() - d.bodies.BlockWrite = info - d.bodiesMutex.Unlock() + d.setBodiesWriteBlockUpdate(info) } } @@ -82,9 +78,7 @@ func (d *DiagnosticClient) runBodiesProcessedListener(rootCtx context.Context) { case <-rootCtx.Done(): return case info := <-ch: - d.bodiesMutex.Lock() - d.bodies.Processed = info - d.bodiesMutex.Unlock() + d.setBodiesProcessedUpdate(info) } } @@ -102,15 +96,53 @@ func (d *DiagnosticClient) runBodiesProcessingListener(rootCtx context.Context) case <-rootCtx.Done(): return case info := <-ch: - d.bodiesMutex.Lock() - d.bodies.Processing = info - d.bodiesMutex.Unlock() + d.setBodiesProcessingUpdate(info) } } }() } +func (d *DiagnosticClient) SetBodiesDownloadBlockUpdate(info BodiesDownloadBlockUpdate) { + d.bodiesMutex.Lock() + defer d.bodiesMutex.Unlock() + d.setBodiesDownloadBlockUpdate(info) +} + +func (d *DiagnosticClient) setBodiesDownloadBlockUpdate(info BodiesDownloadBlockUpdate) { + d.bodies.BlockDownload = info +} + +func (d *DiagnosticClient) SetBodiesWriteBlockUpdate(info BodiesWriteBlockUpdate) { + d.bodiesMutex.Lock() + defer d.bodiesMutex.Unlock() + d.setBodiesWriteBlockUpdate(info) +} + +func (d *DiagnosticClient) setBodiesWriteBlockUpdate(info BodiesWriteBlockUpdate) { + d.bodies.BlockWrite = info +} + +func (d *DiagnosticClient) SetBodiesProcessingUpdate(info BodiesProcessingUpdate) { + d.bodiesMutex.Lock() + defer d.bodiesMutex.Unlock() + d.setBodiesProcessingUpdate(info) +} + +func (d *DiagnosticClient) setBodiesProcessingUpdate(info BodiesProcessingUpdate) { + d.bodies.Processing = info +} + +func (d *DiagnosticClient) SetBodiesProcessedUpdate(info BodiesProcessedUpdate) { + d.bodiesMutex.Lock() + defer d.bodiesMutex.Unlock() + d.setBodiesProcessedUpdate(info) +} + +func (d *DiagnosticClient) setBodiesProcessedUpdate(info BodiesProcessedUpdate) { + d.bodies.Processed = info +} + func (d *DiagnosticClient) BodiesInfoJson(w io.Writer) { d.bodiesMutex.Lock() defer d.bodiesMutex.Unlock() diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index ea83615c163..5cdb3fa648e 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -91,7 +91,7 @@ type SyncStatistics struct { SnapshotDownload SnapshotDownloadStatistics `json:"snapshotDownload"` SnapshotIndexing SnapshotIndexingStatistics `json:"snapshotIndexing"` SnapshotFillDB SnapshotFillDBStatistics `json:"snapshotFillDB"` - BlockExecution BlockExecutionStatistics `json:"blockExecution"` + BlockExecution BlockEexcStatsData `json:"blockExecution"` SyncFinished bool `json:"syncFinished"` } @@ -167,19 +167,6 @@ type SnapshotFillDBStageUpdate struct { Stage SnapshotFillDBStage `json:"stage"` TimeElapsed float64 `json:"timeElapsed"` } -type BlockExecutionStatistics struct { - From uint64 `json:"from"` - To uint64 `json:"to"` - BlockNumber uint64 `json:"blockNumber"` - BlkPerSec float64 `json:"blkPerSec"` - TxPerSec float64 `json:"txPerSec"` - MgasPerSec float64 `json:"mgasPerSec"` - GasState float64 `json:"gasState"` - Batch uint64 `json:"batch"` - Alloc uint64 `json:"alloc"` - Sys uint64 `json:"sys"` - TimeElapsed float64 `json:"timeElapsed"` -} type SnapshoFilesList struct { Files []string `json:"files"` From 78723f539f0945b5d452eae69f766f45ab156713 Mon Sep 17 00:00:00 2001 From: dvovk Date: Sat, 13 Jul 2024 17:30:37 +0200 Subject: [PATCH 3/5] upd --- erigon-lib/diagnostics/bodies.go | 56 +++++++------------------------- 1 file changed, 12 insertions(+), 44 deletions(-) diff --git a/erigon-lib/diagnostics/bodies.go b/erigon-lib/diagnostics/bodies.go index def8cfefae2..d52ebcac4d7 100644 --- a/erigon-lib/diagnostics/bodies.go +++ b/erigon-lib/diagnostics/bodies.go @@ -42,7 +42,9 @@ func (d *DiagnosticClient) runBodiesBlockDownloadListener(rootCtx context.Contex case <-rootCtx.Done(): return case info := <-ch: - d.setBodiesDownloadBlockUpdate(info) + d.bodiesMutex.Lock() + d.bodies.BlockDownload = info + d.bodiesMutex.Unlock() } } @@ -60,7 +62,9 @@ func (d *DiagnosticClient) runBodiesBlockWriteListener(rootCtx context.Context) case <-rootCtx.Done(): return case info := <-ch: - d.setBodiesWriteBlockUpdate(info) + d.bodiesMutex.Lock() + d.bodies.BlockWrite = info + d.bodiesMutex.Unlock() } } @@ -78,7 +82,9 @@ func (d *DiagnosticClient) runBodiesProcessedListener(rootCtx context.Context) { case <-rootCtx.Done(): return case info := <-ch: - d.setBodiesProcessedUpdate(info) + d.bodiesMutex.Lock() + d.bodies.Processed = info + d.bodiesMutex.Unlock() } } @@ -96,53 +102,15 @@ func (d *DiagnosticClient) runBodiesProcessingListener(rootCtx context.Context) case <-rootCtx.Done(): return case info := <-ch: - d.setBodiesProcessingUpdate(info) + d.bodiesMutex.Lock() + d.bodies.Processing = info + d.bodiesMutex.Unlock() } } }() } -func (d *DiagnosticClient) SetBodiesDownloadBlockUpdate(info BodiesDownloadBlockUpdate) { - d.bodiesMutex.Lock() - defer d.bodiesMutex.Unlock() - d.setBodiesDownloadBlockUpdate(info) -} - -func (d *DiagnosticClient) setBodiesDownloadBlockUpdate(info BodiesDownloadBlockUpdate) { - d.bodies.BlockDownload = info -} - -func (d *DiagnosticClient) SetBodiesWriteBlockUpdate(info BodiesWriteBlockUpdate) { - d.bodiesMutex.Lock() - defer d.bodiesMutex.Unlock() - d.setBodiesWriteBlockUpdate(info) -} - -func (d *DiagnosticClient) setBodiesWriteBlockUpdate(info BodiesWriteBlockUpdate) { - d.bodies.BlockWrite = info -} - -func (d *DiagnosticClient) SetBodiesProcessingUpdate(info BodiesProcessingUpdate) { - d.bodiesMutex.Lock() - defer d.bodiesMutex.Unlock() - d.setBodiesProcessingUpdate(info) -} - -func (d *DiagnosticClient) setBodiesProcessingUpdate(info BodiesProcessingUpdate) { - d.bodies.Processing = info -} - -func (d *DiagnosticClient) SetBodiesProcessedUpdate(info BodiesProcessedUpdate) { - d.bodiesMutex.Lock() - defer d.bodiesMutex.Unlock() - d.setBodiesProcessedUpdate(info) -} - -func (d *DiagnosticClient) setBodiesProcessedUpdate(info BodiesProcessedUpdate) { - d.bodies.Processed = info -} - func (d *DiagnosticClient) BodiesInfoJson(w io.Writer) { d.bodiesMutex.Lock() defer d.bodiesMutex.Unlock() From 298ab7b95de95197d76bcea3ef958a31c14c43fa Mon Sep 17 00:00:00 2001 From: dvovk Date: Sat, 13 Jul 2024 17:35:34 +0200 Subject: [PATCH 4/5] upd --- erigon-lib/diagnostics/block_execution.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/erigon-lib/diagnostics/block_execution.go b/erigon-lib/diagnostics/block_execution.go index 13d6290f700..7d41071ce2e 100644 --- a/erigon-lib/diagnostics/block_execution.go +++ b/erigon-lib/diagnostics/block_execution.go @@ -18,6 +18,8 @@ package diagnostics import ( "context" + "encoding/json" + "io" "sync" "github.com/ledgerwatch/erigon-lib/log/v3" @@ -78,3 +80,9 @@ func (d *DiagnosticClient) runBlockExecutionListener(rootCtx context.Context) { } }() } + +func (d *DiagnosticClient) BlockExecutionInfoJson(w io.Writer) { + if err := json.NewEncoder(w).Encode(d.syncStats.BlockExecution.Data()); err != nil { + log.Debug("[diagnostics] BlockExecutionInfoJson", "err", err) + } +} From 9b8c11c992a16508246cb4c51943da6b28b5a1fe Mon Sep 17 00:00:00 2001 From: dvovk Date: Sat, 13 Jul 2024 18:09:32 +0200 Subject: [PATCH 5/5] save --- erigon-lib/diagnostics/block_execution.go | 4 ++-- erigon-lib/diagnostics/client.go | 1 + erigon-lib/diagnostics/entities.go | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/diagnostics/block_execution.go b/erigon-lib/diagnostics/block_execution.go index 7d41071ce2e..54cf9de70d8 100644 --- a/erigon-lib/diagnostics/block_execution.go +++ b/erigon-lib/diagnostics/block_execution.go @@ -72,7 +72,7 @@ func (d *DiagnosticClient) runBlockExecutionListener(rootCtx context.Context) { case <-rootCtx.Done(): return case info := <-ch: - d.syncStats.BlockExecution.SetData(info) + d.BlockExecution.SetData(info) if d.syncStats.SyncFinished { return } @@ -82,7 +82,7 @@ func (d *DiagnosticClient) runBlockExecutionListener(rootCtx context.Context) { } func (d *DiagnosticClient) BlockExecutionInfoJson(w io.Writer) { - if err := json.NewEncoder(w).Encode(d.syncStats.BlockExecution.Data()); err != nil { + if err := json.NewEncoder(w).Encode(d.BlockExecution.Data()); err != nil { log.Debug("[diagnostics] BlockExecutionInfoJson", "err", err) } } diff --git a/erigon-lib/diagnostics/client.go b/erigon-lib/diagnostics/client.go index 0f653b589a1..983a63459ee 100644 --- a/erigon-lib/diagnostics/client.go +++ b/erigon-lib/diagnostics/client.go @@ -41,6 +41,7 @@ type DiagnosticClient struct { syncStages []SyncStage syncStats SyncStatistics + BlockExecution BlockEexcStatsData snapshotFileList SnapshoFilesList mu sync.Mutex headerMutex sync.Mutex diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index 5cdb3fa648e..08bbeca5f20 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -91,7 +91,6 @@ type SyncStatistics struct { SnapshotDownload SnapshotDownloadStatistics `json:"snapshotDownload"` SnapshotIndexing SnapshotIndexingStatistics `json:"snapshotIndexing"` SnapshotFillDB SnapshotFillDBStatistics `json:"snapshotFillDB"` - BlockExecution BlockEexcStatsData `json:"blockExecution"` SyncFinished bool `json:"syncFinished"` }