command to print historical chain statistics as CSV

cmd/geth/chaincmd.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -41,10 +42,24 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/params/types/genesisT"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/montanaflynn/stats"
 	"gopkg.in/urfave/cli.v1"
 )
 
 var (
+	unclesCommand = cli.Command{
+		Action:    utils.MigrateFlags(statsCmd),
+		Name:      "stats",
+		Usage:     "Print stats by 10,000-block interval",
+		ArgsUsage: "",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
+		Description: `
+Prints stats by interval in CSV format.
+`,
+	}
 	initCommand = cli.Command{
 		Action:    utils.MigrateFlags(initGenesis),
 		Name:      "init",
@@ -234,6 +249,88 @@ Use "ethereum dump 0" to dump the genesis block.`,
 	}
 )
 
+func statsCmd(ctx *cli.Context) error {
+	stack := makeFullNode(ctx)
+	defer stack.Close()
+
+	chain, db := utils.MakeChain(ctx, stack, true)
+	defer db.Close()
+
+	interval := uint64(10_000)
+
+	var unclesTally = make(map[int]int)
+	var difficultyStatsSet = []float64{}
+	var timedeltaStatsSet = []float64{}
+
+	// stats.Median(difficultyStatsSet)
+
+	writer := csv.NewWriter(os.Stdout)
+
+	writer.Write([]string{"block",
+		"uncles_zero", "uncles_one", "uncles_two",
+
+		"difficulty_median",
+		"difficulty_mean",
+		"difficulty_p1", "difficulty_p5", "difficulty_p25", "difficulty_p50", "difficulty_p75", "difficulty_p95", "difficulty_p99",
+
+		"timedelta_median",
+		"timedelta_mean",
+		"timedelta_p1", "timedelta_p5", "timedelta_p25", "timedelta_p50", "timedelta_p75", "timedelta_p95", "timedelta_p99",
+
+		"block_timestamp_date", "block_timestamp_time",
+	})
+
+	toS := func(v interface{}, err error) string {
+		if err != nil {
+			utils.Fatalf("%v: %v", v, err)
+		}
+		return fmt.Sprintf("%v", v)
+	}
+
+	cb := chain.CurrentBlock()
+	var parentTime uint64
+	for i := uint64(0); i <= cb.Number().Uint64(); i++ {
+		bl := chain.GetBlockByNumber(i)
+
+		unclesTally[len(bl.Uncles())]++
+
+		difficultyStatsSet = append(difficultyStatsSet, float64(bl.Difficulty().Uint64()))
+
+		if i > 1 {
+			deltaTime := bl.Time() - parentTime
+			timedeltaStatsSet = append(timedeltaStatsSet, float64(deltaTime))
+		}
+		parentTime = bl.Time()
+
+		if i > 0 && i%interval == 0 {
+			writer.Write([]string{toS(i, nil),
+				toS(unclesTally[0], nil), toS(unclesTally[1], nil), toS(unclesTally[2], nil),
+
+				toS(stats.Median(difficultyStatsSet)),
+				toS(stats.Mean(difficultyStatsSet)),
+				toS(stats.Percentile(difficultyStatsSet, 01)), toS(stats.Percentile(difficultyStatsSet, 05)), toS(stats.Percentile(difficultyStatsSet, 25)),
+				toS(stats.Percentile(difficultyStatsSet, 50)),
+				toS(stats.Percentile(difficultyStatsSet, 75)), toS(stats.Percentile(difficultyStatsSet, 95)), toS(stats.Percentile(difficultyStatsSet, 99)),
+
+				toS(stats.Median(timedeltaStatsSet)),
+				toS(stats.Mean(timedeltaStatsSet)),
+				toS(stats.Percentile(timedeltaStatsSet, 01)), toS(stats.Percentile(timedeltaStatsSet, 05)), toS(stats.Percentile(timedeltaStatsSet, 25)),
+				toS(stats.Percentile(timedeltaStatsSet, 50)),
+				toS(stats.Percentile(timedeltaStatsSet, 75)), toS(stats.Percentile(timedeltaStatsSet, 95)), toS(stats.Percentile(timedeltaStatsSet, 99)),
+
+				toS(time.Unix(int64(bl.Time()), 0).UTC().Format("2006-01-02"), nil),
+				toS(time.Unix(int64(bl.Time()), 0).UTC().Format("15:04:05"), nil),
+			})
+			writer.Flush()
+
+			unclesTally = make(map[int]int)
+			difficultyStatsSet = []float64{}
+			timedeltaStatsSet = []float64{}
+		}
+	}
+	return nil
+}
+
 // initGenesis will initialise the given JSON format genesis file and writes it as
 // the zero'd block (i.e. genesis) or will fail hard if it can't succeed.
 func initGenesis(ctx *cli.Context) error {

cmd/geth/main.go

@@ -228,6 +228,7 @@ func init() {
 	app.Copyright = "Copyright 2013-2020 The core-geth and go-ethereum Authors"
 	app.Commands = []cli.Command{
 		// See chaincmd.go:
+		unclesCommand,
 		initCommand,
 		importCommand,
 		exportCommand,

consensus/ethash/consensus.go

@@ -184,6 +184,8 @@ func (ethash *Ethash) VerifyUncles(chain consensus.ChainReader, block *types.Blo
 	uncles, ancestors := mapset.NewSet(), make(map[common.Hash]*types.Header)
 
 	number, parent := block.NumberU64()-1, block.ParentHash()
+	// 0, 1, 2, 3, 4, 5, 6
+	// parent:0, grandparent:1, ...
 	for i := 0; i < 7; i++ {
 		ancestor := chain.GetBlock(parent, number)
 		if ancestor == nil {
@@ -594,15 +596,33 @@ func accumulateRewards(config ctypes.ChainConfigurator, state *state.StateDB, he
 	blockReward := ctypes.EthashBlockReward(config, header.Number)
 
 	// Accumulate the rewards for the miner and any included uncles
-	reward := new(big.Int).Set(blockReward)
+	reward := new(big.Int).Set(blockReward) // eg. 5, 3, 2
 	r := new(big.Int)
+
+	// Can be 1 or 2 uncles.
 	for _, uncle := range uncles {
+
+		// header.Number = 10_000_45
+
+		// a. 10_000_038 + 8 => 10_000_046 (uncles are limited to latest 7 blocks)
+		// b. 10_000_044 + 8 => 10_000_52
 		r.Add(uncle.Number, big8)
+
+		// a. 10_000_046 - 10_000_45 => 1
+		// b. 10_000_52 - 10_000_45 => 7
 		r.Sub(r, header.Number)
+
+		// a. 1 * 2 => 2
+		// b. 7 * 2 => 14
 		r.Mul(r, blockReward)
+
+		// a. 2 / 8 => 0.25 ETH / uncle for uncle-miner
+		// b. 14 / 8 => 1.75 ETH / uncle for uncle-miner
 		r.Div(r, big8)
+
 		state.AddBalance(uncle.Coinbase, r)
 
+		// 2 / 32 => 1/16 =~> 0.0625 ETH (bonus for winner miner for including uncle)
 		r.Div(blockReward, big32)
 		reward.Add(reward, r)
 	}

eth/downloader/queue.go

@@ -358,6 +358,9 @@ func (q *queue) Results(block bool) []*fetchResult {
 		closed = q.closed
 		q.lock.Unlock()
 	}
+
+	// processed=68638801
+	// processed=19025313
 	// Regardless if closed or not, we can still deliver whatever we have
 	results := q.resultCache.GetCompleted(maxResultsProcess)
 	for _, result := range results {

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/mattn/go-colorable v0.1.0
 	github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/montanaflynn/stats v0.6.3
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
 	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c

go.sum

@@ -241,6 +241,8 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/montanaflynn/stats v0.6.3 h1:F8446DrvIF5V5smZfZ8K9nrmmix0AFgevPdLruGOmzk=
+github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=