From 40eee4289ea4c30e385c618f97453e0b7f6e27c2 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Tue, 2 Jan 2024 15:36:59 -0700 Subject: [PATCH 1/3] app hash error print --- go.mod | 1 + go.sum | 4 ++-- server/mock/store.go | 4 ++++ server/start.go | 22 ++++++++++++++++++++++ store/types/store.go | 3 +++ 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b8c8ac122d25..261fb83a38a0 100644 --- a/go.mod +++ b/go.mod @@ -181,6 +181,7 @@ require ( replace ( // use cosmos fork of keyring github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0 + github.com/cometbft/cometbft => github.com/osmosis-labs/cometbft v0.0.0-20240102222746-c0d08de96c15 // dgrijalva/jwt-go is deprecated and doesn't receive security updates. // TODO: remove it: https://github.com/cosmos/cosmos-sdk/issues/13134 github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2 diff --git a/go.sum b/go.sum index f827f8cff5d1..c1d7a3479157 100644 --- a/go.sum +++ b/go.sum @@ -312,8 +312,6 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coinbase/rosetta-sdk-go/types v1.0.0 h1:jpVIwLcPoOeCR6o1tU+Xv7r5bMONNbHU7MuEHboiFuA= github.com/coinbase/rosetta-sdk-go/types v1.0.0/go.mod h1:eq7W2TMRH22GTW0N0beDnN931DW0/WOI1R2sdHNHG4c= -github.com/cometbft/cometbft v0.37.2 h1:XB0yyHGT0lwmJlFmM4+rsRnczPlHoAKFX6K8Zgc2/Jc= -github.com/cometbft/cometbft v0.37.2/go.mod h1:Y2MMMN//O5K4YKd8ze4r9jmk4Y7h0ajqILXbH5JQFVs= github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo= github.com/cometbft/cometbft-db v0.7.0/go.mod h1:yiKJIm2WKrt6x8Cyxtq9YTEcIMPcEe4XPxhgX59Fzf0= github.com/confio/ics23/go v0.9.0 h1:cWs+wdbS2KRPZezoaaj+qBleXgUk5WOQFMP3CQFGTr4= @@ -806,6 +804,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= +github.com/osmosis-labs/cometbft v0.0.0-20240102222746-c0d08de96c15 h1:ok6Bp1Vp4U02zRt08hJUH+cCAVnQZGqLFUE26svAyFU= +github.com/osmosis-labs/cometbft v0.0.0-20240102222746-c0d08de96c15/go.mod h1:Y2MMMN//O5K4YKd8ze4r9jmk4Y7h0ajqILXbH5JQFVs= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= diff --git a/server/mock/store.go b/server/mock/store.go index 192c58e5526e..45624f159c84 100644 --- a/server/mock/store.go +++ b/server/mock/store.go @@ -160,6 +160,10 @@ func (ms multiStore) LatestVersion() int64 { panic("not implemented") } +func (ms multiStore) GetCommitInfo(key int64) (*storetypes.CommitInfo, error) { + panic("not implemented") +} + var _ sdk.KVStore = kvStore{} type kvStore struct { diff --git a/server/start.go b/server/start.go index 061b736cf6a5..2080266d5c97 100644 --- a/server/start.go +++ b/server/start.go @@ -332,8 +332,22 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App } if err := tmNode.Start(); err != nil { + // add err logic for retrieving commits + // app.CommitMultiStore().GetCommitInfo() return err } + + // Start a goroutine to listen for errors from appHashErrorsCh + go func() { + for appHashError := range tmNode.ConsensusReactor().AppHashErrorsCh() { + // When an error is received, call returnCommitInfo + if appHashError.Err != nil { + if commitInfoErr := returnCommitInfo(app, int64(appHashError.Height)); commitInfoErr != nil { + ctx.Logger.Error("failed to return commit info", "err", commitInfoErr) + } + } + } + }() } // Add the tx service to the gRPC router. We only need to register this @@ -566,3 +580,11 @@ func wrapCPUProfile(ctx *Context, callback func() error) error { return WaitForQuitSignals() } + +func returnCommitInfo(app types.Application, version int64) error { + commitInfoForHeight, err := app.CommitMultiStore().GetCommitInfo(version) + if err != nil { + return err + } + return fmt.Errorf("CommitInfo for height %d: %s", version, commitInfoForHeight.String()) +} diff --git a/store/types/store.go b/store/types/store.go index 992173195661..01a71306d068 100644 --- a/store/types/store.go +++ b/store/types/store.go @@ -154,6 +154,9 @@ type CommitMultiStore interface { // Panics on a nil key. GetCommitKVStore(key StoreKey) CommitKVStore + // Panics on a nil version. + GetCommitInfo(ver int64) (*CommitInfo, error) + // Load the latest persisted version. Called once after all calls to // Mount*Store() are complete. LoadLatestVersion() error From 436d5244ba5f6e4b02c47bd6260baa16bf3f41df Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Tue, 2 Jan 2024 23:07:14 -0700 Subject: [PATCH 2/3] extra logging for app hash --- go.mod | 2 +- go.sum | 4 ++-- server/start.go | 48 ++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 261fb83a38a0..b77a2fe25ac6 100644 --- a/go.mod +++ b/go.mod @@ -181,7 +181,7 @@ require ( replace ( // use cosmos fork of keyring github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0 - github.com/cometbft/cometbft => github.com/osmosis-labs/cometbft v0.0.0-20240102222746-c0d08de96c15 + github.com/cometbft/cometbft => github.com/osmosis-labs/cometbft v0.0.0-20240103055822-28da358e3146 // dgrijalva/jwt-go is deprecated and doesn't receive security updates. // TODO: remove it: https://github.com/cosmos/cosmos-sdk/issues/13134 github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2 diff --git a/go.sum b/go.sum index c1d7a3479157..6b8df5b084ab 100644 --- a/go.sum +++ b/go.sum @@ -804,8 +804,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= -github.com/osmosis-labs/cometbft v0.0.0-20240102222746-c0d08de96c15 h1:ok6Bp1Vp4U02zRt08hJUH+cCAVnQZGqLFUE26svAyFU= -github.com/osmosis-labs/cometbft v0.0.0-20240102222746-c0d08de96c15/go.mod h1:Y2MMMN//O5K4YKd8ze4r9jmk4Y7h0ajqILXbH5JQFVs= +github.com/osmosis-labs/cometbft v0.0.0-20240103055822-28da358e3146 h1:Qkshh58fZCRoHSzbyW8NJf1wBhf9bg1MPFZEuyuI9lE= +github.com/osmosis-labs/cometbft v0.0.0-20240103055822-28da358e3146/go.mod h1:Y2MMMN//O5K4YKd8ze4r9jmk4Y7h0ajqILXbH5JQFVs= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= diff --git a/server/start.go b/server/start.go index 2080266d5c97..7db8de42b91b 100644 --- a/server/start.go +++ b/server/start.go @@ -9,6 +9,8 @@ import ( "net/http" "os" "runtime/pprof" + "sort" + "strings" "time" "github.com/cometbft/cometbft/abci/server" @@ -22,6 +24,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "encoding/hex" + "cosmossdk.io/tools/rosetta" crgserver "cosmossdk.io/tools/rosetta/lib/server" "github.com/cosmos/cosmos-sdk/client" @@ -32,6 +36,7 @@ import ( servergrpc "github.com/cosmos/cosmos-sdk/server/grpc" "github.com/cosmos/cosmos-sdk/server/types" pruningtypes "github.com/cosmos/cosmos-sdk/store/pruning/types" + storetypes "github.com/cosmos/cosmos-sdk/store/types" "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/types/mempool" @@ -337,12 +342,12 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return err } - // Start a goroutine to listen for errors from appHashErrorsCh + // Start a goroutine to listen for appHash errors from consensus layer. go func() { - for appHashError := range tmNode.ConsensusReactor().AppHashErrorsCh() { - // When an error is received, call returnCommitInfo + for appHashError := range tmNode.BCReactor().AppHashErrorsCh() { + // If an error is received, call returnCommitInfo if appHashError.Err != nil { - if commitInfoErr := returnCommitInfo(app, int64(appHashError.Height)); commitInfoErr != nil { + if commitInfoErr := returnCommitInfo(ctx, app, int64(appHashError.Height)); commitInfoErr != nil { ctx.Logger.Error("failed to return commit info", "err", commitInfoErr) } } @@ -581,10 +586,41 @@ func wrapCPUProfile(ctx *Context, callback func() error) error { return WaitForQuitSignals() } -func returnCommitInfo(app types.Application, version int64) error { +// returnCommitInfo returns the individual app hashes for every module given a version (height). +func returnCommitInfo(ctx *Context, app types.Application, version int64) error { commitInfoForHeight, err := app.CommitMultiStore().GetCommitInfo(version) if err != nil { return err } - return fmt.Errorf("CommitInfo for height %d: %s", version, commitInfoForHeight.String()) + + // Create a new slice of StoreInfos for storing the modified hashes. + storeInfos := make([]storetypes.StoreInfo, len(commitInfoForHeight.StoreInfos)) + + for i, storeInfo := range commitInfoForHeight.StoreInfos { + // Convert the hash to a hexadecimal string. + hash := strings.ToUpper(hex.EncodeToString(storeInfo.CommitId.Hash)) + + // Create a new StoreInfo with the modified hash. + storeInfos[i] = storetypes.StoreInfo{ + Name: storeInfo.Name, + CommitId: storetypes.CommitID{ + Version: storeInfo.CommitId.Version, + Hash: []byte(hash), + }, + } + } + + // Sort the storeInfos slice based on the module name. + sort.Slice(storeInfos, func(i, j int) bool { + return storeInfos[i].Name < storeInfos[j].Name + }) + + // Create a new CommitInfo with the modified StoreInfos. + commitInfoForHeight = &storetypes.CommitInfo{ + Version: commitInfoForHeight.Version, + StoreInfos: storeInfos, + } + + ctx.Logger.Error("your node has app hashed. Compare each module's hashes with a node that did not app hash to determine the problematic module", "commitInfo", commitInfoForHeight.String()) + return nil } From 7944aa15b0fcd6dca7da14c1b5ae7190caaffd2b Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Sat, 6 Jan 2024 12:43:12 -0700 Subject: [PATCH 3/3] remove old comment --- server/start.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/start.go b/server/start.go index 7db8de42b91b..374dc5e98572 100644 --- a/server/start.go +++ b/server/start.go @@ -337,8 +337,6 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App } if err := tmNode.Start(); err != nil { - // add err logic for retrieving commits - // app.CommitMultiStore().GetCommitInfo() return err }