extra logging for app hash

go.mod

@@ -181,7 +181,7 @@ require (
 replace (
 	// use cosmos fork of keyring
 	github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0
-	github.com/cometbft/cometbft => github.com/osmosis-labs/cometbft v0.0.0-20240102222746-c0d08de96c15
+	github.com/cometbft/cometbft => github.com/osmosis-labs/cometbft v0.0.0-20240103055822-28da358e3146
 	// dgrijalva/jwt-go is deprecated and doesn't receive security updates.
 	// TODO: remove it: https://github.com/cosmos/cosmos-sdk/issues/13134
 	github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2

go.sum

@@ -804,8 +804,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA=
-github.com/osmosis-labs/cometbft v0.0.0-20240102222746-c0d08de96c15 h1:ok6Bp1Vp4U02zRt08hJUH+cCAVnQZGqLFUE26svAyFU=
-github.com/osmosis-labs/cometbft v0.0.0-20240102222746-c0d08de96c15/go.mod h1:Y2MMMN//O5K4YKd8ze4r9jmk4Y7h0ajqILXbH5JQFVs=
+github.com/osmosis-labs/cometbft v0.0.0-20240103055822-28da358e3146 h1:Qkshh58fZCRoHSzbyW8NJf1wBhf9bg1MPFZEuyuI9lE=
+github.com/osmosis-labs/cometbft v0.0.0-20240103055822-28da358e3146/go.mod h1:Y2MMMN//O5K4YKd8ze4r9jmk4Y7h0ajqILXbH5JQFVs=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=

server/start.go

@@ -9,6 +9,8 @@ import (
 	"net/http"
 	"os"
 	"runtime/pprof"
+	"sort"
+	"strings"
 	"time"
 
 	"github.com/cometbft/cometbft/abci/server"
@@ -22,6 +24,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 
+	"encoding/hex"
+
 	"cosmossdk.io/tools/rosetta"
 	crgserver "cosmossdk.io/tools/rosetta/lib/server"
 	"github.com/cosmos/cosmos-sdk/client"
@@ -32,6 +36,7 @@ import (
 	servergrpc "github.com/cosmos/cosmos-sdk/server/grpc"
 	"github.com/cosmos/cosmos-sdk/server/types"
 	pruningtypes "github.com/cosmos/cosmos-sdk/store/pruning/types"
+	storetypes "github.com/cosmos/cosmos-sdk/store/types"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/types/mempool"
@@ -337,12 +342,12 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App
 			return err
 		}
 
-		// Start a goroutine to listen for errors from appHashErrorsCh
+		// Start a goroutine to listen for appHash errors from consensus layer.
 		go func() {
-			for appHashError := range tmNode.ConsensusReactor().AppHashErrorsCh() {
-				// When an error is received, call returnCommitInfo
+			for appHashError := range tmNode.BCReactor().AppHashErrorsCh() {
+				// If an error is received, call returnCommitInfo
 				if appHashError.Err != nil {
-					if commitInfoErr := returnCommitInfo(app, int64(appHashError.Height)); commitInfoErr != nil {
+					if commitInfoErr := returnCommitInfo(ctx, app, int64(appHashError.Height)); commitInfoErr != nil {
 						ctx.Logger.Error("failed to return commit info", "err", commitInfoErr)
 					}
 				}
@@ -581,10 +586,41 @@ func wrapCPUProfile(ctx *Context, callback func() error) error {
 	return WaitForQuitSignals()
 }
 
-func returnCommitInfo(app types.Application, version int64) error {
+// returnCommitInfo returns the individual app hashes for every module given a version (height).
+func returnCommitInfo(ctx *Context, app types.Application, version int64) error {
 	commitInfoForHeight, err := app.CommitMultiStore().GetCommitInfo(version)
 	if err != nil {
 		return err
 	}
-	return fmt.Errorf("CommitInfo for height %d: %s", version, commitInfoForHeight.String())
+
+	// Create a new slice of StoreInfos for storing the modified hashes.
+	storeInfos := make([]storetypes.StoreInfo, len(commitInfoForHeight.StoreInfos))
+
+	for i, storeInfo := range commitInfoForHeight.StoreInfos {
+		// Convert the hash to a hexadecimal string.
+		hash := strings.ToUpper(hex.EncodeToString(storeInfo.CommitId.Hash))
+
+		// Create a new StoreInfo with the modified hash.
+		storeInfos[i] = storetypes.StoreInfo{
+			Name: storeInfo.Name,
+			CommitId: storetypes.CommitID{
+				Version: storeInfo.CommitId.Version,
+				Hash:    []byte(hash),
+			},
+		}
+	}
+
+	// Sort the storeInfos slice based on the module name.
+	sort.Slice(storeInfos, func(i, j int) bool {
+		return storeInfos[i].Name < storeInfos[j].Name
+	})
+
+	// Create a new CommitInfo with the modified StoreInfos.
+	commitInfoForHeight = &storetypes.CommitInfo{
+		Version:    commitInfoForHeight.Version,
+		StoreInfos: storeInfos,
+	}
+
+	ctx.Logger.Error("your node has app hashed. Compare each module's hashes with a node that did not app hash to determine the problematic module", "commitInfo", commitInfoForHeight.String())
+	return nil
 }