From 62139b62295c645ce5f5065b8f8f0629e33120f1 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Sat, 27 Jan 2024 21:36:46 -0700 Subject: [PATCH 01/11] add testnetify logic --- server/start.go | 347 ++++++++++++++++++++++++++++++++++++++------ server/types/app.go | 6 + server/util.go | 7 + 3 files changed, 314 insertions(+), 46 deletions(-) diff --git a/server/start.go b/server/start.go index 374dc5e98572..6391b5099f07 100644 --- a/server/start.go +++ b/server/start.go @@ -3,6 +3,7 @@ package server // DONTCOVER import ( + "bufio" "errors" "fmt" "net" @@ -13,6 +14,7 @@ import ( "strings" "time" + db "github.com/cometbft/cometbft-db" "github.com/cometbft/cometbft/abci/server" tcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" "github.com/cometbft/cometbft/node" @@ -20,6 +22,7 @@ import ( pvm "github.com/cometbft/cometbft/privval" "github.com/cometbft/cometbft/proxy" "github.com/cometbft/cometbft/rpc/client/local" + "github.com/cometbft/cometbft/store" "github.com/spf13/cobra" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -28,6 +31,12 @@ import ( "cosmossdk.io/tools/rosetta" crgserver "cosmossdk.io/tools/rosetta/lib/server" + cfg "github.com/cometbft/cometbft/config" + cmtjson "github.com/cometbft/cometbft/libs/json" + cmtstate "github.com/cometbft/cometbft/proto/tendermint/state" + cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" + sm "github.com/cometbft/cometbft/state" + cmttypes "github.com/cometbft/cometbft/types" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" @@ -40,6 +49,7 @@ import ( "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/types/mempool" + "github.com/cosmos/cosmos-sdk/x/genutil" ) const ( @@ -153,7 +163,7 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. // amino is needed here for backwards compatibility of REST routes err = wrapCPUProfile(serverCtx, func() error { - return startInProcess(serverCtx, clientCtx, appCreator) + return startInProcess(serverCtx, clientCtx, appCreator, nil, "", "") }) errCode, ok := err.(ErrorCode) if !ok { @@ -165,49 +175,7 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. }, } - cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") - cmd.Flags().Bool(flagWithTendermint, true, "Run abci app embedded in-process with tendermint") - cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address") - cmd.Flags().String(flagTransport, "socket", "Transport protocol: socket, grpc") - cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file") - cmd.Flags().String(FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)") - cmd.Flags().IntSlice(FlagUnsafeSkipUpgrades, []int{}, "Skip a set of upgrade heights to continue the old binary") - cmd.Flags().Uint64(FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node") - cmd.Flags().Uint64(FlagHaltTime, 0, "Minimum block time (in Unix seconds) at which to gracefully halt the chain and shutdown the node") - cmd.Flags().Bool(FlagInterBlockCache, true, "Enable inter-block caching") - cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file") - cmd.Flags().Bool(FlagTrace, false, "Provide full stack traces for errors in ABCI Log") - cmd.Flags().String(FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") - cmd.Flags().Uint64(FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") - cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')") - cmd.Flags().Uint(FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks") - cmd.Flags().Uint64(FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune Tendermint blocks") - - cmd.Flags().Bool(FlagAPIEnable, false, "Define if the API server should be enabled") - cmd.Flags().Bool(FlagAPISwagger, false, "Define if swagger documentation should automatically be registered (Note: the API must also be enabled)") - cmd.Flags().String(FlagAPIAddress, serverconfig.DefaultAPIAddress, "the API server address to listen on") - cmd.Flags().Uint(FlagAPIMaxOpenConnections, 1000, "Define the number of maximum open connections") - cmd.Flags().Uint(FlagRPCReadTimeout, 10, "Define the Tendermint RPC read timeout (in seconds)") - cmd.Flags().Uint(FlagRPCWriteTimeout, 0, "Define the Tendermint RPC write timeout (in seconds)") - cmd.Flags().Uint(FlagRPCMaxBodyBytes, 1000000, "Define the Tendermint maximum request body (in bytes)") - cmd.Flags().Bool(FlagAPIEnableUnsafeCORS, false, "Define if CORS should be enabled (unsafe - use it at your own risk)") - - cmd.Flags().Bool(flagGRPCOnly, false, "Start the node in gRPC query only mode (no Tendermint process is started)") - cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") - cmd.Flags().String(flagGRPCAddress, serverconfig.DefaultGRPCAddress, "the gRPC server address to listen on") - - cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled)") - cmd.Flags().String(flagGRPCWebAddress, serverconfig.DefaultGRPCWebAddress, "The gRPC-Web server address to listen on") - - cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") - cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep") - - cmd.Flags().Bool(FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree") - - cmd.Flags().Int(FlagMempoolMaxTxs, mempool.DefaultMaxTx, "Sets MaxTx value for the app-side mempool") - - // add support for all Tendermint-specific command line options - tcmd.AddNodeFlags(cmd) + addStartNodeFlags(cmd, defaultNodeHome) return cmd } @@ -268,7 +236,11 @@ func startStandAlone(ctx *Context, appCreator types.AppCreator) error { return WaitForQuitSignals() } -func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.AppCreator) error { +func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.AppCreator, testnetAppCreator types.TestnetAppCreator, newChainID, newOperatorAddress string) error { + if appCreator != nil && testnetAppCreator != nil { + return errors.New("cannot provide both appCreator and testnetAppCreator") + } + cfg := ctx.Config home := cfg.RootDir @@ -303,7 +275,16 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return err } - app := appCreator(ctx.Logger, db, traceWriter, ctx.Viper) + var app types.Application + + if appCreator != nil { + app = appCreator(ctx.Logger, db, traceWriter, ctx.Viper) + } else if testnetAppCreator != nil { + app, err = testnetify(ctx, cfg, home, newChainID, newOperatorAddress, testnetAppCreator, db) + if err != nil { + return err + } + } nodeKey, err := p2p.LoadOrGenNodeKey(cfg.NodeKeyFile()) if err != nil { @@ -622,3 +603,277 @@ func returnCommitInfo(ctx *Context, app types.Application, version int64) error ctx.Logger.Error("your node has app hashed. Compare each module's hashes with a node that did not app hash to determine the problematic module", "commitInfo", commitInfoForHeight.String()) return nil } + +// InPlaceTestnetCreator utilizes the provided chainID and operatorAddress as well as the local private validator key to +// control the network represented in the data folder. This is useful to create testnets nearly identical to your +// mainnet environment. +func InPlaceTestnetCreator(testnetAppCreator types.TestnetAppCreator, defaultNodeHome string) *cobra.Command { + cmd := &cobra.Command{ + Use: "in-place-testnet [newChainID] [newOperatorAddress]", + Short: "Create and start a testnet from current local state", + Long: `Create and start a testnet from current local state. +After utilizing this command the network will start. If the network is stopped, +the normal "start" command should be used. Re-using this command on state that +has already been run on could result in unexpected behavior. + +Additionally, the first block may take up to one minute to be committed, depending +on how old the block is. For instance, if a snapshot was taken weeks ago and we want +to turn this into a testnet, it is possible lots of pending state needs to be commited +(expiring locks, etc.). It is recommended that you should wait at the very least for this +block to be commited before stopping the daemon. Then, as explained above, should use the +normal "start" command after that to restart the daemon. +`, + Example: "in-place-testnet localosmosis osmo12smx2wdlyttvyzvzg54y2vnqwq2qjateuf7thj", + Args: cobra.ExactArgs(2), + PreRunE: func(cmd *cobra.Command, _ []string) error { + serverCtx := GetServerContextFromCmd(cmd) + + if err := serverCtx.Viper.BindPFlags(cmd.Flags()); err != nil { + return err + } + + _, err := GetPruningOptionsFromFlags(serverCtx.Viper) + return err + }, + RunE: func(cmd *cobra.Command, args []string) error { + serverCtx := GetServerContextFromCmd(cmd) + clientCtx, err := client.GetClientQueryContext(cmd) + if err != nil { + return err + } + + newChainID := args[0] + newOperatorAddress := args[1] + + // Confirmation prompt to prevent accidental modification of state. + reader := bufio.NewReader(os.Stdin) + fmt.Println("This operation will modify state in your data folder and cannot be undone. Do you want to continue? (y/n)") + text, _ := reader.ReadString('\n') + response := strings.TrimSpace(strings.ToLower(text)) + if response != "y" && response != "yes" { + fmt.Println("Operation cancelled.") + return nil + } + + return startInProcess(serverCtx, clientCtx, nil, testnetAppCreator, newChainID, newOperatorAddress) + }, + } + + addStartNodeFlags(cmd, defaultNodeHome) + return cmd +} + +// testnetify modifies both state and blockStore, allowing the provided operator address and local validator key to control the network +// that the state in the data folder represents. The chainID of the local genesis file is modified to match the provided chainID. +func testnetify(ctx *Context, config *cfg.Config, home, newChainID, newOperatorAddress string, testnetAppCreator types.TestnetAppCreator, db db.DB) (types.Application, error) { + traceWriterFile := ctx.Viper.GetString(flagTraceStore) + traceWriter, err := openTraceWriter(traceWriterFile) + if err != nil { + return nil, err + } + + genDocProvider := node.DefaultGenesisDocProviderFunc(config) + + // Initialize blockStore and stateDB. + blockStoreDB, err := node.DefaultDBProvider(&node.DBContext{ID: "blockstore", Config: config}) + if err != nil { + return nil, err + } + blockStore := store.NewBlockStore(blockStoreDB) + + stateDB, err := node.DefaultDBProvider(&node.DBContext{ID: "state", Config: config}) + if err != nil { + return nil, err + } + + defer blockStore.Close() + defer stateDB.Close() + + privValidator := pvm.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) + userPubKey, err := privValidator.GetPubKey() + if err != nil { + return nil, err + } + validatorAddress := userPubKey.Address() + if err != nil { + return nil, err + } + + stateStore := sm.NewStore(stateDB, sm.StoreOptions{ + DiscardABCIResponses: config.Storage.DiscardABCIResponses, + }) + + state, genDoc, err := node.LoadStateFromDBOrGenesisDocProvider(stateDB, genDocProvider) + if err != nil { + return nil, err + } + + genDoc.ChainID = newChainID + genFilePath := config.GenesisFile() + if err = genutil.ExportGenesisFile(genDoc, genFilePath); err != nil { + return nil, err + } + + // There are times when a user stops their node between commits, resulting in a mismatch between the + // blockStore and state. For convenience, we just discard the uncommited blockStore block and operate on + // the lastBlockHeight in state. + if blockStore.Height() != state.LastBlockHeight { + blockStore.DeleteLatestBlock() + } + + block := blockStore.LoadBlock(blockStore.Height()) + + block.ChainID = newChainID + state.ChainID = newChainID + + block.LastBlockID = state.LastBlockID + block.LastCommit.BlockID = state.LastBlockID + + // Create a vote from our validator + vote := cmttypes.Vote{ + Type: cmtproto.PrecommitType, + Height: state.LastBlockHeight, + Round: 0, + BlockID: state.LastBlockID, + Timestamp: time.Now(), + ValidatorAddress: validatorAddress, + ValidatorIndex: 0, + Signature: []byte{}, + } + + // Sign the vote, and copy the proto changes from the act of signing to the vote itself + voteProto := vote.ToProto() + err = privValidator.SignVote(newChainID, voteProto) + if err != nil { + return nil, err + } + vote.Signature = voteProto.Signature + vote.Timestamp = voteProto.Timestamp + + // Modify the block's lastCommit to be signed only by our validator + block.LastCommit.Signatures[0].ValidatorAddress = validatorAddress + block.LastCommit.Signatures[0].Signature = vote.Signature + block.LastCommit.Signatures = []cmttypes.CommitSig{block.LastCommit.Signatures[0]} + + // Load the seenCommit of the lastBlockHeight and modify it to be signed from our validator + seenCommit := blockStore.LoadSeenCommit(state.LastBlockHeight) + seenCommit.BlockID = state.LastBlockID + seenCommit.Round = vote.Round + seenCommit.Signatures[0].Signature = vote.Signature + seenCommit.Signatures[0].ValidatorAddress = validatorAddress + seenCommit.Signatures[0].Timestamp = vote.Timestamp + seenCommit.Signatures = []cmttypes.CommitSig{seenCommit.Signatures[0]} + blockStore.SaveSeenCommit(state.LastBlockHeight, seenCommit) + + // Create ValidatorSet struct containing just our valdiator. + newVal := &cmttypes.Validator{ + Address: validatorAddress, + PubKey: userPubKey, + VotingPower: 900000000000000, + } + newValSet := &cmttypes.ValidatorSet{ + Validators: []*cmttypes.Validator{newVal}, + Proposer: newVal, + } + + // Replace all valSets in state to be the valSet with just our validator. + state.Validators = newValSet + state.LastValidators = newValSet + state.NextValidators = newValSet + state.LastHeightValidatorsChanged = blockStore.Height() + + stateStore.Save(state) + + // Create a ValidatorsInfo struct to store in stateDB. + valSet, err := state.Validators.ToProto() + if err != nil { + return nil, err + } + valInfo := &cmtstate.ValidatorsInfo{ + ValidatorSet: valSet, + LastHeightChanged: state.LastBlockHeight, + } + buf, err := valInfo.Marshal() + if err != nil { + return nil, err + } + + // Modfiy Validators stateDB entry. + err = stateDB.Set([]byte(fmt.Sprintf("validatorsKey:%v", blockStore.Height())), buf) + if err != nil { + return nil, err + } + + // Modify LastValidators stateDB entry. + err = stateDB.Set([]byte(fmt.Sprintf("validatorsKey:%v", blockStore.Height()-1)), buf) + if err != nil { + return nil, err + } + + // Modify NextValidators stateDB entry. + err = stateDB.Set([]byte(fmt.Sprintf("validatorsKey:%v", blockStore.Height()+1)), buf) + if err != nil { + return nil, err + } + + b, err := cmtjson.Marshal(genDoc) + if err != nil { + return nil, err + } + if err := stateDB.SetSync([]byte("genesisDoc"), b); err != nil { + return nil, err + } + + // testnetAppCreator makes any application side changes that must be made due to the above modifications. + // Also, it makes any optional application side changes to make running the testnet easier (voting times, fund accounts, etc). + testnetApp := testnetAppCreator(ctx.Logger, db, traceWriter, validatorAddress, userPubKey, newOperatorAddress, ctx.Viper) + + return testnetApp, err +} + +// addStartNodeFlags should be added to any CLI commands that start the network. +func addStartNodeFlags(cmd *cobra.Command, defaultNodeHome string) { + cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") + cmd.Flags().Bool(flagWithTendermint, true, "Run abci app embedded in-process with tendermint") + cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address") + cmd.Flags().String(flagTransport, "socket", "Transport protocol: socket, grpc") + cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file") + cmd.Flags().String(FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)") + cmd.Flags().IntSlice(FlagUnsafeSkipUpgrades, []int{}, "Skip a set of upgrade heights to continue the old binary") + cmd.Flags().Uint64(FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node") + cmd.Flags().Uint64(FlagHaltTime, 0, "Minimum block time (in Unix seconds) at which to gracefully halt the chain and shutdown the node") + cmd.Flags().Bool(FlagInterBlockCache, true, "Enable inter-block caching") + cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file") + cmd.Flags().Bool(FlagTrace, false, "Provide full stack traces for errors in ABCI Log") + cmd.Flags().String(FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") + cmd.Flags().Uint64(FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint(FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks") + cmd.Flags().Uint64(FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune Tendermint blocks") + + cmd.Flags().Bool(FlagAPIEnable, false, "Define if the API server should be enabled") + cmd.Flags().Bool(FlagAPISwagger, false, "Define if swagger documentation should automatically be registered (Note: the API must also be enabled)") + cmd.Flags().String(FlagAPIAddress, serverconfig.DefaultAPIAddress, "the API server address to listen on") + cmd.Flags().Uint(FlagAPIMaxOpenConnections, 1000, "Define the number of maximum open connections") + cmd.Flags().Uint(FlagRPCReadTimeout, 10, "Define the Tendermint RPC read timeout (in seconds)") + cmd.Flags().Uint(FlagRPCWriteTimeout, 0, "Define the Tendermint RPC write timeout (in seconds)") + cmd.Flags().Uint(FlagRPCMaxBodyBytes, 1000000, "Define the Tendermint maximum request body (in bytes)") + cmd.Flags().Bool(FlagAPIEnableUnsafeCORS, false, "Define if CORS should be enabled (unsafe - use it at your own risk)") + + cmd.Flags().Bool(flagGRPCOnly, false, "Start the node in gRPC query only mode (no Tendermint process is started)") + cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") + cmd.Flags().String(flagGRPCAddress, serverconfig.DefaultGRPCAddress, "the gRPC server address to listen on") + + cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled)") + cmd.Flags().String(flagGRPCWebAddress, serverconfig.DefaultGRPCWebAddress, "The gRPC-Web server address to listen on") + + cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") + cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep") + + cmd.Flags().Bool(FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree") + + cmd.Flags().Int(FlagMempoolMaxTxs, mempool.DefaultMaxTx, "Sets MaxTx value for the app-side mempool") + + // add support for all Tendermint-specific command line options + tcmd.AddNodeFlags(cmd) +} diff --git a/server/types/app.go b/server/types/app.go index eb8d4e78c83e..73206864721b 100644 --- a/server/types/app.go +++ b/server/types/app.go @@ -7,6 +7,8 @@ import ( dbm "github.com/cometbft/cometbft-db" abci "github.com/cometbft/cometbft/abci/types" + "github.com/cometbft/cometbft/crypto" + "github.com/cometbft/cometbft/libs/bytes" "github.com/cometbft/cometbft/libs/log" tmproto "github.com/cometbft/cometbft/proto/tendermint/types" tmtypes "github.com/cometbft/cometbft/types" @@ -72,6 +74,10 @@ type ( // application using various configurations. AppCreator func(log.Logger, dbm.DB, io.Writer, AppOptions) Application + // TestnetAppCreator serves a similar purpose as AppCreator, however is used for + // passing more information to the application layer needed to create a testnet from existing state. + TestnetAppCreator func(log.Logger, dbm.DB, io.Writer, bytes.HexBytes, crypto.PubKey, string, AppOptions) Application + // ModuleInitFlags takes a start command and adds modules specific init flags. ModuleInitFlags func(startCmd *cobra.Command) diff --git a/server/util.go b/server/util.go index 9a6ae40f786e..0127fd1d2092 100644 --- a/server/util.go +++ b/server/util.go @@ -324,6 +324,13 @@ func AddCommands(rootCmd *cobra.Command, defaultNodeHome string, appCreator type ) } +// AddTestnetCreatorCommand allows chains to create a testnet from the state existing in their node's data directory. +func AddTestnetCreatorCommand(rootCmd *cobra.Command, defaultNodeHome string, appCreator types.TestnetAppCreator, addStartFlags types.ModuleInitFlags) { + testnetCreateCmd := InPlaceTestnetCreator(appCreator, defaultNodeHome) + addStartFlags(testnetCreateCmd) + rootCmd.AddCommand(testnetCreateCmd) +} + // https://stackoverflow.com/questions/23558425/how-do-i-get-the-local-ip-address-in-go // TODO there must be a better way to get external IP func ExternalIP() (string, error) { From 00992da035262e0e6d5a7287980eca40d6437a06 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Sat, 27 Jan 2024 21:42:43 -0700 Subject: [PATCH 02/11] clarify explanation --- server/start.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/start.go b/server/start.go index 6391b5099f07..c8acc01ac144 100644 --- a/server/start.go +++ b/server/start.go @@ -614,14 +614,13 @@ func InPlaceTestnetCreator(testnetAppCreator types.TestnetAppCreator, defaultNod Long: `Create and start a testnet from current local state. After utilizing this command the network will start. If the network is stopped, the normal "start" command should be used. Re-using this command on state that -has already been run on could result in unexpected behavior. +has already been modified by this command could result in unexpected behavior. Additionally, the first block may take up to one minute to be committed, depending on how old the block is. For instance, if a snapshot was taken weeks ago and we want to turn this into a testnet, it is possible lots of pending state needs to be commited -(expiring locks, etc.). It is recommended that you should wait at the very least for this -block to be commited before stopping the daemon. Then, as explained above, should use the -normal "start" command after that to restart the daemon. +(expiring locks, etc.). It is recommended that you should wait for this block to be commited +before stopping the daemon. `, Example: "in-place-testnet localosmosis osmo12smx2wdlyttvyzvzg54y2vnqwq2qjateuf7thj", Args: cobra.ExactArgs(2), From f7601f76ba26daa9479f7b0f39fe0d746f9e0ec3 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Sat, 27 Jan 2024 21:49:17 -0700 Subject: [PATCH 03/11] handle errors --- server/start.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/server/start.go b/server/start.go index c8acc01ac144..8758c7838d28 100644 --- a/server/start.go +++ b/server/start.go @@ -717,7 +717,10 @@ func testnetify(ctx *Context, config *cfg.Config, home, newChainID, newOperatorA // blockStore and state. For convenience, we just discard the uncommited blockStore block and operate on // the lastBlockHeight in state. if blockStore.Height() != state.LastBlockHeight { - blockStore.DeleteLatestBlock() + err = blockStore.DeleteLatestBlock() + if err != nil { + return nil, err + } } block := blockStore.LoadBlock(blockStore.Height()) @@ -762,7 +765,10 @@ func testnetify(ctx *Context, config *cfg.Config, home, newChainID, newOperatorA seenCommit.Signatures[0].ValidatorAddress = validatorAddress seenCommit.Signatures[0].Timestamp = vote.Timestamp seenCommit.Signatures = []cmttypes.CommitSig{seenCommit.Signatures[0]} - blockStore.SaveSeenCommit(state.LastBlockHeight, seenCommit) + err = blockStore.SaveSeenCommit(state.LastBlockHeight, seenCommit) + if err != nil { + return nil, err + } // Create ValidatorSet struct containing just our valdiator. newVal := &cmttypes.Validator{ @@ -781,7 +787,10 @@ func testnetify(ctx *Context, config *cfg.Config, home, newChainID, newOperatorA state.NextValidators = newValSet state.LastHeightValidatorsChanged = blockStore.Height() - stateStore.Save(state) + err = stateStore.Save(state) + if err != nil { + return nil, err + } // Create a ValidatorsInfo struct to store in stateDB. valSet, err := state.Validators.ToProto() From 7e9bc41b4266e21aa96caf3c390676df2dc7e101 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Sun, 28 Jan 2024 22:52:54 -0700 Subject: [PATCH 04/11] remove testnet app creator --- server/start.go | 50 ++++++++++++++++++++++++++++++++------------- server/types/app.go | 6 ------ server/util.go | 2 +- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/server/start.go b/server/start.go index 8758c7838d28..6f4c72b8a4da 100644 --- a/server/start.go +++ b/server/start.go @@ -99,6 +99,13 @@ const ( // mempool flags FlagMempoolMaxTxs = "mempool.max-txs" + + // testnet keys + KeyIsTestnet = "is-testnet" + KeyNewChainID = "new-chain-ID" + KeyNewOpAddr = "new-operator-addr" + KeyNewValAddr = "new-validator-addr" + KeyUserPubKey = "user-pub-key" ) // StartCmd runs the service passed in, either stand-alone or in-process with @@ -163,7 +170,7 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. // amino is needed here for backwards compatibility of REST routes err = wrapCPUProfile(serverCtx, func() error { - return startInProcess(serverCtx, clientCtx, appCreator, nil, "", "") + return startInProcess(serverCtx, clientCtx, appCreator) }) errCode, ok := err.(ErrorCode) if !ok { @@ -236,11 +243,7 @@ func startStandAlone(ctx *Context, appCreator types.AppCreator) error { return WaitForQuitSignals() } -func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.AppCreator, testnetAppCreator types.TestnetAppCreator, newChainID, newOperatorAddress string) error { - if appCreator != nil && testnetAppCreator != nil { - return errors.New("cannot provide both appCreator and testnetAppCreator") - } - +func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.AppCreator) error { cfg := ctx.Config home := cfg.RootDir @@ -275,15 +278,29 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return err } + isTestnet, ok := ctx.Viper.Get(KeyIsTestnet).(bool) + if !ok { + isTestnet = false + } + var app types.Application - if appCreator != nil { - app = appCreator(ctx.Logger, db, traceWriter, ctx.Viper) - } else if testnetAppCreator != nil { - app, err = testnetify(ctx, cfg, home, newChainID, newOperatorAddress, testnetAppCreator, db) + if isTestnet { + newChainID, ok := ctx.Viper.Get(KeyNewChainID).(string) + if !ok { + return fmt.Errorf("expected string for key %s", KeyNewChainID) + } + newOperatorAddress, ok := ctx.Viper.Get(KeyNewOpAddr).(string) + if !ok { + return fmt.Errorf("expected string for key %s", KeyNewOpAddr) + } + + app, err = testnetify(ctx, cfg, home, newChainID, newOperatorAddress, appCreator, db) if err != nil { return err } + } else { + app = appCreator(ctx.Logger, db, traceWriter, ctx.Viper) } nodeKey, err := p2p.LoadOrGenNodeKey(cfg.NodeKeyFile()) @@ -607,7 +624,7 @@ func returnCommitInfo(ctx *Context, app types.Application, version int64) error // InPlaceTestnetCreator utilizes the provided chainID and operatorAddress as well as the local private validator key to // control the network represented in the data folder. This is useful to create testnets nearly identical to your // mainnet environment. -func InPlaceTestnetCreator(testnetAppCreator types.TestnetAppCreator, defaultNodeHome string) *cobra.Command { +func InPlaceTestnetCreator(testnetAppCreator types.AppCreator, defaultNodeHome string) *cobra.Command { cmd := &cobra.Command{ Use: "in-place-testnet [newChainID] [newOperatorAddress]", Short: "Create and start a testnet from current local state", @@ -653,8 +670,11 @@ before stopping the daemon. fmt.Println("Operation cancelled.") return nil } + serverCtx.Viper.Set(KeyIsTestnet, true) + serverCtx.Viper.Set(KeyNewChainID, newChainID) + serverCtx.Viper.Set(KeyNewOpAddr, newOperatorAddress) - return startInProcess(serverCtx, clientCtx, nil, testnetAppCreator, newChainID, newOperatorAddress) + return startInProcess(serverCtx, clientCtx, testnetAppCreator) }, } @@ -664,7 +684,7 @@ before stopping the daemon. // testnetify modifies both state and blockStore, allowing the provided operator address and local validator key to control the network // that the state in the data folder represents. The chainID of the local genesis file is modified to match the provided chainID. -func testnetify(ctx *Context, config *cfg.Config, home, newChainID, newOperatorAddress string, testnetAppCreator types.TestnetAppCreator, db db.DB) (types.Application, error) { +func testnetify(ctx *Context, config *cfg.Config, home, newChainID, newOperatorAddress string, testnetAppCreator types.AppCreator, db db.DB) (types.Application, error) { traceWriterFile := ctx.Viper.GetString(flagTraceStore) traceWriter, err := openTraceWriter(traceWriterFile) if err != nil { @@ -834,7 +854,9 @@ func testnetify(ctx *Context, config *cfg.Config, home, newChainID, newOperatorA // testnetAppCreator makes any application side changes that must be made due to the above modifications. // Also, it makes any optional application side changes to make running the testnet easier (voting times, fund accounts, etc). - testnetApp := testnetAppCreator(ctx.Logger, db, traceWriter, validatorAddress, userPubKey, newOperatorAddress, ctx.Viper) + ctx.Viper.Set(KeyNewValAddr, validatorAddress) + ctx.Viper.Set(KeyUserPubKey, userPubKey) + testnetApp := testnetAppCreator(ctx.Logger, db, traceWriter, ctx.Viper) return testnetApp, err } diff --git a/server/types/app.go b/server/types/app.go index 73206864721b..eb8d4e78c83e 100644 --- a/server/types/app.go +++ b/server/types/app.go @@ -7,8 +7,6 @@ import ( dbm "github.com/cometbft/cometbft-db" abci "github.com/cometbft/cometbft/abci/types" - "github.com/cometbft/cometbft/crypto" - "github.com/cometbft/cometbft/libs/bytes" "github.com/cometbft/cometbft/libs/log" tmproto "github.com/cometbft/cometbft/proto/tendermint/types" tmtypes "github.com/cometbft/cometbft/types" @@ -74,10 +72,6 @@ type ( // application using various configurations. AppCreator func(log.Logger, dbm.DB, io.Writer, AppOptions) Application - // TestnetAppCreator serves a similar purpose as AppCreator, however is used for - // passing more information to the application layer needed to create a testnet from existing state. - TestnetAppCreator func(log.Logger, dbm.DB, io.Writer, bytes.HexBytes, crypto.PubKey, string, AppOptions) Application - // ModuleInitFlags takes a start command and adds modules specific init flags. ModuleInitFlags func(startCmd *cobra.Command) diff --git a/server/util.go b/server/util.go index 0127fd1d2092..8b41d194aa42 100644 --- a/server/util.go +++ b/server/util.go @@ -325,7 +325,7 @@ func AddCommands(rootCmd *cobra.Command, defaultNodeHome string, appCreator type } // AddTestnetCreatorCommand allows chains to create a testnet from the state existing in their node's data directory. -func AddTestnetCreatorCommand(rootCmd *cobra.Command, defaultNodeHome string, appCreator types.TestnetAppCreator, addStartFlags types.ModuleInitFlags) { +func AddTestnetCreatorCommand(rootCmd *cobra.Command, defaultNodeHome string, appCreator types.AppCreator, addStartFlags types.ModuleInitFlags) { testnetCreateCmd := InPlaceTestnetCreator(appCreator, defaultNodeHome) addStartFlags(testnetCreateCmd) rootCmd.AddCommand(testnetCreateCmd) From 330576777880897e04ca77d75dc89f6880534268 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Mon, 29 Jan 2024 11:14:20 -0700 Subject: [PATCH 05/11] simplify some logic --- server/start.go | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/server/start.go b/server/start.go index 6f4c72b8a4da..d894f75e0ee8 100644 --- a/server/start.go +++ b/server/start.go @@ -31,7 +31,6 @@ import ( "cosmossdk.io/tools/rosetta" crgserver "cosmossdk.io/tools/rosetta/lib/server" - cfg "github.com/cometbft/cometbft/config" cmtjson "github.com/cometbft/cometbft/libs/json" cmtstate "github.com/cometbft/cometbft/proto/tendermint/state" cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" @@ -278,24 +277,10 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return err } - isTestnet, ok := ctx.Viper.Get(KeyIsTestnet).(bool) - if !ok { - isTestnet = false - } - var app types.Application - if isTestnet { - newChainID, ok := ctx.Viper.Get(KeyNewChainID).(string) - if !ok { - return fmt.Errorf("expected string for key %s", KeyNewChainID) - } - newOperatorAddress, ok := ctx.Viper.Get(KeyNewOpAddr).(string) - if !ok { - return fmt.Errorf("expected string for key %s", KeyNewOpAddr) - } - - app, err = testnetify(ctx, cfg, home, newChainID, newOperatorAddress, appCreator, db) + if isTestnet, ok := ctx.Viper.Get(KeyIsTestnet).(bool); ok && isTestnet { + app, err = testnetify(ctx, home, appCreator, db) if err != nil { return err } @@ -684,7 +669,14 @@ before stopping the daemon. // testnetify modifies both state and blockStore, allowing the provided operator address and local validator key to control the network // that the state in the data folder represents. The chainID of the local genesis file is modified to match the provided chainID. -func testnetify(ctx *Context, config *cfg.Config, home, newChainID, newOperatorAddress string, testnetAppCreator types.AppCreator, db db.DB) (types.Application, error) { +func testnetify(ctx *Context, home string, testnetAppCreator types.AppCreator, db db.DB) (types.Application, error) { + config := ctx.Config + + newChainID, ok := ctx.Viper.Get(KeyNewChainID).(string) + if !ok { + return nil, fmt.Errorf("expected string for key %s", KeyNewChainID) + } + traceWriterFile := ctx.Viper.GetString(flagTraceStore) traceWriter, err := openTraceWriter(traceWriterFile) if err != nil { From 4fc5038cc65292212b3a897dc973c19ec06f574d Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Mon, 29 Jan 2024 11:20:30 -0700 Subject: [PATCH 06/11] remove redundancy --- server/start.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/server/start.go b/server/start.go index d894f75e0ee8..6defcb66edd7 100644 --- a/server/start.go +++ b/server/start.go @@ -6,6 +6,7 @@ import ( "bufio" "errors" "fmt" + "io" "net" "net/http" "os" @@ -280,7 +281,7 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App var app types.Application if isTestnet, ok := ctx.Viper.Get(KeyIsTestnet).(bool); ok && isTestnet { - app, err = testnetify(ctx, home, appCreator, db) + app, err = testnetify(ctx, home, appCreator, db, traceWriter) if err != nil { return err } @@ -669,7 +670,7 @@ before stopping the daemon. // testnetify modifies both state and blockStore, allowing the provided operator address and local validator key to control the network // that the state in the data folder represents. The chainID of the local genesis file is modified to match the provided chainID. -func testnetify(ctx *Context, home string, testnetAppCreator types.AppCreator, db db.DB) (types.Application, error) { +func testnetify(ctx *Context, home string, testnetAppCreator types.AppCreator, db db.DB, traceWriter io.WriteCloser) (types.Application, error) { config := ctx.Config newChainID, ok := ctx.Viper.Get(KeyNewChainID).(string) @@ -677,12 +678,6 @@ func testnetify(ctx *Context, home string, testnetAppCreator types.AppCreator, d return nil, fmt.Errorf("expected string for key %s", KeyNewChainID) } - traceWriterFile := ctx.Viper.GetString(flagTraceStore) - traceWriter, err := openTraceWriter(traceWriterFile) - if err != nil { - return nil, err - } - genDocProvider := node.DefaultGenesisDocProviderFunc(config) // Initialize blockStore and stateDB. From f6f6dd597616c40a7d649531102420205b802cc3 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Mon, 29 Jan 2024 11:22:28 -0700 Subject: [PATCH 07/11] lints --- server/start.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/start.go b/server/start.go index 6defcb66edd7..ba4a6fb818a4 100644 --- a/server/start.go +++ b/server/start.go @@ -621,8 +621,8 @@ has already been modified by this command could result in unexpected behavior. Additionally, the first block may take up to one minute to be committed, depending on how old the block is. For instance, if a snapshot was taken weeks ago and we want -to turn this into a testnet, it is possible lots of pending state needs to be commited -(expiring locks, etc.). It is recommended that you should wait for this block to be commited +to turn this into a testnet, it is possible lots of pending state needs to be committed +(expiring locks, etc.). It is recommended that you should wait for this block to be committed before stopping the daemon. `, Example: "in-place-testnet localosmosis osmo12smx2wdlyttvyzvzg54y2vnqwq2qjateuf7thj", @@ -653,7 +653,7 @@ before stopping the daemon. text, _ := reader.ReadString('\n') response := strings.TrimSpace(strings.ToLower(text)) if response != "y" && response != "yes" { - fmt.Println("Operation cancelled.") + fmt.Println("Operation canceled.") return nil } serverCtx.Viper.Set(KeyIsTestnet, true) @@ -721,7 +721,7 @@ func testnetify(ctx *Context, home string, testnetAppCreator types.AppCreator, d } // There are times when a user stops their node between commits, resulting in a mismatch between the - // blockStore and state. For convenience, we just discard the uncommited blockStore block and operate on + // blockStore and state. For convenience, we just discard the uncommitted blockStore block and operate on // the lastBlockHeight in state. if blockStore.Height() != state.LastBlockHeight { err = blockStore.DeleteLatestBlock() From 42a877ae2895f18fcfbee0c8d7c1a519a4ba9f49 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Mon, 29 Jan 2024 11:47:21 -0700 Subject: [PATCH 08/11] gci imports --- server/start.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/server/start.go b/server/start.go index ba4a6fb818a4..61a8ed78e379 100644 --- a/server/start.go +++ b/server/start.go @@ -4,6 +4,7 @@ package server import ( "bufio" + "encoding/hex" "errors" "fmt" "io" @@ -15,27 +16,21 @@ import ( "strings" "time" + "cosmossdk.io/tools/rosetta" + crgserver "cosmossdk.io/tools/rosetta/lib/server" db "github.com/cometbft/cometbft-db" "github.com/cometbft/cometbft/abci/server" tcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" + cmtjson "github.com/cometbft/cometbft/libs/json" "github.com/cometbft/cometbft/node" "github.com/cometbft/cometbft/p2p" pvm "github.com/cometbft/cometbft/privval" - "github.com/cometbft/cometbft/proxy" - "github.com/cometbft/cometbft/rpc/client/local" - "github.com/cometbft/cometbft/store" - "github.com/spf13/cobra" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - - "encoding/hex" - - "cosmossdk.io/tools/rosetta" - crgserver "cosmossdk.io/tools/rosetta/lib/server" - cmtjson "github.com/cometbft/cometbft/libs/json" cmtstate "github.com/cometbft/cometbft/proto/tendermint/state" cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" + "github.com/cometbft/cometbft/proxy" + "github.com/cometbft/cometbft/rpc/client/local" sm "github.com/cometbft/cometbft/state" + "github.com/cometbft/cometbft/store" cmttypes "github.com/cometbft/cometbft/types" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" @@ -50,6 +45,9 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/types/mempool" "github.com/cosmos/cosmos-sdk/x/genutil" + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) const ( From 2339c3afc72f50b660cb6f1e945cd67bc27024c7 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Thu, 1 Feb 2024 15:36:43 -0700 Subject: [PATCH 09/11] add set store loader method --- baseapp/options.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/baseapp/options.go b/baseapp/options.go index e669495f647d..fab11203e237 100644 --- a/baseapp/options.go +++ b/baseapp/options.go @@ -96,6 +96,11 @@ func SetChainID(chainID string) func(*BaseApp) { return func(app *BaseApp) { app.chainID = chainID } } +// SetStoreLoader allows us to customize the rootMultiStore initialization. +func SetStoreLoader(loader StoreLoader) func(*BaseApp) { + return func(app *BaseApp) { app.SetStoreLoader(loader) } +} + func (app *BaseApp) SetName(name string) { if app.sealed { panic("SetName() on sealed BaseApp") From 8ffea11429d207f19d2e17c1a3d88c73c3a31241 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Thu, 1 Feb 2024 16:12:47 -0700 Subject: [PATCH 10/11] trigger upgrade flag --- server/start.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/start.go b/server/start.go index 61a8ed78e379..9716c5a70337 100644 --- a/server/start.go +++ b/server/start.go @@ -99,11 +99,12 @@ const ( FlagMempoolMaxTxs = "mempool.max-txs" // testnet keys - KeyIsTestnet = "is-testnet" - KeyNewChainID = "new-chain-ID" - KeyNewOpAddr = "new-operator-addr" - KeyNewValAddr = "new-validator-addr" - KeyUserPubKey = "user-pub-key" + KeyIsTestnet = "is-testnet" + KeyNewChainID = "new-chain-ID" + KeyNewOpAddr = "new-operator-addr" + KeyNewValAddr = "new-validator-addr" + KeyUserPubKey = "user-pub-key" + KeyTriggerTestnetUpgrade = "trigger-testnet-upgrade" ) // StartCmd runs the service passed in, either stand-alone or in-process with @@ -663,6 +664,7 @@ before stopping the daemon. } addStartNodeFlags(cmd, defaultNodeHome) + cmd.Flags().String(KeyTriggerTestnetUpgrade, "", "If set (example: \"v21\"), triggers the v21 upgrade handler to run on the first block of the testnet") return cmd } From cf4b1bdc6a293081b3806a3eb0ee92faf0574827 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Thu, 1 Feb 2024 16:53:48 -0700 Subject: [PATCH 11/11] add extra comments to the CLI command --- server/start.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/start.go b/server/start.go index 9716c5a70337..173e42a06dbe 100644 --- a/server/start.go +++ b/server/start.go @@ -623,6 +623,13 @@ on how old the block is. For instance, if a snapshot was taken weeks ago and we to turn this into a testnet, it is possible lots of pending state needs to be committed (expiring locks, etc.). It is recommended that you should wait for this block to be committed before stopping the daemon. + +If the --trigger-testnet-upgrade flag is set, the upgrade handler specified by the flag will be run +on the first block of the testnet. + +Regardless of whether the flag is set or not, if any new stores are introduced in the daemon being run, +those stores will be registered in order to prevent panics. Therefore, you only need to set the flag if +you want to test the upgrade handler itself. `, Example: "in-place-testnet localosmosis osmo12smx2wdlyttvyzvzg54y2vnqwq2qjateuf7thj", Args: cobra.ExactArgs(2),