From 55886522235b15c7ffa95c357d84458d8528cd8c Mon Sep 17 00:00:00 2001 From: adu Date: Tue, 30 Aug 2022 20:48:50 +0800 Subject: [PATCH 1/2] add PruningCmd and change PruneStores signature --- CHANGELOG.md | 1 + client/pruning/main.go | 117 +++++++++++++++++++++++++++++++++++++++ simapp/simd/cmd/root.go | 2 + store/rootmulti/store.go | 33 +++++++---- 4 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 client/pruning/main.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e1e90db8dc4..99f23e746d22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (testutil) [#12973](https://github.com/cosmos/cosmos-sdk/pull/12973) Add generic `testutil.RandSliceElem` function which selects a random element from the list. * (client) [#12936](https://github.com/cosmos/cosmos-sdk/pull/12936) Add capability to preprocess transactions before broadcasting from a higher level chain. * (x/authz) [#13047](https://github.com/cosmos/cosmos-sdk/pull/13047) Add a GetAuthorization function to the keeper. +* (cli) [#12742](https://github.com/cosmos/cosmos-sdk/pull/12742) Add the `prune` CLI cmd to manually prune app store history versions based on the pruning options. ### Improvements diff --git a/client/pruning/main.go b/client/pruning/main.go new file mode 100644 index 000000000000..dca943032ed6 --- /dev/null +++ b/client/pruning/main.go @@ -0,0 +1,117 @@ +package pruning + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cosmos/cosmos-sdk/client/flags" + pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types" + "github.com/cosmos/cosmos-sdk/server" + servertypes "github.com/cosmos/cosmos-sdk/server/types" + "github.com/cosmos/cosmos-sdk/store/rootmulti" + "github.com/tendermint/tendermint/libs/log" + dbm "github.com/tendermint/tm-db" +) + +const FlagAppDBBackend = "app-db-backend" + +// PruningCmd prunes the sdk root multi store history versions based on the pruning options +// specified by command flags. +func PruningCmd(appCreator servertypes.AppCreator) *cobra.Command { + cmd := &cobra.Command{ + Use: "prune", + Short: "Prune app history states by keeping the recent heights and deleting old heights", + Long: `Prune app history states by keeping the recent heights and deleting old heights. + The pruning option is provided via the '--pruning' flag or alternatively with '--pruning-keep-recent' + + For '--pruning' the options are as follows: + + default: the last 362880 states are kept + nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node) + everything: 2 latest states will be kept + custom: allow pruning options to be manually specified through 'pruning-keep-recent'. + besides pruning options, database home directory and database backend type should also be specified via flags + '--home' and '--app-db-backend'. + valid app-db-backend type includes 'goleveldb', 'cleveldb', 'rocksdb', 'boltdb', and 'badgerdb'. + `, + Example: "prune --home './' --app-db-backend 'goleveldb' --pruning 'custom' --pruning-keep-recent 100", + RunE: func(cmd *cobra.Command, _ []string) error { + vp := viper.New() + + // Bind flags to the Context's Viper so we can get pruning options. + if err := vp.BindPFlags(cmd.Flags()); err != nil { + return err + } + pruningOptions, err := server.GetPruningOptionsFromFlags(vp) + if err != nil { + return err + } + fmt.Printf("get pruning options from command flags, strategy: %v, keep-recent: %v\n", + pruningOptions.Strategy, + pruningOptions.KeepRecent, + ) + + home := vp.GetString(flags.FlagHome) + db, err := openDB(home, server.GetAppDBBackend(vp)) + if err != nil { + return err + } + + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + app := appCreator(logger, db, nil, vp) + cms := app.CommitMultiStore() + + rootMultiStore, ok := cms.(*rootmulti.Store) + if !ok { + return fmt.Errorf("currently only support the pruning of rootmulti.Store type") + } + latestHeight := rootmulti.GetLatestVersion(db) + // valid heights should be greater than 0. + if latestHeight <= 0 { + return fmt.Errorf("the database has no valid heights to prune, the latest height: %v", latestHeight) + } + + var pruningHeights []int64 + for height := int64(1); height < latestHeight; height++ { + if height < latestHeight-int64(pruningOptions.KeepRecent) { + pruningHeights = append(pruningHeights, height) + } + } + if len(pruningHeights) == 0 { + fmt.Printf("no heights to prune\n") + return nil + } + fmt.Printf( + "pruning heights start from %v, end at %v\n", + pruningHeights[0], + pruningHeights[len(pruningHeights)-1], + ) + + err = rootMultiStore.PruneStores(false, pruningHeights) + if err != nil { + return err + } + fmt.Printf("successfully pruned the application root multi stores\n") + return nil + }, + } + + cmd.Flags().String(flags.FlagHome, "", "The database home directory") + cmd.Flags().String(FlagAppDBBackend, "", "The type of database for application and snapshots databases") + cmd.Flags().String(server.FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") + cmd.Flags().Uint64(server.FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint64(server.FlagPruningInterval, 0, + `Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom'), + this is not used by this command but kept for compatibility with the complete pruning options`) + + return cmd +} + +func openDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) { + dataDir := filepath.Join(rootDir, "data") + return dbm.NewDB("application", backendType, dataDir) +} diff --git a/simapp/simd/cmd/root.go b/simapp/simd/cmd/root.go index 9b0965458d70..b24a1bd1bc67 100644 --- a/simapp/simd/cmd/root.go +++ b/simapp/simd/cmd/root.go @@ -20,6 +20,7 @@ import ( "github.com/cosmos/cosmos-sdk/client/debug" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/client/keys" + "github.com/cosmos/cosmos-sdk/client/pruning" "github.com/cosmos/cosmos-sdk/client/rpc" "github.com/cosmos/cosmos-sdk/server" serverconfig "github.com/cosmos/cosmos-sdk/server/config" @@ -182,6 +183,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) { NewTestnetCmd(simapp.ModuleBasics, banktypes.GenesisBalancesIterator{}), debug.Cmd(), config.Cmd(), + pruning.PruningCmd(newApp), ) server.AddCommands(rootCmd, simapp.DefaultNodeHome, newApp, appExport, addModuleInitFlags) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 0372c4b9d10b..221294fb7aa0 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -162,7 +162,7 @@ func (rs *Store) StoreKeysByName() map[string]types.StoreKey { // LoadLatestVersionAndUpgrade implements CommitMultiStore func (rs *Store) LoadLatestVersionAndUpgrade(upgrades *types.StoreUpgrades) error { - ver := getLatestVersion(rs.db) + ver := GetLatestVersion(rs.db) return rs.loadVersion(ver, upgrades) } @@ -173,7 +173,7 @@ func (rs *Store) LoadVersionAndUpgrade(ver int64, upgrades *types.StoreUpgrades) // LoadLatestVersion implements CommitMultiStore. func (rs *Store) LoadLatestVersion() error { - ver := getLatestVersion(rs.db) + ver := GetLatestVersion(rs.db) return rs.loadVersion(ver, nil) } @@ -391,7 +391,7 @@ func (rs *Store) ListeningEnabled(key types.StoreKey) bool { func (rs *Store) LastCommitID() types.CommitID { if rs.lastCommitInfo == nil { return types.CommitID{ - Version: getLatestVersion(rs.db), + Version: GetLatestVersion(rs.db), } } @@ -539,17 +539,28 @@ func (rs *Store) handlePruning(version int64) error { } rs.logger.Info("prune start", "height", version) defer rs.logger.Info("prune end", "height", version) - return rs.pruneStores() + return rs.PruneStores(true, nil) } -func (rs *Store) pruneStores() error { - pruningHeights, err := rs.pruningManager.GetFlushAndResetPruningHeights() - if err != nil { - return err +// PruneStores prunes the specific heights of the multi store. +// If clearPruningManager is true, the pruning manager will return the pruning heights, +// and they are appended to the pruningHeights to be pruned. +func (rs *Store) PruneStores(clearPruningManager bool, pruningHeights []int64) (err error) { + if clearPruningManager { + heights, err := rs.pruningManager.GetFlushAndResetPruningHeights() + if err != nil { + return err + } + + if len(heights) == 0 { + rs.logger.Debug("no heights to be pruned from pruning manager") + } + + pruningHeights = append(pruningHeights, heights...) } if len(pruningHeights) == 0 { - rs.logger.Debug("pruning skipped; no heights to prune") + rs.logger.Debug("no heights need to be pruned") return nil } @@ -689,7 +700,7 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error { if height == 0 { return sdkerrors.Wrap(sdkerrors.ErrLogic, "cannot snapshot height 0") } - if height > uint64(getLatestVersion(rs.db)) { + if height > uint64(GetLatestVersion(rs.db)) { return sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot snapshot future height %v", height) } @@ -981,7 +992,7 @@ type storeParams struct { initialVersion uint64 } -func getLatestVersion(db dbm.DB) int64 { +func GetLatestVersion(db dbm.DB) int64 { bz, err := db.Get([]byte(latestVersionKey)) if err != nil { panic(err) From 70e026b1a27fa1256dd76493aa93a157456b9b05 Mon Sep 17 00:00:00 2001 From: adu Date: Wed, 31 Aug 2022 20:21:10 +0800 Subject: [PATCH 2/2] the mimimum default pruning interval is 10 --- client/pruning/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/pruning/main.go b/client/pruning/main.go index dca943032ed6..fb62dd21a050 100644 --- a/client/pruning/main.go +++ b/client/pruning/main.go @@ -104,7 +104,7 @@ func PruningCmd(appCreator servertypes.AppCreator) *cobra.Command { cmd.Flags().String(FlagAppDBBackend, "", "The type of database for application and snapshots databases") cmd.Flags().String(server.FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") cmd.Flags().Uint64(server.FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") - cmd.Flags().Uint64(server.FlagPruningInterval, 0, + cmd.Flags().Uint64(server.FlagPruningInterval, 10, `Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom'), this is not used by this command but kept for compatibility with the complete pruning options`)