From ccb4d55a7c978ca5433bd5726ff08799e7e4727e Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 11 Oct 2023 16:27:44 +0800 Subject: [PATCH 1/3] cmd, core: resolve scheme from a read-write database (#28313) * cmd, core: resolve scheme from a read-write database * cmd, core, eth: move the scheme check in the ethereum constructor * cmd/geth: dump should in ro mode * cmd: reverts --- cmd/geth/chaincmd.go | 2 +- cmd/utils/flags.go | 54 +++--------------------------------- core/genesis.go | 12 ++++---- core/genesis_test.go | 2 +- core/rawdb/accessors_trie.go | 35 +++++++++++++++++++++++ eth/backend.go | 9 ++++-- eth/ethconfig/config.go | 6 +++- 7 files changed, 59 insertions(+), 61 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 2edf676ad4..c8055ae839 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -655,7 +655,7 @@ func dump(ctx *cli.Context) error { if err != nil { return err } - triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup + triedb := utils.MakeTrieDatabase(ctx, db, true, true) // always enable preimage lookup defer triedb.Close() state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c882b00da8..14d2c23923 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1884,15 +1884,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(StateHistoryFlag.Name) { cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) } - // Parse state scheme, abort the process if it's not compatible. - chaindb := tryMakeReadOnlyDatabase(ctx, stack) - scheme, err := ParseStateScheme(ctx, chaindb) - chaindb.Close() - if err != nil { - Fatalf("%v", err) + if ctx.IsSet(StateSchemeFlag.Name) { + cfg.StateScheme = ctx.String(StateSchemeFlag.Name) } - cfg.StateScheme = scheme - // Parse transaction history flag, if user is still using legacy config // file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'. if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit { @@ -2355,7 +2349,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } - scheme, err := ParseStateScheme(ctx, chainDb) + scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), chainDb) if err != nil { Fatalf("%v", err) } @@ -2418,52 +2412,12 @@ func MakeConsolePreloads(ctx *cli.Context) []string { return preloads } -// ParseStateScheme checks if the specified state scheme is compatible with -// the stored state. -// -// - If the provided scheme is none, use the scheme consistent with persistent -// state, or fallback to hash-based scheme if state is empty. -// -// - If the provided scheme is hash, use hash-based scheme or error out if not -// compatible with persistent state scheme. -// -// - If the provided scheme is path: use path-based scheme or error out if not -// compatible with persistent state scheme. -func ParseStateScheme(ctx *cli.Context, disk ethdb.Database) (string, error) { - // If state scheme is not specified, use the scheme consistent - // with persistent state, or fallback to hash mode if database - // is empty. - provided, err := compareCLIWithConfig(ctx) - if err != nil { - log.Error("failed to compare CLI with config", "error", err) - return "", err - } - - stored := rawdb.ReadStateScheme(disk) - if provided == "" { - if stored == "" { - // use default scheme for empty database, flip it when - // path mode is chosen as default - log.Info("State scheme set to default", "scheme", rawdb.HashScheme) - return rawdb.HashScheme, nil - } - log.Info("State scheme set to already existing disk db", "scheme", stored) - return stored, nil // reuse scheme of persistent scheme - } - // If state scheme is specified, ensure it's compatible with persistent state. - if stored == "" || provided == stored { - log.Info("State scheme set by user", "scheme", provided) - return provided, nil - } - return "", fmt.Errorf("incompatible state scheme, db stored: %s, user provided: %s", stored, provided) -} - // MakeTrieDatabase constructs a trie database based on the configured scheme. func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database { config := &trie.Config{ Preimages: preimage, } - scheme, err := ParseStateScheme(ctx, disk) + scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), disk) if err != nil { Fatalf("%v", err) } diff --git a/core/genesis.go b/core/genesis.go index 48625a8b75..bd565e7d3f 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -120,8 +120,8 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { return nil } -// deriveHash computes the state root according to the genesis specification. -func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { +// hash computes the state root according to the genesis specification. +func (ga *GenesisAlloc) hash() (common.Hash, error) { // Create an ephemeral in-memory database for computing hash, // all the derived states will be discarded to not pollute disk. db := state.NewDatabase(rawdb.NewMemoryDatabase()) @@ -142,9 +142,9 @@ func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { return root, err } -// flush is very similar with deriveHash, but the main difference is -// all the generated states will be persisted into the given database. -// Also, the genesis state specification will be flushed as well. +// flush is very similar with hash, but the main difference is all the generated +// states will be persisted into the given database. Also, the genesis state +// specification will be flushed as well. func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error { trieConfig := triedb.Config() if trieConfig != nil { @@ -455,7 +455,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { // ToBlock returns the genesis block according to genesis specification. func (g *Genesis) ToBlock() *types.Block { - root, err := g.Alloc.deriveHash() + root, err := g.Alloc.hash() if err != nil { panic(err) } diff --git a/core/genesis_test.go b/core/genesis_test.go index 815ab5d020..a91d0ff3c2 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -209,7 +209,7 @@ func TestReadWriteGenesisAlloc(t *testing.T) { {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, } - hash, _ = alloc.deriveHash() + hash, _ = alloc.hash() ) blob, _ := json.Marshal(alloc) rawdb.WriteGenesisStateSpec(db, hash, blob) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index 458e6a99c3..cd90647346 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -314,3 +314,38 @@ func ValidateStateScheme(stateScheme string) bool { } return false } + +// ParseStateScheme checks if the specified state scheme is compatible with +// the stored state. +// +// - If the provided scheme is none, use the scheme consistent with persistent +// state, or fallback to hash-based scheme if state is empty. +// +// - If the provided scheme is hash, use hash-based scheme or error out if not +// compatible with persistent state scheme. +// +// - If the provided scheme is path: use path-based scheme or error out if not +// compatible with persistent state scheme. +func ParseStateScheme(provided string, disk ethdb.Database) (string, error) { + // If state scheme is not specified, use the scheme consistent + // with persistent state, or fallback to hash mode if database + // is empty. + stored := ReadStateScheme(disk) + if provided == "" { + if stored == "" { + // use default scheme for empty database, flip it when + // path mode is chosen as default + log.Info("State schema set to default", "scheme", "hash") + return HashScheme, nil + } + log.Info("State scheme set to already existing", "scheme", stored) + return stored, nil // reuse scheme of persistent scheme + } + // If state scheme is specified, ensure it's compatible with + // persistent state. + if stored == "" || provided == stored { + log.Info("State scheme set by user", "scheme", provided) + return provided, nil + } + return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, provided) +} diff --git a/eth/backend.go b/eth/backend.go index 61021099a7..60f2e249aa 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -156,7 +156,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - if config.StateScheme == rawdb.HashScheme { + scheme, err := rawdb.ParseStateScheme(config.StateScheme, chainDb) + if err != nil { + return nil, err + } + // Try to recover offline state pruning only in hash-based. + if scheme == rawdb.HashScheme { if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, config.TriesInMemory); err != nil { log.Error("Failed to recover state", "error", err) } @@ -242,8 +247,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TriesInMemory: config.TriesInMemory, Preimages: config.Preimages, StateHistory: config.StateHistory, - StateScheme: config.StateScheme, PathSyncFlush: config.PathSyncFlush, + StateScheme: scheme, } ) bcOps := make([]core.BlockChainOption, 0) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 7fdc6f4388..2742d33b44 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -112,9 +112,13 @@ type Config struct { TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. - StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top PathSyncFlush bool `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top + // State scheme represents the scheme used to store ethereum states and trie + // nodes on top. It can be 'hash', 'path', or none which means use the scheme + // consistent with persistent state. + StateScheme string `toml:",omitempty"` + // RequiredBlocks is a set of block number -> hash mappings which must be in the // canonical chain of all remote peers. Setting the option makes geth verify the // presence of these blocks for every new peer connection. From fee8a25957b58ebe883feb4facfe1ca1d765a540 Mon Sep 17 00:00:00 2001 From: VM Date: Fri, 12 Jan 2024 15:36:26 +0800 Subject: [PATCH 2/3] fix: optimize resolveChainFreezerDir func --- cmd/geth/genesis_test.go | 4 +- cmd/utils/flags.go | 24 ++++++-- core/rawdb/database.go | 36 +++++++----- core/rawdb/database_test.go | 111 ++++++++++++++++++++++++++++++++++++ core/rawdb/freezer.go | 2 +- core/rawdb/prunedfreezer.go | 2 +- eth/backend.go | 25 ++++---- 7 files changed, 167 insertions(+), 37 deletions(-) diff --git a/cmd/geth/genesis_test.go b/cmd/geth/genesis_test.go index 0aa3d8596d..54ab7e434d 100644 --- a/cmd/geth/genesis_test.go +++ b/cmd/geth/genesis_test.go @@ -176,12 +176,12 @@ func TestCustomBackend(t *testing.T) { { // Can't start pebble on top of leveldb initArgs: []string{"--db.engine", "leveldb"}, execArgs: []string{"--db.engine", "pebble"}, - execExpect: `Fatal: Could not open database: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`, + execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`, }, { // Can't start leveldb on top of pebble initArgs: []string{"--db.engine", "pebble"}, execArgs: []string{"--db.engine", "leveldb"}, - execExpect: `Fatal: Could not open database: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`, + execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`, }, { // Reject invalid backend choice initArgs: []string{"--db.engine", "mssql"}, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 14d2c23923..a360b0f9fd 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1848,7 +1848,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if cfg.SyncMode == downloader.FullSync { cfg.PruneAncientData = ctx.Bool(PruneAncientDataFlag.Name) } else { - log.Crit("pruneancient parameter didn't take effect for current syncmode") + log.Crit("pruneancient parameter can only be used with syncmode=full") } } if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { @@ -1884,9 +1884,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(StateHistoryFlag.Name) { cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) } - if ctx.IsSet(StateSchemeFlag.Name) { - cfg.StateScheme = ctx.String(StateSchemeFlag.Name) + scheme, err := compareCLIWithConfig(ctx) + if err != nil { + Fatalf("%v", err) } + cfg.StateScheme = scheme // Parse transaction history flag, if user is still using legacy config // file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'. if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit { @@ -2040,7 +2042,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { readonly = false } // Check if we have an already initialized chain and fall back to - // that if so. Otherwise we need to generate a new genesis spec. + // that if so. Otherwise, we need to generate a new genesis spec. chaindb := MakeChainDatabase(ctx, stack, readonly, false) if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) { cfg.Genesis = nil // fallback to db content @@ -2275,6 +2277,8 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFree // tryMakeReadOnlyDatabase try to open the chain database in read-only mode, // or fallback to write mode if the database is not initialized. +// +//nolint:unused func tryMakeReadOnlyDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { // If datadir doesn't exist we need to open db in write-mode // so database engine can create files. @@ -2349,7 +2353,11 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } - scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), chainDb) + provided, err := compareCLIWithConfig(ctx) + if err != nil { + Fatalf("%v", err) + } + scheme, err := rawdb.ParseStateScheme(provided, chainDb) if err != nil { Fatalf("%v", err) } @@ -2417,7 +2425,11 @@ func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, read config := &trie.Config{ Preimages: preimage, } - scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), disk) + provided, err := compareCLIWithConfig(ctx) + if err != nil { + Fatalf("%v", err) + } + scheme, err := rawdb.ParseStateScheme(provided, disk) if err != nil { Fatalf("%v", err) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 7a5d0aeb48..8a79ae9de5 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -224,20 +224,19 @@ func resolveChainFreezerDir(ancient string) string { // sub folder, if not then two possibilities: // - chain freezer is not initialized // - chain freezer exists in legacy location (root ancient folder) - freezer := path.Join(ancient, chainFreezerName) - if !common.FileExist(freezer) { - if !common.FileExist(ancient) { - // The entire ancient store is not initialized, still use the sub - // folder for initialization. - } else { - // Ancient root is already initialized, then we hold the assumption - // that chain freezer is also initialized and located in root folder. - // In this case fallback to legacy location. - freezer = ancient - log.Info("Found legacy ancient chain path", "location", ancient) - } + chain := path.Join(ancient, chainFreezerName) + state := path.Join(ancient, stateFreezerName) + if common.FileExist(chain) { + return chain + } + if common.FileExist(state) { + return chain + } + if common.FileExist(ancient) { + log.Info("Found legacy ancient chain path", "location", ancient) + chain = ancient } - return freezer + return chain } // NewDatabaseWithFreezer creates a high level database on top of a given key- @@ -264,6 +263,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st WriteAncientType(db, PruneFreezerType) } return &freezerdb{ + ancientRoot: ancient, KeyValueStore: db, AncientStore: frdb, }, nil @@ -336,7 +336,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st break } } - // We are about to exit on error. Print database metdata beore exiting + // We are about to exit on error. Print database metdata before exiting printChainMetadata(db) return nil, fmt.Errorf("gap in the chain between ancients [0 - #%d] and leveldb [#%d - #%d] ", frozen-1, number, head) @@ -365,7 +365,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // freezer. } } - // no prune ancinet start success + // no prune ancient start success if !readonly { WriteAncientType(db, EntireFreezerType) } @@ -516,6 +516,11 @@ func Open(o OpenOptions) (ethdb.Database, error) { if err != nil { return nil, err } + if ReadAncientType(kvdb) == PruneFreezerType { + if !o.PruneAncientData { + log.Warn("Disk db is pruned") + } + } if len(o.AncientsDirectory) == 0 { return kvdb, nil } @@ -556,6 +561,7 @@ func (s *stat) Size() string { func (s *stat) Count() string { return s.count.String() } + func AncientInspect(db ethdb.Database) error { offset := counter(ReadOffSetOfCurrentAncientFreezer(db)) // Get number of ancient rows inside the freezer. diff --git a/core/rawdb/database_test.go b/core/rawdb/database_test.go index a0d7b5ec66..bd5fa0ccc5 100644 --- a/core/rawdb/database_test.go +++ b/core/rawdb/database_test.go @@ -15,3 +15,114 @@ // along with the go-ethereum library. If not, see . package rawdb + +import ( + "fmt" + "os" + "testing" +) + +const ( + mockChainFreezerPath = "/geth/chaindata/ancient/chain" + mockStateFreezerPath = "/geth/chaindata/ancient/state" + mockAncientFreezerPath = "/geth/chaindata/ancient" +) + +func Test_resolveChainFreezerDir(t *testing.T) { + tests := []struct { + name string + fn func(dir string) string + ancient string + wantedResult string + }{ + { + name: "run geth in pruned mode and chain dir is existent", + fn: func(dir string) string { + path := fmt.Sprintf("%s%s", dir, mockChainFreezerPath) + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatalf("Failed to mkdir all dirs, error: %v", err) + } + return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) + }, + wantedResult: mockChainFreezerPath, + }, + { + name: "run geth in path and pruned mode; chain is nonexistent and state is existent", + fn: func(dir string) string { + path := fmt.Sprintf("%s%s", dir, mockStateFreezerPath) + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatalf("Failed to mkdir all dirs, error: %v", err) + } + return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) + }, + wantedResult: mockChainFreezerPath, + }, + { + name: "run geth in hash and pruned mode; ancient block data locates in ancient dir", + fn: func(dir string) string { + path := fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatalf("Failed to mkdir all dirs, error: %v", err) + } + return path + }, + wantedResult: mockAncientFreezerPath, + }, + { + name: "run geth in pruned mode and there is no ancient dir", + fn: func(dir string) string { + return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) + }, + wantedResult: mockChainFreezerPath, + }, + { + name: "run geth in non-pruned mode; ancient is existent and state dir is nonexistent", + fn: func(dir string) string { + path := fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatalf("Failed to mkdir all dirs, error: %v", err) + } + return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) + }, + wantedResult: mockAncientFreezerPath, + }, + // { + // name: "run geth in non-pruned mode and chain dir is existent", + // fn: func(dir string) string { + // path := fmt.Sprintf("%s%s", dir, mockChainFreezerPath) + // if err := os.MkdirAll(path, 0700); err != nil { + // t.Fatalf("Failed to mkdir all dirs, error: %v", err) + // } + // return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) + // }, + // wantedResult: mockChainFreezerPath, + // }, + // { + // name: "run geth in non-pruned mode, ancient and chain dir is nonexistent", + // fn: func(dir string) string { + // return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) + // }, + // wantedResult: mockChainFreezerPath, + // }, + // { + // name: "run geth in non-pruned mode; ancient dir is existent, chain and state dir is nonexistent", + // fn: func(dir string) string { + // path := fmt.Sprintf("%s%s", dir, mockStateFreezerPath) + // if err := os.MkdirAll(path, 0700); err != nil { + // t.Fatalf("Failed to mkdir all dirs, error: %v", err) + // } + // return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) + // }, + // wantedResult: mockChainFreezerPath, + // }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempDir := t.TempDir() + got := resolveChainFreezerDir(tt.fn(tempDir)) + if got != fmt.Sprintf("%s%s", tempDir, tt.wantedResult) { + t.Fatalf("resolveChainFreezerDir() = %s, wanted = %s", got, tt.wantedResult) + } + }) + } +} diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 0a7bbe436a..c9fb3a3ef4 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -152,7 +152,7 @@ func NewFreezer(datadir string, namespace string, readonly bool, offset uint64, } // Some blocks in ancientDB may have already been frozen and been pruned, so adding the offset to - // reprensent the absolute number of blocks already frozen. + // represent the absolute number of blocks already frozen. freezer.frozen.Add(offset) freezer.tail.Add(offset) diff --git a/core/rawdb/prunedfreezer.go b/core/rawdb/prunedfreezer.go index ffc6647816..ddc77cd49a 100644 --- a/core/rawdb/prunedfreezer.go +++ b/core/rawdb/prunedfreezer.go @@ -57,7 +57,7 @@ func newPrunedFreezer(datadir string, db ethdb.KeyValueStore, offset uint64) (*p // delete ancient dir if err := os.RemoveAll(datadir); err != nil && !os.IsNotExist(err) { - log.Warn("remove the ancient dir failed.", "path", datadir, "error", err) + log.Warn("Failed to remove the ancient dir", "path", datadir, "error", err) return nil, err } log.Info("Opened ancientdb with nodata mode", "database", datadir, "frozen", freezer.frozen) diff --git a/eth/backend.go b/eth/backend.go index 60f2e249aa..25791aeca3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -130,6 +130,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { log.Warn("Sanitizing invalid miner gas price", "provided", config.Miner.GasPrice, "updated", ethconfig.Defaults.Miner.GasPrice) config.Miner.GasPrice = new(big.Int).Set(ethconfig.Defaults.Miner.GasPrice) } + + // Assemble the Ethereum object + chainDb, err := stack.OpenAndMergeDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles, + config.DatabaseFreezer, config.DatabaseDiff, "eth/db/chaindata/", false, config.PersistDiff, config.PruneAncientData) + if err != nil { + return nil, err + } + config.StateScheme, err = rawdb.ParseStateScheme(config.StateScheme, chainDb) + if err != nil { + return nil, err + } // Redistribute memory allocation from in-memory trie node garbage collection // to other caches when an archive node is requested. if config.StateScheme == rawdb.HashScheme && config.NoPruning && config.TrieDirtyCache > 0 { @@ -150,18 +161,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) - // Assemble the Ethereum object - chainDb, err := stack.OpenAndMergeDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles, - config.DatabaseFreezer, config.DatabaseDiff, "eth/db/chaindata/", false, config.PersistDiff, config.PruneAncientData) - if err != nil { - return nil, err - } - scheme, err := rawdb.ParseStateScheme(config.StateScheme, chainDb) - if err != nil { - return nil, err - } // Try to recover offline state pruning only in hash-based. - if scheme == rawdb.HashScheme { + if config.StateScheme == rawdb.HashScheme { if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, config.TriesInMemory); err != nil { log.Error("Failed to recover state", "error", err) } @@ -247,8 +248,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TriesInMemory: config.TriesInMemory, Preimages: config.Preimages, StateHistory: config.StateHistory, + StateScheme: config.StateScheme, PathSyncFlush: config.PathSyncFlush, - StateScheme: scheme, } ) bcOps := make([]core.BlockChainOption, 0) From a17fef4f155b354d7d79924d30cb6cc8cb16cc3a Mon Sep 17 00:00:00 2001 From: VM Date: Mon, 15 Jan 2024 11:09:41 +0800 Subject: [PATCH 3/3] test: add UT for resolveChainFreezerDir function --- core/rawdb/accessors_trie.go | 4 +-- core/rawdb/database.go | 4 +++ core/rawdb/database_test.go | 54 +++++++----------------------------- 3 files changed, 16 insertions(+), 46 deletions(-) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index cd90647346..5248fbecab 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -338,7 +338,7 @@ func ParseStateScheme(provided string, disk ethdb.Database) (string, error) { log.Info("State schema set to default", "scheme", "hash") return HashScheme, nil } - log.Info("State scheme set to already existing", "scheme", stored) + log.Info("State scheme set to already existing disk db", "scheme", stored) return stored, nil // reuse scheme of persistent scheme } // If state scheme is specified, ensure it's compatible with @@ -347,5 +347,5 @@ func ParseStateScheme(provided string, disk ethdb.Database) (string, error) { log.Info("State scheme set by user", "scheme", provided) return provided, nil } - return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, provided) + return "", fmt.Errorf("incompatible state scheme, stored: %s, user provided: %s", stored, provided) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 8a79ae9de5..ea300d5fd9 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -219,6 +219,10 @@ func NewFreezerDb(db ethdb.KeyValueStore, frz, namespace string, readonly bool, // resolveChainFreezerDir is a helper function which resolves the absolute path // of chain freezer by considering backward compatibility. +// +// rules: +// 1. in path mode, block data is stored in chain dir and state data is in state dir. +// 2. in hash mode, block data is stored in chain dir or ancient dir(before big merge), no state dir. func resolveChainFreezerDir(ancient string) string { // Check if the chain freezer is already present in the specified // sub folder, if not then two possibilities: diff --git a/core/rawdb/database_test.go b/core/rawdb/database_test.go index bd5fa0ccc5..baec1663d3 100644 --- a/core/rawdb/database_test.go +++ b/core/rawdb/database_test.go @@ -36,7 +36,8 @@ func Test_resolveChainFreezerDir(t *testing.T) { wantedResult string }{ { - name: "run geth in pruned mode and chain dir is existent", + // chain dir is existent, so it should be returned. + name: "1", fn: func(dir string) string { path := fmt.Sprintf("%s%s", dir, mockChainFreezerPath) if err := os.MkdirAll(path, 0700); err != nil { @@ -47,7 +48,9 @@ func Test_resolveChainFreezerDir(t *testing.T) { wantedResult: mockChainFreezerPath, }, { - name: "run geth in path and pruned mode; chain is nonexistent and state is existent", + // chain dir is nonexistent and state dir is existent; so chain + // dir should be returned. + name: "2", fn: func(dir string) string { path := fmt.Sprintf("%s%s", dir, mockStateFreezerPath) if err := os.MkdirAll(path, 0700); err != nil { @@ -58,7 +61,9 @@ func Test_resolveChainFreezerDir(t *testing.T) { wantedResult: mockChainFreezerPath, }, { - name: "run geth in hash and pruned mode; ancient block data locates in ancient dir", + // both chain dir and state dir are nonexistent, if ancient dir is + // existent, so ancient dir should be returned. + name: "3", fn: func(dir string) string { path := fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) if err := os.MkdirAll(path, 0700); err != nil { @@ -69,52 +74,13 @@ func Test_resolveChainFreezerDir(t *testing.T) { wantedResult: mockAncientFreezerPath, }, { - name: "run geth in pruned mode and there is no ancient dir", + // ancient dir is nonexistent, so chain dir should be returned. + name: "4", fn: func(dir string) string { return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) }, wantedResult: mockChainFreezerPath, }, - { - name: "run geth in non-pruned mode; ancient is existent and state dir is nonexistent", - fn: func(dir string) string { - path := fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) - if err := os.MkdirAll(path, 0700); err != nil { - t.Fatalf("Failed to mkdir all dirs, error: %v", err) - } - return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) - }, - wantedResult: mockAncientFreezerPath, - }, - // { - // name: "run geth in non-pruned mode and chain dir is existent", - // fn: func(dir string) string { - // path := fmt.Sprintf("%s%s", dir, mockChainFreezerPath) - // if err := os.MkdirAll(path, 0700); err != nil { - // t.Fatalf("Failed to mkdir all dirs, error: %v", err) - // } - // return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) - // }, - // wantedResult: mockChainFreezerPath, - // }, - // { - // name: "run geth in non-pruned mode, ancient and chain dir is nonexistent", - // fn: func(dir string) string { - // return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) - // }, - // wantedResult: mockChainFreezerPath, - // }, - // { - // name: "run geth in non-pruned mode; ancient dir is existent, chain and state dir is nonexistent", - // fn: func(dir string) string { - // path := fmt.Sprintf("%s%s", dir, mockStateFreezerPath) - // if err := os.MkdirAll(path, 0700); err != nil { - // t.Fatalf("Failed to mkdir all dirs, error: %v", err) - // } - // return fmt.Sprintf("%s%s", dir, mockAncientFreezerPath) - // }, - // wantedResult: mockChainFreezerPath, - // }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {