Make db inspector for extending multiple ancient stores

core/blockchain.go

@@ -126,7 +126,7 @@ const (
 	BlockChainVersion uint64 = 8
 )
 
-// CacheConfig contains the configuration values for the trie caching/pruning
+// CacheConfig contains the configuration values for the trie database
 // that's resident in a blockchain.
 type CacheConfig struct {
 	TrieCleanLimit      int           // Memory allowance (MB) to use for caching trie nodes in memory

core/rawdb/ancient_scheme.go

@@ -16,8 +16,6 @@
 
 package rawdb
 
-import "fmt"
-
 // The list of table names of chain freezer. (headers, hashes, bodies, difficulties)
 
 const (
@@ -54,36 +52,3 @@ var (
 
 // freezers the collections of all builtin freezers.
 var freezers = []string{chainFreezerName}
-
-// InspectFreezerTable dumps out the index of a specific freezer table. The passed
-// ancient indicates the path of root ancient directory where the chain freezer can
-// be opened. Start and end specifiy the range for dumping out indexes.
-// Note this function can only used for debugging purpose.
-func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
-	var (
-		path   string
-		tables map[string]bool
-	)
-
-	switch freezerName {
-	case chainFreezerName:
-		path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
-	default:
-		return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
-	}
-	noSnappy, exit := tables[tableName]
-	if !exit {
-		// If the tableName is not exit in the tables, return an error.
-		var names []string
-		for name := range tables {
-			names = append(names, name)
-		}
-		return fmt.Errorf("unknown table name, supported ones: %v", names)
-	}
-	table, err := newFreezerTable(path, tableName, noSnappy)
-	if err != nil {
-		return err
-	}
-	table.dumpIndexStdout(start, end)
-	return nil
-}

core/rawdb/ancient_utils.go

@@ -0,0 +1,125 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+type tableSize struct {
+	name string
+	size common.StorageSize
+}
+
+// freezerInfo contains the basic information of the freezer.
+type freezerInfo struct {
+	name  string      // The identifier of freezer
+	head  uint64      // The number of last stored item in the freezer
+	tail  uint64      // The number of first stored item in the freezer
+	sizes []tableSize // The storage size per table
+}
+
+// count returns the number of stored items in the freezer.
+func (info *freezerInfo) count() uint64 {
+	return info.head - info.tail + 1
+}
+
+// size returns the storage size of the entire freezer.
+func (info *freezerInfo) size() common.StorageSize {
+	var total common.StorageSize
+	for _, table := range info.sizes {
+		total += table.size
+	}
+	return total
+}
+
+// inspectFreezers inspects all freezers registered in the system.
+func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
+	var (
+		infos []freezerInfo
+	)
+	for _, freezer := range freezers {
+		switch freezer {
+		case chainFreezerName: // We only support chain freezer for now.
+			// Chain ancient store is a bit special. It's always opened along
+			// with a key-value store, inspect the chain store directly.
+			info := freezerInfo{name: freezer}
+			// Retrieve storage size of every contained table.
+			for table := range chainFreezerNoSnappy {
+				size, err := db.AncientSize(table)
+				if err != nil {
+					return nil, err
+				}
+				info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
+			}
+			// Retrieve the number of last stored item
+			ancients, err := db.Ancients()
+			if err != nil {
+				return nil, err
+			}
+			info.head = ancients - 1
+
+			// Retrieve the number of first stored item
+			tail, err := db.Tail()
+			if err != nil {
+				return nil, err
+			}
+			info.tail = tail
+			infos = append(infos, info)
+		default:
+			return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers)
+		}
+	}
+	return infos, nil
+}
+
+// InspectFreezerTable dumps out the index of a specific freezer table. The passed
+// ancient indicates the path of root ancient directory where the chain freezer can
+// be opened. Start and end specifiy the range for dumping out indexes.
+// Note this function can only used for debugging purpose.
+func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
+	var (
+		path   string
+		tables map[string]bool
+	)
+
+	switch freezerName {
+	case chainFreezerName:
+		path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
+	default:
+		return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
+	}
+	noSnappy, exit := tables[tableName]
+	if !exit {
+		// If the tableName is not exit in the tables, return an error.
+		var names []string
+		for name := range tables {
+			names = append(names, name)
+		}
+		return fmt.Errorf("unknown table name, supported ones: %v", names)
+	}
+	table, err := newFreezerTable(path, tableName, noSnappy)
+	if err != nil {
+		return err
+	}
+
+	table.dumpIndexStdout(start, end)
+	return nil
+}

core/rawdb/chain_freezer.go

@@ -73,6 +73,11 @@ func (f *chainFreezer) Close() error {
 	return err
 }
 
+// Tail returns an error as we don't have a backing chain freezer.
+func (f *chainFreezer) Tail() (uint64, error) {
+	return 0, errNotSupported
+}
+
 // freeze is a background thread that periodically checks the blockchain for any
 // import progress and moves ancient data from the fast database into the freezer.
 //

core/rawdb/database.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -87,6 +88,11 @@ type nofreezedb struct {
 	ethdb.KeyValueStore
 }
 
+// Tail returns an error as we don't have a backing chain freezer.
+func (db *nofreezedb) Tail() (uint64, error) {
+	return 0, errNotSupported
+}
+
 // HasAncient returns an error as we don't have a backing chain freezer.
 func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) {
 	return false, errNotSupported
@@ -445,13 +451,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		cliqueSnaps     stat
 		consortiumSnaps stat
 
-		// Ancient store statistics
-		ancientHeadersSize  common.StorageSize
-		ancientBodiesSize   common.StorageSize
-		ancientReceiptsSize common.StorageSize
-		ancientTdsSize      common.StorageSize
-		ancientHashesSize   common.StorageSize
-
 		// Les statistic
 		chtTrieNodes   stat
 		bloomTrieNodes stat
@@ -540,20 +539,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			logged = time.Now()
 		}
 	}
-	// Inspect append-only file store then.
-	ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
-	for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} {
-		if size, err := db.AncientSize(category); err == nil {
-			*ancientSizes[i] += common.StorageSize(size)
-			total += common.StorageSize(size)
-		}
-	}
-	// Get number of ancient rows inside the freezer
-	ancients := counter(0)
-	if count, err := db.Ancients(); err == nil {
-		ancients = counter(count)
-	}
-	// Display the database statistic.
+
+	// Display the database statistic of key-value store.
 	stats := [][]string{
 		{"Key-Value store", "Headers", headers.Size(), headers.Count()},
 		{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
@@ -571,14 +558,25 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
 		{"Key-Value store", "Consortium snapshots", consortiumSnaps.Size(), consortiumSnaps.Count()},
 		{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
-		{"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()},
-		{"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()},
-		{"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()},
-		{"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()},
-		{"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()},
 		{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
 		{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
 	}
+	// Inspect all registered append-only file store then.
+	ancients, err := inspectFreezers(db)
+	if err != nil {
+		return err
+	}
+	for _, ancient := range ancients {
+		for _, table := range ancient.sizes {
+			stats = append(stats, []string{
+				fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
+				strings.Title(table.name),
+				table.size.String(),
+				fmt.Sprintf("%d", ancient.count()),
+			})
+		}
+		total += ancient.size()
+	}
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"Database", "Category", "Size", "Items"})
 	table.SetFooter([]string{"", "Total", total.String(), " "})

core/rawdb/freezer.go

@@ -219,6 +219,12 @@ func (f *Freezer) AncientSize(kind string) (uint64, error) {
 	return 0, errUnknownTable
 }
 
+// Tail returns an error as we don't have a backing chain freezer.
+func (f *Freezer) Tail() (uint64, error) {
+	// return f.tail.Load(), nil, in the next implementing, right now just keep it zero
+	return 0, nil
+}
+
 // ReadAncients runs the given read operation while ensuring that no writes take place
 // on the underlying freezer.
 func (f *Freezer) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {

core/rawdb/table.go

@@ -68,6 +68,12 @@ func (t *table) AncientRange(kind string, start, count, maxBytes uint64) ([][]by
 	return t.db.AncientRange(kind, start, count, maxBytes)
 }
 
+// Tail is a noop passthrough that just forwards the request to the underlying
+// database.
+func (t *table) Tail() (uint64, error) {
+	return t.db.Tail()
+}
+
 // Ancients is a noop passthrough that just forwards the request to the underlying
 // database.
 func (t *table) Ancients() (uint64, error) {

core/state/state_object.go

@@ -392,9 +392,9 @@ func (s *stateObject) updateRoot(db Database) {
 	s.data.Root = s.trie.Hash()
 }
 
-// CommitTrie the storage trie of the object to db.
-// This updates the trie root.
-func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) {
+// commitTrie submits the storage changes into the storage trie and re-computes
+// the root. Besides, all trie changes will be collected in a nodeset and returned.
+func (s *stateObject) commitTrie(db Database) (*trie.NodeSet, error) {
 	// If nothing changed, don't bother with hashing anything
 	if s.updateTrie(db) == nil {
 		return nil, nil

core/state/statedb.go

@@ -986,7 +986,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
 				obj.dirtyCode = false
 			}
 			// Write any storage changes in the state object to its storage trie
-			nodeSet, err := obj.CommitTrie(s.db)
+			nodeSet, err := obj.commitTrie(s.db)
 			if err != nil {
 				return common.Hash{}, err
 			}
@@ -1000,6 +1000,12 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
 			}
 		}
 	}
+	// If the contract is destructed, the storage is still left in the
+	// database as dangling data. Theoretically it's should be wiped from
+	// database as well, but in hash-based-scheme it's extremely hard to
+	// determine that if the trie nodes are also referenced by other storage,
+	// and in path-based-scheme some technical challenges are still unsolved.
+	// Although it won't affect the correctness but please fix it TODO(rjl493456442).
 	if len(s.stateObjectsDirty) > 0 {
 		s.stateObjectsDirty = make(map[common.Address]struct{})
 	}

ethdb/database.go

@@ -87,6 +87,10 @@ type AncientReaderOp interface {
 	// Ancients returns the ancient item numbers in the ancient store.
 	Ancients() (uint64, error)
 
+	// Tail returns the number of first stored item in the ancient store.
+	// This number can also be interpreted as the total deleted items.
+	Tail() (uint64, error)
+
 	// AncientSize returns the ancient size of the specified category.
 	AncientSize(kind string) (uint64, error)
 }