Skip to content

Commit

Permalink
core/rawdb: refactor db inspector for extending multiple ancient store
Browse files Browse the repository at this point in the history
  • Loading branch information
huyngopt1994 committed Sep 17, 2024
1 parent 1faf71b commit 5cff5c8
Show file tree
Hide file tree
Showing 7 changed files with 170 additions and 61 deletions.
35 changes: 0 additions & 35 deletions core/rawdb/ancient_scheme.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@

package rawdb

import "fmt"

// The list of table names of chain freezer. (headers, hashes, bodies, difficulties)

const (
Expand Down Expand Up @@ -54,36 +52,3 @@ var (

// freezers the collections of all builtin freezers.
var freezers = []string{chainFreezerName}

// InspectFreezerTable dumps out the index of a specific freezer table. The passed
// ancient indicates the path of root ancient directory where the chain freezer can
// be opened. Start and end specifiy the range for dumping out indexes.
// Note this function can only used for debugging purpose.
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
var (
path string
tables map[string]bool
)

switch freezerName {
case chainFreezerName:
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
default:
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
}
noSnappy, exit := tables[tableName]
if !exit {
// If the tableName is not exit in the tables, return an error.
var names []string
for name := range tables {
names = append(names, name)
}
return fmt.Errorf("unknown table name, supported ones: %v", names)
}
table, err := newFreezerTable(path, tableName, noSnappy)
if err != nil {
return err
}
table.dumpIndexStdout(start, end)
return nil
}
125 changes: 125 additions & 0 deletions core/rawdb/ancient_utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package rawdb

import (
"fmt"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
)

type tableSize struct {
name string
size common.StorageSize
}

// freezerInfo contains the basic information of the freezer.
type freezerInfo struct {
name string // The identifier of freezer
head uint64 // The number of last stored item in the freezer
tail uint64 // The number of first stored item in the freezer
sizes []tableSize // The storage size per table
}

// count returns the number of stored items in the freezer.
func (info *freezerInfo) count() uint64 {
return info.head - info.tail + 1
}

// size returns the storage size of the entire freezer.
func (info *freezerInfo) size() common.StorageSize {
var total common.StorageSize
for _, table := range info.sizes {
total += table.size
}
return total
}

// inspectFreezers inspects all freezers registered in the system.
func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
var (
infos []freezerInfo
)
for _, freezer := range freezers {
switch freezer {
case chainFreezerName: // We only support chain freezer for now.
// Chain ancient store is a bit special. It's always opened along
// with a key-value store, inspect the chain store directly.
info := freezerInfo{name: freezer}
// Retrieve storage size of every contained table.
for table := range chainFreezerNoSnappy {
size, err := db.AncientSize(table)
if err != nil {
return nil, err
}
info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
}
// Retrieve the number of last stored item
ancients, err := db.Ancients()
if err != nil {
return nil, err
}
info.head = ancients - 1

// Retrieve the number of first stored item
tail, err := db.Tail()
if err != nil {
return nil, err
}
info.tail = tail
infos = append(infos, info)
default:
return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers)
}
}
return infos, nil
}

// InspectFreezerTable dumps out the index of a specific freezer table. The passed
// ancient indicates the path of root ancient directory where the chain freezer can
// be opened. Start and end specifiy the range for dumping out indexes.
// Note this function can only used for debugging purpose.
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
var (
path string
tables map[string]bool
)

switch freezerName {
case chainFreezerName:
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
default:
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
}
noSnappy, exit := tables[tableName]
if !exit {
// If the tableName is not exit in the tables, return an error.
var names []string
for name := range tables {
names = append(names, name)
}
return fmt.Errorf("unknown table name, supported ones: %v", names)
}
table, err := newFreezerTable(path, tableName, noSnappy)
if err != nil {
return err
}

table.dumpIndexStdout(start, end)
return nil
}
5 changes: 5 additions & 0 deletions core/rawdb/chain_freezer.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@ func (f *chainFreezer) Close() error {
return err
}

// Tail returns an error as we don't have a backing chain freezer.
func (f *chainFreezer) Tail() (uint64, error) {
return 0, errNotSupported
}

// freeze is a background thread that periodically checks the blockchain for any
// import progress and moves ancient data from the fast database into the freezer.
//
Expand Down
50 changes: 24 additions & 26 deletions core/rawdb/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"os"
"path"
"path/filepath"
"strings"
"time"

"github.com/ethereum/go-ethereum/common"
Expand Down Expand Up @@ -87,6 +88,11 @@ type nofreezedb struct {
ethdb.KeyValueStore
}

// Tail returns an error as we don't have a backing chain freezer.
func (db *nofreezedb) Tail() (uint64, error) {
return 0, errNotSupported
}

// HasAncient returns an error as we don't have a backing chain freezer.
func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) {
return false, errNotSupported
Expand Down Expand Up @@ -445,13 +451,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
cliqueSnaps stat
consortiumSnaps stat

// Ancient store statistics
ancientHeadersSize common.StorageSize
ancientBodiesSize common.StorageSize
ancientReceiptsSize common.StorageSize
ancientTdsSize common.StorageSize
ancientHashesSize common.StorageSize

// Les statistic
chtTrieNodes stat
bloomTrieNodes stat
Expand Down Expand Up @@ -540,20 +539,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
logged = time.Now()
}
}
// Inspect append-only file store then.
ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} {
if size, err := db.AncientSize(category); err == nil {
*ancientSizes[i] += common.StorageSize(size)
total += common.StorageSize(size)
}
}
// Get number of ancient rows inside the freezer
ancients := counter(0)
if count, err := db.Ancients(); err == nil {
ancients = counter(count)
}
// Display the database statistic.

// Display the database statistic of key-value store.
stats := [][]string{
{"Key-Value store", "Headers", headers.Size(), headers.Count()},
{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
Expand All @@ -571,14 +558,25 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
{"Key-Value store", "Consortium snapshots", consortiumSnaps.Size(), consortiumSnaps.Count()},
{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
{"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()},
{"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()},
{"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()},
{"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()},
{"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()},
{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
}
// Inspect all registered append-only file store then.
ancients, err := inspectFreezers(db)
if err != nil {
return err
}
for _, ancient := range ancients {
for _, table := range ancient.sizes {
stats = append(stats, []string{
fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
strings.Title(table.name),
table.size.String(),
fmt.Sprintf("%d", ancient.count()),
})
}
total += ancient.size()
}
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Database", "Category", "Size", "Items"})
table.SetFooter([]string{"", "Total", total.String(), " "})
Expand Down
6 changes: 6 additions & 0 deletions core/rawdb/freezer.go
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,12 @@ func (f *Freezer) AncientSize(kind string) (uint64, error) {
return 0, errUnknownTable
}

// Tail returns an error as we don't have a backing chain freezer.
func (f *Freezer) Tail() (uint64, error) {
// return f.tail.Load(), nil, in the next implementing, right now just keep it zero
return 0, nil
}

// ReadAncients runs the given read operation while ensuring that no writes take place
// on the underlying freezer.
func (f *Freezer) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
Expand Down
6 changes: 6 additions & 0 deletions core/rawdb/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@ func (t *table) AncientRange(kind string, start, count, maxBytes uint64) ([][]by
return t.db.AncientRange(kind, start, count, maxBytes)
}

// Tail is a noop passthrough that just forwards the request to the underlying
// database.
func (t *table) Tail() (uint64, error) {
return t.db.Tail()
}

// Ancients is a noop passthrough that just forwards the request to the underlying
// database.
func (t *table) Ancients() (uint64, error) {
Expand Down
4 changes: 4 additions & 0 deletions ethdb/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ type AncientReaderOp interface {
// Ancients returns the ancient item numbers in the ancient store.
Ancients() (uint64, error)

// Tail returns the number of first stored item in the ancient store.
// This number can also be interpreted as the total deleted items.
Tail() (uint64, error)

// AncientSize returns the ancient size of the specified category.
AncientSize(kind string) (uint64, error)
}
Expand Down

0 comments on commit 5cff5c8

Please sign in to comment.