Merge pull request #1882 from Fynnss/add_hash_trie_node_prune_tool

cmd/geth: add hbss to pbss convert tool

cmd/geth/dbcmd.go

@@ -19,6 +19,7 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"math"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -69,6 +70,10 @@ Remove blockchain and state databases`,
 			dbExportCmd,
 			dbMetadataCmd,
 			dbCheckStateContentCmd,
+			dbHbss2PbssCmd,
+			dbPruneHashTrieCmd,
+			dbTrieGetCmd,
+			dbTrieDeleteCmd,
 		},
 	}
 	dbInspectCmd = &cli.Command{
@@ -91,6 +96,54 @@ Remove blockchain and state databases`,
 For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates
 a data corruption.`,
 	}
+	dbHbss2PbssCmd = &cli.Command{
+		Action:    hbss2pbss,
+		Name:      "hbss-to-pbss",
+		ArgsUsage: "<jobnum (optional)>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+		},
+		Usage:       "Convert Hash-Base to Path-Base trie node.",
+		Description: `This command iterates the entire trie node database and convert the hash-base node to path-base node.`,
+	}
+	dbTrieGetCmd = &cli.Command{
+		Action:    dbTrieGet,
+		Name:      "trie-get",
+		Usage:     "Show the value of a trie node path key",
+		ArgsUsage: "[trie owner] <path-base key>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.StateSchemeFlag,
+		},
+		Description: "This command looks up the specified trie node key from the database.",
+	}
+	dbTrieDeleteCmd = &cli.Command{
+		Action:    dbTrieDelete,
+		Name:      "trie-delete",
+		Usage:     "delete the specify trie node",
+		ArgsUsage: "[trie owner] <hash-base key> | <path-base key>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.StateSchemeFlag,
+		},
+		Description: "This command delete the specify trie node from the database.",
+	}
+	dbPruneHashTrieCmd = &cli.Command{
+		Action:    pruneHashTrie,
+		Name:      "prune-hash-trie",
+		ArgsUsage: "",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+		},
+		Usage:       "[Caution]Prune all the hash trie node in diskdb",
+		Description: `This command iterates the entrie kv in leveldb and delete all the hash trie node.`,
+	}
 	dbStatCmd = &cli.Command{
 		Action: dbStats,
 		Name:   "stats",
@@ -410,6 +463,133 @@ func dbGet(ctx *cli.Context) error {
 	return nil
 }
 
+// dbTrieGet shows the value of a given database key
+func dbTrieGet(ctx *cli.Context) error {
+	if ctx.NArg() < 1 || ctx.NArg() > 2 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, false, false)
+	defer db.Close()
+
+	scheme := ctx.String(utils.StateSchemeFlag.Name)
+	if scheme == "" {
+		scheme = rawdb.HashScheme
+	}
+
+	if scheme == rawdb.PathScheme {
+		var (
+			pathKey []byte
+			owner   []byte
+			err     error
+		)
+		if ctx.NArg() == 1 {
+			pathKey, err = hexutil.Decode(ctx.Args().Get(0))
+			if err != nil {
+				log.Info("Could not decode the value", "error", err)
+				return err
+			}
+			nodeVal, hash := rawdb.ReadAccountTrieNode(db, pathKey)
+			log.Info("TrieGet result ", "PathKey", common.Bytes2Hex(pathKey), "Hash: ", hash, "node: ", trie.NodeString(hash.Bytes(), nodeVal))
+		} else if ctx.NArg() == 2 {
+			owner, err = hexutil.Decode(ctx.Args().Get(0))
+			if err != nil {
+				log.Info("Could not decode the value", "error", err)
+				return err
+			}
+			pathKey, err = hexutil.Decode(ctx.Args().Get(1))
+			if err != nil {
+				log.Info("Could not decode the value", "error", err)
+				return err
+			}
+
+			nodeVal, hash := rawdb.ReadStorageTrieNode(db, common.BytesToHash(owner), pathKey)
+			log.Info("TrieGet result ", "PathKey: ", common.Bytes2Hex(pathKey), "Owner: ", common.BytesToHash(owner), "Hash: ", hash, "node: ", trie.NodeString(hash.Bytes(), nodeVal))
+		}
+	} else if scheme == rawdb.HashScheme {
+		if ctx.NArg() == 1 {
+			hashKey, err := hexutil.Decode(ctx.Args().Get(0))
+			if err != nil {
+				log.Info("Could not decode the value", "error", err)
+				return err
+			}
+			val, err := db.Get(hashKey)
+			if err != nil {
+				log.Error("db get failed, ", "error: ", err)
+				return err
+			}
+			log.Info("TrieGet result ", "HashKey: ", common.BytesToHash(hashKey), "node: ", trie.NodeString(hashKey, val))
+		} else {
+			log.Error("args too much")
+		}
+	}
+
+	return nil
+}
+
+// dbTrieDelete delete the trienode of a given database key
+func dbTrieDelete(ctx *cli.Context) error {
+	if ctx.NArg() < 1 || ctx.NArg() > 2 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, false, false)
+	defer db.Close()
+
+	scheme := ctx.String(utils.StateSchemeFlag.Name)
+	if scheme == "" {
+		scheme = rawdb.HashScheme
+	}
+
+	if scheme == rawdb.PathScheme {
+		var (
+			pathKey []byte
+			owner   []byte
+			err     error
+		)
+		if ctx.NArg() == 1 {
+			pathKey, err = hexutil.Decode(ctx.Args().Get(0))
+			if err != nil {
+				log.Info("Could not decode the value", "error", err)
+				return err
+			}
+			rawdb.DeleteAccountTrieNode(db, pathKey)
+		} else if ctx.NArg() == 2 {
+			owner, err = hexutil.Decode(ctx.Args().Get(0))
+			if err != nil {
+				log.Info("Could not decode the value", "error", err)
+				return err
+			}
+			pathKey, err = hexutil.Decode(ctx.Args().Get(1))
+			if err != nil {
+				log.Info("Could not decode the value", "error", err)
+				return err
+			}
+			rawdb.DeleteStorageTrieNode(db, common.BytesToHash(owner), pathKey)
+		}
+	} else if scheme == rawdb.HashScheme {
+		if ctx.NArg() == 1 {
+			hashKey, err := hexutil.Decode(ctx.Args().Get(0))
+			if err != nil {
+				log.Info("Could not decode the value", "error", err)
+				return err
+			}
+			err = db.Delete(hashKey)
+			if err != nil {
+				log.Error("db delete failed", "err", err)
+				return err
+			}
+		} else {
+			log.Error("args too much")
+		}
+	}
+	return nil
+}
+
 // dbDelete deletes a key from the database
 func dbDelete(ctx *cli.Context) error {
 	if ctx.NArg() != 1 {
@@ -724,3 +904,90 @@ func showMetaData(ctx *cli.Context) error {
 	table.Render()
 	return nil
 }
+
+func hbss2pbss(ctx *cli.Context) error {
+	if ctx.NArg() > 1 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+
+	var jobnum uint64
+	var err error
+	if ctx.NArg() == 1 {
+		jobnum, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64)
+		if err != nil {
+			return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err)
+		}
+	} else {
+		// by default
+		jobnum = 1000
+	}
+
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, false, false)
+	db.Sync()
+	defer db.Close()
+
+	config := trie.HashDefaults
+	triedb := trie.NewDatabase(db, config)
+	triedb.Cap(0)
+	log.Info("hbss2pbss triedb", "scheme", triedb.Scheme())
+	defer triedb.Close()
+
+	headerHash := rawdb.ReadHeadHeaderHash(db)
+	blockNumber := rawdb.ReadHeaderNumber(db, headerHash)
+	if blockNumber == nil {
+		log.Error("read header number failed.")
+		return fmt.Errorf("read header number failed")
+	}
+
+	log.Info("hbss2pbss converting", "HeaderHash: ", headerHash.String(), ", blockNumber: ", *blockNumber)
+
+	var headerBlockHash common.Hash
+	var trieRootHash common.Hash
+
+	if *blockNumber != math.MaxUint64 {
+		headerBlockHash = rawdb.ReadCanonicalHash(db, *blockNumber)
+		if headerBlockHash == (common.Hash{}) {
+			return fmt.Errorf("ReadHeadBlockHash empty hash")
+		}
+		blockHeader := rawdb.ReadHeader(db, headerBlockHash, *blockNumber)
+		trieRootHash = blockHeader.Root
+		fmt.Println("Canonical Hash: ", headerBlockHash.String(), ", TrieRootHash: ", trieRootHash.String())
+	}
+	if (trieRootHash == common.Hash{}) {
+		log.Error("Empty root hash")
+		return fmt.Errorf("Empty root hash.")
+	}
+
+	id := trie.StateTrieID(trieRootHash)
+	theTrie, err := trie.New(id, triedb)
+	if err != nil {
+		log.Error("fail to new trie tree", "err", err, "rootHash", err, trieRootHash.String())
+		return err
+	}
+
+	h2p, err := trie.NewHbss2Pbss(theTrie, triedb, trieRootHash, *blockNumber, jobnum)
+	if err != nil {
+		log.Error("fail to new hash2pbss", "err", err, "rootHash", err, trieRootHash.String())
+		return err
+	}
+	h2p.Run()
+
+	return nil
+}
+
+func pruneHashTrie(ctx *cli.Context) error {
+	if ctx.NArg() != 0 {
+		return fmt.Errorf("required none argument")
+	}
+
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, false, false)
+	defer db.Close()
+
+	return rawdb.PruneHashTrieNodeInDataBase(db)
+}

core/rawdb/database.go

@@ -449,6 +449,28 @@ func (s *stat) Count() string {
 	return s.count.String()
 }
 
+func PruneHashTrieNodeInDataBase(db ethdb.Database) error {
+	it := db.NewIterator([]byte{}, []byte{})
+	defer it.Release()
+
+	total_num := 0
+	for it.Next() {
+		var key = it.Key()
+		switch {
+		case IsLegacyTrieNode(key, it.Value()):
+			db.Delete(key)
+			total_num++
+			if total_num%100000 == 0 {
+				log.Info("Pruning ", "Complete progress: ", total_num, "hash-base trie nodes")
+			}
+		default:
+			continue
+		}
+	}
+	log.Info("Pruning ", "Complete progress", total_num, "hash-base trie nodes")
+	return nil
+}
+
 // InspectDatabase traverses the entire database and checks the size
 // of all different categories of data.
 func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {