Skip to content

Commit

Permalink
[db]log mptrie node (#3634)
Browse files Browse the repository at this point in the history
Co-authored-by: huofei <[email protected]>
Co-authored-by: Jeremy Chou <[email protected]>
Co-authored-by: CoderZhi <[email protected]>
  • Loading branch information
4 people authored Jan 13, 2023
1 parent 22bf29b commit dab026e
Show file tree
Hide file tree
Showing 8 changed files with 317 additions and 5 deletions.
18 changes: 18 additions & 0 deletions db/trie/mptrie/branchnode.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ func newBranchNode(
}
}
}
if err := logNode(_nodeTypeBranch, _actionTypeNew, bnode, cli); err != nil {
return nil, err
}
return bnode, nil
}

Expand All @@ -69,6 +72,9 @@ func newRootBranchNode(cli client, children map[byte]node, indices *SortedList,
}
}
}
if err := logNode(_nodeTypeBranch, _actionTypeNew, bnode, cli); err != nil {
return nil, err
}
return bnode, nil
}

Expand All @@ -85,6 +91,9 @@ func newBranchNodeFromProtoPb(pb *triepb.BranchPb, hashVal []byte) *branchNode {
}
bnode.indices = NewSortedList(bnode.children)
bnode.cacheNode.serializable = bnode
if err := logNode(_nodeTypeBranch, _actionTypeNew, bnode, nil); err != nil {
panic(err)
}
return bnode
}

Expand All @@ -101,6 +110,9 @@ func (b *branchNode) Children() []node {
}

func (b *branchNode) Delete(cli client, key keyType, offset uint8) (node, error) {
if err := logNode(_nodeTypeBranch, _actionTypeDelete, b, cli); err != nil {
return nil, err
}
offsetKey := key[offset]
child, err := b.child(offsetKey)
if err != nil {
Expand Down Expand Up @@ -154,6 +166,9 @@ func (b *branchNode) Delete(cli client, key keyType, offset uint8) (node, error)
}

func (b *branchNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) {
if err := logNode(_nodeTypeBranch, _actionTypeUpsert, b, cli); err != nil {
return nil, err
}
var newChild node
offsetKey := key[offset]
child, err := b.child(offsetKey)
Expand All @@ -171,6 +186,9 @@ func (b *branchNode) Upsert(cli client, key keyType, offset uint8, value []byte)
}

func (b *branchNode) Search(cli client, key keyType, offset uint8) (node, error) {
if err := logNode(_nodeTypeBranch, _actionTypeSearch, b, cli); err != nil {
return nil, err
}
child, err := b.child(key[offset])
if err != nil {
return nil, err
Expand Down
9 changes: 5 additions & 4 deletions db/trie/mptrie/branchnode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,11 @@ func equals(bn *branchNode, clone *branchNode) bool {

func TestBranchNodeClone(t *testing.T) {
require := require.New(t)
cli := &merklePatriciaTrie{async: true, hashFunc: DefaultHashFunc}
t.Run("dirty empty root", func(t *testing.T) {
children := map[byte]node{}
indices := NewSortedList(children)
node, err := newRootBranchNode(nil, children, indices, true)
node, err := newRootBranchNode(cli, children, indices, true)
require.NoError(err)
bn, ok := node.(*branchNode)
require.True(ok)
Expand All @@ -64,7 +65,7 @@ func TestBranchNodeClone(t *testing.T) {
t.Run("clean empty root", func(t *testing.T) {
children := map[byte]node{}
indices := NewSortedList(children)
node, err := newRootBranchNode(nil, children, indices, false)
node, err := newRootBranchNode(cli, children, indices, false)
require.NoError(err)
bn, ok := node.(*branchNode)
require.True(ok)
Expand All @@ -81,7 +82,7 @@ func TestBranchNodeClone(t *testing.T) {
children['c'] = &hashNode{hashVal: []byte("c")}
children['d'] = &hashNode{hashVal: []byte("d")}
indices := NewSortedList(children)
node, err := newBranchNode(&merklePatriciaTrie{async: true}, children, indices)
node, err := newBranchNode(cli, children, indices)
require.NoError(err)
bn, ok := node.(*branchNode)
require.True(ok)
Expand All @@ -105,7 +106,7 @@ func TestBranchNodeProto(t *testing.T) {
children: children,
indices: indices,
}
cli := &merklePatriciaTrie{async: true}
cli := &merklePatriciaTrie{async: true, hashFunc: DefaultHashFunc}
proto, err := bnode.proto(cli, true)
require.NoError(err)
nodepb, ok := proto.(*triepb.NodePb)
Expand Down
5 changes: 5 additions & 0 deletions db/trie/mptrie/cachenode.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
package mptrie

import (
"errors"

"google.golang.org/protobuf/proto"
)

Expand All @@ -24,6 +26,9 @@ func (cn *cacheNode) hash(cli client, flush bool) ([]byte, error) {
if len(cn.hashVal) != 0 {
return cn.hashVal, nil
}
if cli == nil {
return []byte{}, errors.New("client cannot be nil")
}
pb, err := cn.proto(cli, flush)
if err != nil {
return nil, err
Expand Down
15 changes: 15 additions & 0 deletions db/trie/mptrie/extensionnode.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ func newExtensionNode(
return nil, err
}
}
if err := logNode(_nodeTypeExtension, _actionTypeNew, e, cli); err != nil {
return nil, err
}
return e, nil
}

Expand All @@ -51,10 +54,16 @@ func newExtensionNodeFromProtoPb(pb *triepb.ExtendPb, hashVal []byte) *extension
child: newHashNode(pb.Value),
}
e.cacheNode.serializable = e
if err := logNode(_nodeTypeExtension, _actionTypeNew, e, nil); err != nil {
panic(err)
}
return e
}

func (e *extensionNode) Delete(cli client, key keyType, offset uint8) (node, error) {
if err := logNode(_nodeTypeExtension, _actionTypeDelete, e, cli); err != nil {
return nil, err
}
matched := e.commonPrefixLength(key[offset:])
if matched != uint8(len(e.path)) {
return nil, trie.ErrNotExist
Expand Down Expand Up @@ -85,6 +94,9 @@ func (e *extensionNode) Delete(cli client, key keyType, offset uint8) (node, err
}

func (e *extensionNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) {
if err := logNode(_nodeTypeExtension, _actionTypeUpsert, e, cli); err != nil {
return nil, err
}
matched := e.commonPrefixLength(key[offset:])
if matched == uint8(len(e.path)) {
newChild, err := e.child.Upsert(cli, key, offset+matched, value)
Expand Down Expand Up @@ -120,6 +132,9 @@ func (e *extensionNode) Upsert(cli client, key keyType, offset uint8, value []by
}

func (e *extensionNode) Search(cli client, key keyType, offset uint8) (node, error) {
if err := logNode(_nodeTypeExtension, _actionTypeSearch, e, cli); err != nil {
return nil, err
}
matched := e.commonPrefixLength(key[offset:])
if matched != uint8(len(e.path)) {
return nil, trie.ErrNotExist
Expand Down
17 changes: 16 additions & 1 deletion db/trie/mptrie/leafnode.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ func newLeafNode(
return nil, err
}
}
if err := logNode(_nodeTypeLeaf, _actionTypeNew, l, cli); err != nil {
return nil, err
}
return l, nil
}

Expand All @@ -51,6 +54,9 @@ func newLeafNodeFromProtoPb(pb *triepb.LeafPb, hashVal []byte) *leafNode {
value: pb.Value,
}
l.cacheNode.serializable = l
if err := logNode(_nodeTypeLeaf, _actionTypeNew, l, nil); err != nil {
panic(err)
}
return l
}

Expand All @@ -63,13 +69,19 @@ func (l *leafNode) Value() []byte {
}

func (l *leafNode) Delete(cli client, key keyType, offset uint8) (node, error) {
if err := logNode(_nodeTypeLeaf, _actionTypeDelete, l, cli); err != nil {
return nil, err
}
if !bytes.Equal(l.key[offset:], key[offset:]) {
return nil, trie.ErrNotExist
}
return nil, l.delete(cli)
}

func (l *leafNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) {
if err := logNode(_nodeTypeLeaf, _actionTypeUpsert, l, cli); err != nil {
return nil, err
}
matched := commonPrefixLength(l.key[offset:], key[offset:])
if offset+matched == uint8(len(key)) {
if err := l.delete(cli); err != nil {
Expand Down Expand Up @@ -106,7 +118,10 @@ func (l *leafNode) Upsert(cli client, key keyType, offset uint8, value []byte) (
return newExtensionNode(cli, l.key[offset:offset+matched], bnode)
}

func (l *leafNode) Search(_ client, key keyType, offset uint8) (node, error) {
func (l *leafNode) Search(cli client, key keyType, offset uint8) (node, error) {
if err := logNode(_nodeTypeLeaf, _actionTypeSearch, l, cli); err != nil {
return nil, err
}
if !bytes.Equal(l.key[offset:], key[offset:]) {
return nil, trie.ErrNotExist
}
Expand Down
127 changes: 127 additions & 0 deletions db/trie/mptrie/lognode.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
package mptrie

import (
"bufio"
"os"

"github.com/pkg/errors"
)

var (
enabledLogMptrie = false
logFile *os.File
logWriter *bufio.Writer
)

type nodeType byte
type actionType byte

const (
_nodeTypeLeaf nodeType = 'l'
_nodeTypeExtension nodeType = 'e'
_nodeTypeBranch nodeType = 'b'

_actionTypeSearch actionType = 's'
_actionTypeUpsert actionType = 'u'
_actionTypeDelete actionType = 'd'
_actionTypeNew actionType = 'n'
)

// nodeEvent is the event of node
type nodeEvent struct {
NodeType nodeType
ActionType actionType
KeyLen uint8
Key []byte
PathLen uint8
Path []byte
ChildrenLen uint8
Children []byte
HashLen uint8
HashVal []byte
}

// Bytes returns the bytes of node event
func (e nodeEvent) Bytes() []byte {
b := make([]byte, 0, 1+1+1+e.KeyLen+1+e.PathLen+1+e.ChildrenLen+1+e.HashLen)
b = append(b, byte(e.NodeType))
b = append(b, byte(e.ActionType))
b = append(b, e.KeyLen)
b = append(b, e.Key...)
b = append(b, e.PathLen)
b = append(b, e.Path...)
b = append(b, e.ChildrenLen)
b = append(b, e.Children...)
b = append(b, e.HashLen)
b = append(b, e.HashVal...)
return b
}

// OpenLogDB open the log DB file
func OpenLogDB(dbPath string) error {
var err error
logFile, err = os.OpenFile(dbPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return err
}
logWriter = bufio.NewWriter(logFile)
enabledLogMptrie = true
return nil
}

// CloseLogDB close the log DB file
func CloseLogDB() error {
if !enabledLogMptrie {
return nil
}
if err := logWriter.Flush(); err != nil {
return err
}
return logFile.Close()
}

func logNode(nt nodeType, at actionType, n node, cli client) error {
if !enabledLogMptrie {
return nil
}
nodeKey, nodePath, nodeChildren, hashvalue, err := parseNode(n, cli)
if err != nil {
return err
}
event := nodeEvent{
NodeType: nt,
ActionType: at,
KeyLen: uint8(len(nodeKey)),
Key: nodeKey,
PathLen: uint8(len(nodePath)),
Path: nodePath,
ChildrenLen: uint8(len(nodeChildren)),
Children: nodeChildren,
HashLen: uint8(len(hashvalue)),
HashVal: hashvalue,
}
// write event length
if err = logWriter.WriteByte(byte(len(event.Bytes()))); err != nil {
return err
}
// write event body
_, err = logWriter.Write(event.Bytes())
return err
}

func parseNode(n node, cli client) (nodeKey, nodePath, nodeChildren, hashvalue []byte, err error) {
switch n := n.(type) {
case *leafNode:
nodeKey = n.key
hashvalue, err = n.cacheNode.Hash(cli)
case *extensionNode:
nodePath = n.path
hashvalue, err = n.cacheNode.Hash(cli)
case *branchNode:
nodeChildren = n.indices.List()
hashvalue, err = n.cacheNode.Hash(cli)
default:
err = errors.Errorf("unknown node type %T", n)
}
return
}
Loading

0 comments on commit dab026e

Please sign in to comment.