feat: refactor the node key as version + local nonce(seq id)

Makefile

@@ -53,18 +53,18 @@ lint-fix:
 # bench is the basic tests that shouldn't crash an aws instance
 bench:
 	cd benchmarks && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -bench=Small . && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -bench=Medium . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -bench=Small . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -bench=Medium . && \
 		go test $(LDFLAGS) -run=NOTEST -bench=RandomBytes .
 .PHONY: bench
 
 # fullbench is extra tests needing lots of memory and to run locally
 fullbench:
 	cd benchmarks && \
 		go test $(LDFLAGS) -run=NOTEST -bench=RandomBytes . && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -bench=Small . && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -bench=Medium . && \
-		go test $(LDFLAGS) -tags cleveldb,rocksdb,pebbledb -run=NOTEST -timeout=30m -bench=Large . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -bench=Small . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -bench=Medium . && \
+		go test $(LDFLAGS) -tags pebbledb -run=NOTEST -timeout=30m -bench=Large . && \
 		go test $(LDFLAGS) -run=NOTEST -bench=Mem . && \
 		go test $(LDFLAGS) -run=NOTEST -timeout=60m -bench=LevelDB .
 .PHONY: fullbench

basic_test.go

@@ -2,7 +2,6 @@
 package iavl
 
 import (
-	"bytes"
 	"encoding/hex"
 	mrand "math/rand"
 	"sort"
@@ -171,52 +170,28 @@ func TestBasic(t *testing.T) {
 }
 
 func TestUnit(t *testing.T) {
-	expectHash := func(tree *ImmutableTree, hashCount int64) {
-		// ensure number of new hash calculations is as expected.
-		hash, count, err := tree.root.hashWithCount()
-		require.NoError(t, err)
-		if count != hashCount {
-			t.Fatalf("Expected %v new hashes, got %v", hashCount, count)
-		}
-		// nuke hashes and reconstruct hash, ensure it's the same.
-		tree.root.traverse(tree, true, func(node *Node) bool {
-			node.hash = nil
-			return false
-		})
-		// ensure that the new hash after nuking is the same as the old.
-		newHash, _, err := tree.root.hashWithCount()
-		require.NoError(t, err)
-		if !bytes.Equal(hash, newHash) {
-			t.Fatalf("Expected hash %v but got %v after nuking", hash, newHash)
-		}
-	}
-
-	expectSet := func(tree *MutableTree, i int, repr string, hashCount int64) {
-		origNode := tree.root
+	expectSet := func(tree *MutableTree, i int, repr string) {
+		tree.SaveVersion()
 		updated, err := tree.Set(i2b(i), []byte{})
 		require.NoError(t, err)
 		// ensure node was added & structure is as expected.
-		if updated || P(tree.root) != repr {
+		if updated || P(tree.root, tree.ImmutableTree) != repr {
 			t.Fatalf("Adding %v to %v:\nExpected         %v\nUnexpectedly got %v updated:%v",
-				i, P(origNode), repr, P(tree.root), updated)
+				i, P(tree.lastSaved.root, tree.lastSaved), repr, P(tree.root, tree.ImmutableTree), updated)
 		}
-		// ensure hash calculation requirements
-		expectHash(tree.ImmutableTree, hashCount)
-		tree.root = origNode
+		tree.ImmutableTree = tree.lastSaved.clone()
 	}
 
-	expectRemove := func(tree *MutableTree, i int, repr string, hashCount int64) {
-		origNode := tree.root
+	expectRemove := func(tree *MutableTree, i int, repr string) {
+		tree.SaveVersion()
 		value, removed, err := tree.Remove(i2b(i))
 		require.NoError(t, err)
 		// ensure node was added & structure is as expected.
-		if len(value) != 0 || !removed || P(tree.root) != repr {
+		if len(value) != 0 || !removed || P(tree.root, tree.ImmutableTree) != repr {
 			t.Fatalf("Removing %v from %v:\nExpected         %v\nUnexpectedly got %v value:%v removed:%v",
-				i, P(origNode), repr, P(tree.root), value, removed)
+				i, P(tree.lastSaved.root, tree.lastSaved), repr, P(tree.root, tree.ImmutableTree), value, removed)
 		}
-		// ensure hash calculation requirements
-		expectHash(tree.ImmutableTree, hashCount)
-		tree.root = origNode
+		tree.ImmutableTree = tree.lastSaved.clone()
 	}
 
 	// Test Set cases:
@@ -225,40 +200,40 @@ func TestUnit(t *testing.T) {
 	t1, err := T(N(4, 20))
 
 	require.NoError(t, err)
-	expectSet(t1, 8, "((4 8) 20)", 3)
-	expectSet(t1, 25, "(4 (20 25))", 3)
+	expectSet(t1, 8, "((4 8) 20)")
+	expectSet(t1, 25, "(4 (20 25))")
 
 	t2, err := T(N(4, N(20, 25)))
 
 	require.NoError(t, err)
-	expectSet(t2, 8, "((4 8) (20 25))", 3)
-	expectSet(t2, 30, "((4 20) (25 30))", 4)
+	expectSet(t2, 8, "((4 8) (20 25))")
+	expectSet(t2, 30, "((4 20) (25 30))")
 
 	t3, err := T(N(N(1, 2), 6))
 
 	require.NoError(t, err)
-	expectSet(t3, 4, "((1 2) (4 6))", 4)
-	expectSet(t3, 8, "((1 2) (6 8))", 3)
+	expectSet(t3, 4, "((1 2) (4 6))")
+	expectSet(t3, 8, "((1 2) (6 8))")
 
 	t4, err := T(N(N(1, 2), N(N(5, 6), N(7, 9))))
 
 	require.NoError(t, err)
-	expectSet(t4, 8, "(((1 2) (5 6)) ((7 8) 9))", 5)
-	expectSet(t4, 10, "(((1 2) (5 6)) (7 (9 10)))", 5)
+	expectSet(t4, 8, "(((1 2) (5 6)) ((7 8) 9))")
+	expectSet(t4, 10, "(((1 2) (5 6)) (7 (9 10)))")
 
 	// Test Remove cases:
 
 	t10, err := T(N(N(1, 2), 3))
 
 	require.NoError(t, err)
-	expectRemove(t10, 2, "(1 3)", 1)
-	expectRemove(t10, 3, "(1 2)", 0)
+	expectRemove(t10, 2, "(1 3)")
+	expectRemove(t10, 3, "(1 2)")
 
 	t11, err := T(N(N(N(1, 2), 3), N(4, 5)))
 
 	require.NoError(t, err)
-	expectRemove(t11, 4, "((1 2) (3 5))", 2)
-	expectRemove(t11, 3, "((1 2) (4 5))", 1)
+	expectRemove(t11, 4, "((1 2) (3 5))")
+	expectRemove(t11, 3, "((1 2) (4 5))")
 }
 
 func TestRemove(t *testing.T) {

diff.go

@@ -25,7 +25,7 @@ type KVPairReceiver func(pair *KVPair) error
 //
 // The algorithm don't run in constant memory strictly, but it tried the best the only
 // keep minimal intermediate states in memory.
-func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root []byte, receiver KVPairReceiver) error {
+func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot *NodeKey, root *NodeKey, receiver KVPairReceiver) error {
 	curIter, err := NewNodeIterator(root, ndb)
 	if err != nil {
 		return err
@@ -70,7 +70,7 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root
 		sharedNode = nil
 		for curIter.Valid() {
 			node := curIter.GetNode()
-			shared := node.version <= prevVersion
+			shared := node.nodeKey.version <= prevVersion
 			curIter.Next(shared)
 			if shared {
 				sharedNode = node

docs/node/key_format.md

@@ -1,36 +1,15 @@
 # Key Format
 
-Nodes, orphans, and roots are stored under the database with different key formats to ensure there are no key collisions and a structured key from which we can extract useful information.
+Nodes and fastNodes are stored under the database with different key formats to ensure there are no key collisions and a structured key from which we can extract useful information.
 
 ### Nodes
 
-Node KeyFormat: `n|<node.hash>`
+Node KeyFormat: `n|node.nodeKey.version|node.nodeKey.nonce`
 
 Nodes are marshalled and stored under nodekey with prefix `n` to prevent collisions and then appended with the node's hash.
 
-### Orphans
+### FastNodes
 
-Orphan KeyFormat: `o|toVersion|fromVersion|hash`
+FastNode KeyFormat: `f|node.key`
 
-Orphans are marshalled nodes stored with prefix `o` to prevent collisions. You can extract the toVersion, fromVersion and hash from the orphan key by using:
-
-```golang
-// orphanKey: o|50|30|0xABCD
-var toVersion, fromVersion int64
-var hash []byte
-orphanKeyFormat.Scan(orphanKey, &toVersion, &fromVersion, hash)
-
-/*
-toVersion = 50
-fromVersion = 30
-hash = 0xABCD
-*/
-```
-
-The order of the orphan KeyFormat matters. Since deleting a version `v` will delete all orphans whose `toVersion = v`, we can easily retrieve all orphans from nodeDb by iterating over the key prefix: `o|v`.
-
-### Roots
-
-Root KeyFormat: `r|<version>`
-
-Root hash of the IAVL tree at version `v` is stored under the key `r|v` (prefixed with `r` to avoid collision).
+FastNodes are marshalled nodes stored with prefix `f` to prevent collisions. You can extract fast nodes from the database by iterating over the keys with prefix `f`.

docs/node/node.md

@@ -5,74 +5,90 @@ The Node struct stores a node in the IAVL tree.
 ### Structure
 
 ```golang
+// NodeKey represents a key of node in the DB.
+type NodeKey struct {
+	version int64	// version of the IAVL that this node was first added in
+	nonce   int32	// local nonce for the same version
+}
+
 // Node represents a node in a Tree.
 type Node struct {
-	key       []byte // key for the node.
-	value     []byte // value of leaf node. If inner node, value = nil
-	version   int64  // The version of the IAVL that this node was first added in.
-	height    int8   // The height of the node. Leaf nodes have height 0
-	size      int64  // The number of leaves that are under the current node. Leaf nodes have size = 1
-	hash      []byte // hash of above field and leftHash, rightHash
-	leftHash  []byte // hash of left child
-	leftNode  *Node  // pointer to left child
-        rightHash []byte // hash of right child
-	rightNode *Node  // pointer to right child
-	persisted bool   // persisted to disk
+	key           []byte	// key for the node.
+	value         []byte	// value of leaf node. If inner node, value = nil
+	hash          []byte	// hash of above field and left node's hash, right node's hash
+	nodeKey       *NodeKey	// node key of the nodeDB
+	leftNodeKey   *NodeKey	// node key of the left child
+	rightNodeKey  *NodeKey	// node key of the right child
+	size          int64		// number of leaves that are under the current node. Leaf nodes have size = 1
+	leftNode      *Node		// pointer to left child
+	rightNode     *Node		// pointer to right child
+	subtreeHeight int8		// height of the node. Leaf nodes have height 0
 }
 ```
 
-Inner nodes have keys equal to the highest key on their left branch and have values set to nil.
+Inner nodes have keys equal to the highest key on the subtree and have values set to nil.
 
 The version of a node is the first version of the IAVL tree that the node gets added in. Future versions of the IAVL may point to this node if they also contain the node, however the node's version itself does not change.
 
 Size is the number of leaves under a given node. With a full subtree, `node.size = 2^(node.height)`.
 
 ### Marshaling 
 
-Every node is persisted by encoding the key, version, height, size and hash. If the node is a leaf node, then the value is persisted as well. If the node is not a leaf node, then the leftHash and rightHash are persisted as well.
+Every node is persisted by encoding the key, height, and size. If the node is a leaf node, then the value is persisted as well. If the node is not a leaf node, then the hash, leftNodeKey and rightNodeKey are persisted as well.
 
 ```golang
 // Writes the node as a serialized byte slice to the supplied io.Writer.
 func (node *Node) writeBytes(w io.Writer) error {
-	cause := encodeVarint(w, node.height)
-	if cause != nil {
-		return errors.Wrap(cause, "writing height")
+	if node == nil {
+		return errors.New("cannot write nil node")
 	}
-	cause = encodeVarint(w, node.size)
+	cause := encoding.EncodeVarint(w, int64(node.subtreeHeight))
 	if cause != nil {
-		return errors.Wrap(cause, "writing size")
+		return fmt.Errorf("writing height, %w", cause)
 	}
-	cause = encodeVarint(w, node.version)
+	cause = encoding.EncodeVarint(w, node.size)
 	if cause != nil {
-		return errors.Wrap(cause, "writing version")
+		return fmt.Errorf("writing size, %w", cause)
 	}
 
-	// Unlike writeHashBytes, key is written for inner nodes.
-	cause = encodeBytes(w, node.key)
+	// Unlike writeHashByte, key is written for inner nodes.
+	cause = encoding.EncodeBytes(w, node.key)
 	if cause != nil {
-		return errors.Wrap(cause, "writing key")
+		return fmt.Errorf("writing key, %w", cause)
 	}
 
 	if node.isLeaf() {
-		cause = encodeBytes(w, node.value)
+		cause = encoding.EncodeBytes(w, node.value)
 		if cause != nil {
-			return errors.Wrap(cause, "writing value")
+			return fmt.Errorf("writing value, %w", cause)
 		}
 	} else {
-		if node.leftHash == nil {
-			panic("node.leftHash was nil in writeBytes")
+		cause = encoding.EncodeBytes(w, node.hash)
+		if cause != nil {
+			return fmt.Errorf("writing hash, %w", cause)
+		}
+		if node.leftNodeKey == nil {
+			return ErrLeftNodeKeyEmpty
 		}
-		cause = encodeBytes(w, node.leftHash)
+		cause = encoding.EncodeVarint(w, node.leftNodeKey.version)
 		if cause != nil {
-			return errors.Wrap(cause, "writing left hash")
+			return fmt.Errorf("writing the version of left node key, %w", cause)
+		}
+		cause = encoding.EncodeVarint(w, int64(node.leftNodeKey.nonce))
+		if cause != nil {
+			return fmt.Errorf("writing the nonce of left node key, %w", cause)
 		}
 
-		if node.rightHash == nil {
-			panic("node.rightHash was nil in writeBytes")
+		if node.rightNodeKey == nil {
+			return ErrRightNodeKeyEmpty
+		}
+		cause = encoding.EncodeVarint(w, node.rightNodeKey.version)
+		if cause != nil {
+			return fmt.Errorf("writing the version of right node key, %w", cause)
 		}
-		cause = encodeBytes(w, node.rightHash)
+		cause = encoding.EncodeVarint(w, int64(node.rightNodeKey.nonce))
 		if cause != nil {
-			return errors.Wrap(cause, "writing right hash")
+			return fmt.Errorf("writing the nonce of right node key, %w", cause)
 		}
 	}
 	return nil
@@ -86,45 +102,47 @@ A node's hash is calculated by hashing the height, size, and version of the node
 ```golang
 // Writes the node's hash to the given io.Writer. This function expects
 // child hashes to be already set.
-func (node *Node) writeHashBytes(w io.Writer) error {
-	err := encodeVarint(w, node.height)
+func (node *Node) writeHashBytes(w io.Writer, version int64) error {
+	err := encoding.EncodeVarint(w, int64(node.subtreeHeight))
 	if err != nil {
-		return errors.Wrap(err, "writing height")
+		return fmt.Errorf("writing height, %w", err)
 	}
-	err = encodeVarint(w, node.size)
+	err = encoding.EncodeVarint(w, node.size)
 	if err != nil {
-		return errors.Wrap(err, "writing size")
+		return fmt.Errorf("writing size, %w", err)
 	}
-	err = encodeVarint(w, node.version)
+	err = encoding.EncodeVarint(w, version)
 	if err != nil {
-		return errors.Wrap(err, "writing version")
+		return fmt.Errorf("writing version, %w", err)
 	}
 
 	// Key is not written for inner nodes, unlike writeBytes.
 
 	if node.isLeaf() {
-		err = encodeBytes(w, node.key)
+		err = encoding.EncodeBytes(w, node.key)
 		if err != nil {
-			return errors.Wrap(err, "writing key")
+			return fmt.Errorf("writing key, %w", err)
 		}
+
 		// Indirection needed to provide proofs without values.
-		// (e.g. proofLeafNode.ValueHash)
-		valueHash := tmhash.Sum(node.value)
-		err = encodeBytes(w, valueHash)
+		// (e.g. ProofLeafNode.ValueHash)
+		valueHash := sha256.Sum256(node.value)
+
+		err = encoding.EncodeBytes(w, valueHash[:])
 		if err != nil {
-			return errors.Wrap(err, "writing value")
+			return fmt.Errorf("writing value, %w", err)
 		}
 	} else {
-		if node.leftHash == nil || node.rightHash == nil {
-			panic("Found an empty child hash")
+		if node.leftNode == nil || node.rightNode == nil {
+			return ErrEmptyChild
 		}
-		err = encodeBytes(w, node.leftHash)
+		err = encoding.EncodeBytes(w, node.leftNode.hash)
 		if err != nil {
-			return errors.Wrap(err, "writing left hash")
+			return fmt.Errorf("writing left hash, %w", err)
 		}
-		err = encodeBytes(w, node.rightHash)
+		err = encoding.EncodeBytes(w, node.rightNode.hash)
 		if err != nil {
-			return errors.Wrap(err, "writing right hash")
+			return fmt.Errorf("writing right hash, %w", err)
 		}
 	}