Add another hyper tree implementation using a stack of operations

Co-authored-by: Gabriel Díaz <[email protected]>

balloon/history/proof.go

@@ -45,15 +45,15 @@ func (p MembershipProof) AuditPath() navigation.AuditPath {
 }
 
 // Verify verifies a membership proof
-func (p MembershipProof) Verify(eventDigest []byte, expectedDigest hashing.Digest) (correct bool) {
+func (p MembershipProof) Verify(eventDigest []byte, expectedRootHash hashing.Digest) (correct bool) {
 
 	log.Debugf("Verifying membership proof for index %d and version %d", p.Index, p.Version)
 
 	// build a visitable pruned tree and then visit it to recompute root hash
 	visitor := pruning.NewComputeHashVisitor(p.hasher, p.auditPath)
 	recomputed := pruning.PruneToVerify(p.Index, p.Version, eventDigest).Accept(visitor)
 
-	return bytes.Equal(recomputed, expectedDigest)
+	return bytes.Equal(recomputed, expectedRootHash)
 }
 
 type IncrementalProof struct {

balloon/hyper2/navigation/audit.go

@@ -0,0 +1,18 @@
+package navigation
+
+import (
+	"github.com/bbva/qed/hashing"
+)
+
+type AuditPath []hashing.Digest
+
+func NewAuditPath() AuditPath {
+	return make(AuditPath, 0)
+}
+
+func (p AuditPath) Get(index int) (hashing.Digest, bool) {
+	if index >= len(p) {
+		return nil, false
+	}
+	return p[index], true
+}

balloon/hyper2/navigation/position.go

@@ -29,19 +29,19 @@ type Position struct {
 	numBits    uint16
 }
 
-func NewPosition(index []byte, height uint16) *Position {
+func NewPosition(index []byte, height uint16) Position {
 	var b [KeySize]byte // Size of the index plus 2 bytes for the height
 	copy(b[:], index[:len(index)])
 	copy(b[len(index):], util.Uint16AsBytes(height))
-	return &Position{
+	return Position{
 		Index:      index,
 		Height:     height,
 		serialized: b, // memoized
 		numBits:    uint16(len(index)) * 8,
 	}
 }
 
-func NewRootPosition(numBits uint16) *Position {
+func NewRootPosition(numBits uint16) Position {
 	return NewPosition(make([]byte, numBits/8), numBits)
 }
 
@@ -61,16 +61,16 @@ func (p Position) StringId() string {
 	return fmt.Sprintf("%x|%d", p.Index, p.Height)
 }
 
-func (p Position) Left() *Position {
+func (p Position) Left() Position {
 	if p.IsLeaf() {
-		return nil
+		return p
 	}
 	return NewPosition(p.Index, p.Height-1)
 }
 
-func (p Position) Right() *Position {
+func (p Position) Right() Position {
 	if p.IsLeaf() {
-		return nil
+		return p
 	}
 	return NewPosition(p.splitBase(), p.Height-1)
 }
@@ -79,16 +79,16 @@ func (p Position) IsLeaf() bool {
 	return p.Height == 0
 }
 
-func (p Position) FirstDescendant() *Position {
+func (p Position) FirstDescendant() Position {
 	if p.IsLeaf() {
-		return &p
+		return p
 	}
 	return NewPosition(p.Index, 0)
 }
 
-func (p Position) LastDescendant() *Position {
+func (p Position) LastDescendant() Position {
 	if p.IsLeaf() {
-		return &p
+		return p
 	}
 	index := make([]byte, p.numBits/8)
 	copy(index, p.Index)

balloon/hyper2/navigation/test_util.go

@@ -15,6 +15,6 @@ package navigation
 
 import "github.com/bbva/qed/util"
 
-func pos(index uint64, height uint16) *Position {
+func pos(index uint64, height uint16) Position {
 	return NewPosition(util.Uint64AsBytes(index), height)
 }

balloon/hyper2/proof.go

@@ -0,0 +1,38 @@
+package hyper2
+
+import (
+	"github.com/bbva/qed/balloon/hyper2/navigation"
+	"github.com/bbva/qed/hashing"
+	"github.com/bbva/qed/log"
+)
+
+type QueryProof struct {
+	AuditPath  navigation.AuditPath
+	Key, Value []byte
+	hasher     hashing.Hasher
+}
+
+func NewQueryProof(key, value []byte, auditPath navigation.AuditPath, hasher hashing.Hasher) *QueryProof {
+	return &QueryProof{
+		Key:       key,
+		Value:     value,
+		AuditPath: auditPath,
+		hasher:    hasher,
+	}
+}
+
+// Verify verifies a membership query for a provided key from an expected
+// root hash that fixes the hyper tree. Returns true if the proof is valid,
+// false otherwise.
+func (p QueryProof) Verify(key []byte, expectedRootHash hashing.Digest) (valid bool) {
+
+	log.Debugf("Verifying query proof for key %d", p.Key)
+
+	// build a visitable pruned tree and the visit it to recompute the root hash
+	// visitor := pruning.NewComputeHashVisitor(p)
+	// recomputed := pruning.PruneToVerify(key, p.Value, p.hasher.Len()-len(p.AuditPath)).Accept(visitor)
+
+	// return bytes.Equal(key, p.Key) && bytes.Equal(recomputed, expectedRootHash)
+	return true
+
+}

balloon/hyper2/pruning/insert.go

@@ -103,13 +103,14 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
 		// on an internal node with more than one leaf
 
 		rightPos := pos.Right()
+		leftPos := pos.Left()
 		leftLeaves, rightLeaves := leaves.Split(rightPos.Index)
 
-		left, err := traverse(pos.Left(), leftLeaves, batch, 2*iBatch+1)
+		left, err := traverse(&leftPos, leftLeaves, batch, 2*iBatch+1)
 		if err != nil {
 			return nil, err
 		}
-		right, err := traverse(rightPos, rightLeaves, batch, 2*iBatch+2)
+		right, err := traverse(&rightPos, rightLeaves, batch, 2*iBatch+2)
 		if err != nil {
 			return nil, err
 		}
@@ -203,13 +204,14 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
 
 		// on an internal node with more than one leaf
 		rightPos := pos.Right()
+		leftPos := pos.Left()
 		leftLeaves, rightLeaves := leaves.Split(rightPos.Index)
 
-		left, err := traverse(pos.Left(), leftLeaves, batch, 2*iBatch+1)
+		left, err := traverse(&leftPos, leftLeaves, batch, 2*iBatch+1)
 		if err != nil {
 			return nil, err
 		}
-		right, err := traverse(rightPos, rightLeaves, batch, 2*iBatch+2)
+		right, err := traverse(&rightPos, rightLeaves, batch, 2*iBatch+2)
 		if err != nil {
 			return nil, err
 		}
@@ -224,5 +226,6 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
 
 	leaves := make(Leaves, 0)
 	leaves = leaves.InsertSorted(Leaf{index, value})
-	return traverse(navigation.NewRootPosition(uint16(len(index)*8)), leaves, nil, 0)
+	root := navigation.NewRootPosition(uint16(len(index) * 8))
+	return traverse(&root, leaves, nil, 0)
 }

balloon/hyper2/pruning/loader.go

@@ -25,6 +25,7 @@ func NewDefaultBatchLoader(store storage.Store, cache cache.Cache, cacheHeightLi
 	}
 }
 
+// TODO remove errors and panic
 func (l DefaultBatchLoader) Load(pos *navigation.Position) (*BatchNode, error) {
 	if pos.Height > l.cacheHeightLimit {
 		return l.loadBatchFromCache(pos)

balloon/hyper2/pruning/search.go

@@ -0,0 +1,91 @@
+package pruning
+
+import (
+	"bytes"
+
+	"github.com/bbva/qed/balloon/hyper2/navigation"
+)
+
+func PruneToFind(index []byte, batches BatchLoader) (Operation, error) {
+
+	var traverse, traverseBatch func(pos *navigation.Position, batch *BatchNode, iBatch int8) (Operation, error)
+	var discardBranch func(pos *navigation.Position, batch *BatchNode, iBatch int8) Operation
+
+	traverse = func(pos *navigation.Position, batch *BatchNode, iBatch int8) (Operation, error) {
+
+		var err error
+		if batch == nil {
+			batch, err = batches.Load(pos)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		return traverseBatch(pos, batch, iBatch)
+	}
+
+	discardBranch = func(pos *navigation.Position, batch *BatchNode, iBatch int8) Operation {
+		if batch.HasElementAt(iBatch) {
+			return NewUseProvidedOp(pos, batch, iBatch)
+		}
+		return NewGetDefaultOp(pos)
+	}
+
+	traverseBatch = func(pos *navigation.Position, batch *BatchNode, iBatch int8) (Operation, error) {
+
+		// We found a nil value. That means there is no previous node stored on the current
+		// path so we stop traversing because the index does no exist in the tree.
+		// We return a new shortcut without mutating.
+		if !batch.HasElementAt(iBatch) {
+			return NewShortcutLeafOp(pos, batch, iBatch, index, nil), nil
+		}
+
+		// at the end of the batch tree
+		if iBatch > 0 && pos.Height%4 == 0 {
+			op, err := traverse(pos, nil, 0)
+			if err != nil {
+				return nil, err
+			}
+			return NewLeafOp(pos, batch, iBatch, op), nil
+		}
+
+		// on an internal node of the subtree
+
+		// we found a shortcut leaf in our path
+		if batch.HasElementAt(iBatch) && batch.HasLeafAt(iBatch) {
+			key, value := batch.GetLeafKVAt(iBatch)
+			if bytes.Equal(index, key) {
+				// we found the searched index
+				return NewShortcutLeafOp(pos, batch, iBatch, key, value), nil
+			}
+			// we found another shortcut leaf on our path so the we index
+			// we are looking for has never been inserted in the tree
+			return NewShortcutLeafOp(pos, batch, iBatch, key, nil), nil
+		}
+
+		var left, right Operation
+		var err error
+
+		rightPos := pos.Right()
+		leftPos := pos.Left()
+		if bytes.Compare(index, rightPos.Index) < 0 { // go to left
+			left, err = traverse(&leftPos, batch, 2*iBatch+1)
+			if err != nil {
+				return nil, err
+			}
+			right = NewCollectOp(discardBranch(&rightPos, batch, 2*iBatch+2))
+		} else { // go to right
+			left = NewCollectOp(discardBranch(&leftPos, batch, 2*iBatch+1))
+			right, err = traverse(&rightPos, batch, 2*iBatch+2)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		return NewInnerHashOp(pos, batch, iBatch, left, right), nil
+
+	}
+
+	root := navigation.NewRootPosition(uint16(len(index) * 8))
+	return traverse(&root, nil, 0)
+}

balloon/hyper2/pruning/search_test.go

@@ -0,0 +1,93 @@
+package pruning
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPruneToFind(t *testing.T) {
+
+	testCases := []struct {
+		index         []byte
+		cachedBatches map[string][]byte
+		storedBatches map[string][]byte
+		expectedOp    Operation
+	}{
+		{
+			// search for index=0 on an empty tree
+			index:         []byte{0},
+			cachedBatches: map[string][]byte{},
+			storedBatches: map[string][]byte{},
+			expectedOp: shortcut(pos(0, 8), 0, []byte{0x00, 0x00, 0x00, 0x00},
+				[]byte{0}, nil,
+			),
+		},
+		{
+			// search for index=0 on a tree with one leaf (index=0, value=0)
+			index: []byte{0},
+			cachedBatches: map[string][]byte{
+				pos(0, 8).StringId(): []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+			},
+			storedBatches: map[string][]byte{
+				pos(0, 4).StringId(): []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02},
+			},
+			expectedOp: inner(pos(0, 8), 0, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+				inner(pos(0, 7), 1, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+					inner(pos(0, 6), 3, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+						inner(pos(0, 5), 7, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+							leaf(pos(0, 4), 15, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+								shortcut(pos(0, 4), 0, []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02},
+									[]byte{0}, []byte{0},
+								),
+							),
+							collect(getDefault(pos(16, 4))),
+						),
+						collect(getDefault(pos(32, 5))),
+					),
+					collect(getDefault(pos(64, 6))),
+				),
+				collect(getDefault(pos(128, 7))),
+			),
+		},
+		{
+			// search for key=1 on tree with 1 leaf (index: 0, value: 0)
+			index: []byte{1},
+			cachedBatches: map[string][]byte{
+				pos(0, 8).StringId(): []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+			},
+			storedBatches: map[string][]byte{
+				pos(0, 4).StringId(): []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02},
+			},
+			expectedOp: inner(pos(0, 8), 0, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+				inner(pos(0, 7), 1, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+					inner(pos(0, 6), 3, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+						inner(pos(0, 5), 7, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+							leaf(pos(0, 4), 15, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+								shortcut(pos(0, 4), 0, []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02},
+									[]byte{0}, nil,
+								),
+							),
+							collect(getDefault(pos(16, 4))),
+						),
+						collect(getDefault(pos(32, 5))),
+					),
+					collect(getDefault(pos(64, 6))),
+				),
+				collect(getDefault(pos(128, 7))),
+			),
+		},
+	}
+
+	batchLevels := uint16(1)
+	cacheHeightLimit := batchLevels * 4
+
+	for i, c := range testCases {
+		loader := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit)
+		prunedOp, err := PruneToFind(c.index, loader)
+		require.NoError(t, err)
+		assert.Equalf(t, c.expectedOp, prunedOp, "The pruned operation should match for test case %d", i)
+	}
+
+}

balloon/hyper2/pruning/test_util.go

@@ -21,7 +21,8 @@ import (
 )
 
 func pos(index byte, height uint16) *navigation.Position {
-	return navigation.NewPosition([]byte{index}, height)
+	p := navigation.NewPosition([]byte{index}, height)
+	return &p
 }
 
 func inner(pos *navigation.Position, iBatch int8, batch []byte, left, right Operation) *InnerHashOp {