From 17cc74d37fd43dd28ffaf55e19c994dd7d9322fc Mon Sep 17 00:00:00 2001 From: Alvaro Alda Date: Fri, 15 Feb 2019 15:38:29 +0100 Subject: [PATCH] Add another hyper tree implementation using a stack of operations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Gabriel Díaz --- balloon/history/proof.go | 4 +- balloon/hyper2/navigation/audit.go | 18 ++ balloon/hyper2/navigation/position.go | 22 +-- balloon/hyper2/navigation/test_util.go | 2 +- balloon/hyper2/proof.go | 38 ++++ balloon/hyper2/pruning/insert.go | 13 +- balloon/hyper2/pruning/loader.go | 1 + balloon/hyper2/pruning/search.go | 91 +++++++++ balloon/hyper2/pruning/search_test.go | 93 +++++++++ balloon/hyper2/pruning/test_util.go | 3 +- balloon/hyper2/pruning/verify.go | 38 ++++ balloon/hyper2/pruning/visitor.go | 55 ++++++ balloon/hyper2/pruning2/batch.go | 104 ++++++++++ balloon/hyper2/pruning2/insert.go | 219 ++++++++++++++++++++++ balloon/hyper2/pruning2/insert_test.go | 59 ++++++ balloon/hyper2/pruning2/loader.go | 56 ++++++ balloon/hyper2/pruning2/operation.go | 141 ++++++++++++++ balloon/hyper2/pruning2/operation_test.go | 75 ++++++++ balloon/hyper2/pruning2/search.go | 82 ++++++++ balloon/hyper2/pruning2/stack.go | 40 ++++ balloon/hyper2/pruning2/test_util.go | 97 ++++++++++ balloon/hyper2/tree.go | 45 ++++- balloon/hyper2/tree_test.go | 2 +- 23 files changed, 1267 insertions(+), 31 deletions(-) create mode 100644 balloon/hyper2/navigation/audit.go create mode 100644 balloon/hyper2/proof.go create mode 100644 balloon/hyper2/pruning/search.go create mode 100644 balloon/hyper2/pruning/search_test.go create mode 100644 balloon/hyper2/pruning/verify.go create mode 100644 balloon/hyper2/pruning2/batch.go create mode 100644 balloon/hyper2/pruning2/insert.go create mode 100644 balloon/hyper2/pruning2/insert_test.go create mode 100644 balloon/hyper2/pruning2/loader.go create mode 100644 balloon/hyper2/pruning2/operation.go create mode 100644 balloon/hyper2/pruning2/operation_test.go create mode 100644 balloon/hyper2/pruning2/search.go create mode 100644 balloon/hyper2/pruning2/stack.go create mode 100644 balloon/hyper2/pruning2/test_util.go diff --git a/balloon/history/proof.go b/balloon/history/proof.go index 7d4ae34c8..1b0519bb0 100644 --- a/balloon/history/proof.go +++ b/balloon/history/proof.go @@ -45,7 +45,7 @@ func (p MembershipProof) AuditPath() navigation.AuditPath { } // Verify verifies a membership proof -func (p MembershipProof) Verify(eventDigest []byte, expectedDigest hashing.Digest) (correct bool) { +func (p MembershipProof) Verify(eventDigest []byte, expectedRootHash hashing.Digest) (correct bool) { log.Debugf("Verifying membership proof for index %d and version %d", p.Index, p.Version) @@ -53,7 +53,7 @@ func (p MembershipProof) Verify(eventDigest []byte, expectedDigest hashing.Diges visitor := pruning.NewComputeHashVisitor(p.hasher, p.auditPath) recomputed := pruning.PruneToVerify(p.Index, p.Version, eventDigest).Accept(visitor) - return bytes.Equal(recomputed, expectedDigest) + return bytes.Equal(recomputed, expectedRootHash) } type IncrementalProof struct { diff --git a/balloon/hyper2/navigation/audit.go b/balloon/hyper2/navigation/audit.go new file mode 100644 index 000000000..9ca46a910 --- /dev/null +++ b/balloon/hyper2/navigation/audit.go @@ -0,0 +1,18 @@ +package navigation + +import ( + "github.com/bbva/qed/hashing" +) + +type AuditPath []hashing.Digest + +func NewAuditPath() AuditPath { + return make(AuditPath, 0) +} + +func (p AuditPath) Get(index int) (hashing.Digest, bool) { + if index >= len(p) { + return nil, false + } + return p[index], true +} diff --git a/balloon/hyper2/navigation/position.go b/balloon/hyper2/navigation/position.go index 70c8bf615..d6c927950 100644 --- a/balloon/hyper2/navigation/position.go +++ b/balloon/hyper2/navigation/position.go @@ -29,11 +29,11 @@ type Position struct { numBits uint16 } -func NewPosition(index []byte, height uint16) *Position { +func NewPosition(index []byte, height uint16) Position { var b [KeySize]byte // Size of the index plus 2 bytes for the height copy(b[:], index[:len(index)]) copy(b[len(index):], util.Uint16AsBytes(height)) - return &Position{ + return Position{ Index: index, Height: height, serialized: b, // memoized @@ -41,7 +41,7 @@ func NewPosition(index []byte, height uint16) *Position { } } -func NewRootPosition(numBits uint16) *Position { +func NewRootPosition(numBits uint16) Position { return NewPosition(make([]byte, numBits/8), numBits) } @@ -61,16 +61,16 @@ func (p Position) StringId() string { return fmt.Sprintf("%x|%d", p.Index, p.Height) } -func (p Position) Left() *Position { +func (p Position) Left() Position { if p.IsLeaf() { - return nil + return p } return NewPosition(p.Index, p.Height-1) } -func (p Position) Right() *Position { +func (p Position) Right() Position { if p.IsLeaf() { - return nil + return p } return NewPosition(p.splitBase(), p.Height-1) } @@ -79,16 +79,16 @@ func (p Position) IsLeaf() bool { return p.Height == 0 } -func (p Position) FirstDescendant() *Position { +func (p Position) FirstDescendant() Position { if p.IsLeaf() { - return &p + return p } return NewPosition(p.Index, 0) } -func (p Position) LastDescendant() *Position { +func (p Position) LastDescendant() Position { if p.IsLeaf() { - return &p + return p } index := make([]byte, p.numBits/8) copy(index, p.Index) diff --git a/balloon/hyper2/navigation/test_util.go b/balloon/hyper2/navigation/test_util.go index 027931202..cebcc0971 100644 --- a/balloon/hyper2/navigation/test_util.go +++ b/balloon/hyper2/navigation/test_util.go @@ -15,6 +15,6 @@ package navigation import "github.com/bbva/qed/util" -func pos(index uint64, height uint16) *Position { +func pos(index uint64, height uint16) Position { return NewPosition(util.Uint64AsBytes(index), height) } diff --git a/balloon/hyper2/proof.go b/balloon/hyper2/proof.go new file mode 100644 index 000000000..dbf270091 --- /dev/null +++ b/balloon/hyper2/proof.go @@ -0,0 +1,38 @@ +package hyper2 + +import ( + "github.com/bbva/qed/balloon/hyper2/navigation" + "github.com/bbva/qed/hashing" + "github.com/bbva/qed/log" +) + +type QueryProof struct { + AuditPath navigation.AuditPath + Key, Value []byte + hasher hashing.Hasher +} + +func NewQueryProof(key, value []byte, auditPath navigation.AuditPath, hasher hashing.Hasher) *QueryProof { + return &QueryProof{ + Key: key, + Value: value, + AuditPath: auditPath, + hasher: hasher, + } +} + +// Verify verifies a membership query for a provided key from an expected +// root hash that fixes the hyper tree. Returns true if the proof is valid, +// false otherwise. +func (p QueryProof) Verify(key []byte, expectedRootHash hashing.Digest) (valid bool) { + + log.Debugf("Verifying query proof for key %d", p.Key) + + // build a visitable pruned tree and the visit it to recompute the root hash + // visitor := pruning.NewComputeHashVisitor(p) + // recomputed := pruning.PruneToVerify(key, p.Value, p.hasher.Len()-len(p.AuditPath)).Accept(visitor) + + // return bytes.Equal(key, p.Key) && bytes.Equal(recomputed, expectedRootHash) + return true + +} diff --git a/balloon/hyper2/pruning/insert.go b/balloon/hyper2/pruning/insert.go index bb84fef15..b4f82c269 100644 --- a/balloon/hyper2/pruning/insert.go +++ b/balloon/hyper2/pruning/insert.go @@ -103,13 +103,14 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches // on an internal node with more than one leaf rightPos := pos.Right() + leftPos := pos.Left() leftLeaves, rightLeaves := leaves.Split(rightPos.Index) - left, err := traverse(pos.Left(), leftLeaves, batch, 2*iBatch+1) + left, err := traverse(&leftPos, leftLeaves, batch, 2*iBatch+1) if err != nil { return nil, err } - right, err := traverse(rightPos, rightLeaves, batch, 2*iBatch+2) + right, err := traverse(&rightPos, rightLeaves, batch, 2*iBatch+2) if err != nil { return nil, err } @@ -203,13 +204,14 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches // on an internal node with more than one leaf rightPos := pos.Right() + leftPos := pos.Left() leftLeaves, rightLeaves := leaves.Split(rightPos.Index) - left, err := traverse(pos.Left(), leftLeaves, batch, 2*iBatch+1) + left, err := traverse(&leftPos, leftLeaves, batch, 2*iBatch+1) if err != nil { return nil, err } - right, err := traverse(rightPos, rightLeaves, batch, 2*iBatch+2) + right, err := traverse(&rightPos, rightLeaves, batch, 2*iBatch+2) if err != nil { return nil, err } @@ -224,5 +226,6 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches leaves := make(Leaves, 0) leaves = leaves.InsertSorted(Leaf{index, value}) - return traverse(navigation.NewRootPosition(uint16(len(index)*8)), leaves, nil, 0) + root := navigation.NewRootPosition(uint16(len(index) * 8)) + return traverse(&root, leaves, nil, 0) } diff --git a/balloon/hyper2/pruning/loader.go b/balloon/hyper2/pruning/loader.go index 78effa4fc..7153c8f4f 100644 --- a/balloon/hyper2/pruning/loader.go +++ b/balloon/hyper2/pruning/loader.go @@ -25,6 +25,7 @@ func NewDefaultBatchLoader(store storage.Store, cache cache.Cache, cacheHeightLi } } +// TODO remove errors and panic func (l DefaultBatchLoader) Load(pos *navigation.Position) (*BatchNode, error) { if pos.Height > l.cacheHeightLimit { return l.loadBatchFromCache(pos) diff --git a/balloon/hyper2/pruning/search.go b/balloon/hyper2/pruning/search.go new file mode 100644 index 000000000..fbafcb023 --- /dev/null +++ b/balloon/hyper2/pruning/search.go @@ -0,0 +1,91 @@ +package pruning + +import ( + "bytes" + + "github.com/bbva/qed/balloon/hyper2/navigation" +) + +func PruneToFind(index []byte, batches BatchLoader) (Operation, error) { + + var traverse, traverseBatch func(pos *navigation.Position, batch *BatchNode, iBatch int8) (Operation, error) + var discardBranch func(pos *navigation.Position, batch *BatchNode, iBatch int8) Operation + + traverse = func(pos *navigation.Position, batch *BatchNode, iBatch int8) (Operation, error) { + + var err error + if batch == nil { + batch, err = batches.Load(pos) + if err != nil { + return nil, err + } + } + + return traverseBatch(pos, batch, iBatch) + } + + discardBranch = func(pos *navigation.Position, batch *BatchNode, iBatch int8) Operation { + if batch.HasElementAt(iBatch) { + return NewUseProvidedOp(pos, batch, iBatch) + } + return NewGetDefaultOp(pos) + } + + traverseBatch = func(pos *navigation.Position, batch *BatchNode, iBatch int8) (Operation, error) { + + // We found a nil value. That means there is no previous node stored on the current + // path so we stop traversing because the index does no exist in the tree. + // We return a new shortcut without mutating. + if !batch.HasElementAt(iBatch) { + return NewShortcutLeafOp(pos, batch, iBatch, index, nil), nil + } + + // at the end of the batch tree + if iBatch > 0 && pos.Height%4 == 0 { + op, err := traverse(pos, nil, 0) + if err != nil { + return nil, err + } + return NewLeafOp(pos, batch, iBatch, op), nil + } + + // on an internal node of the subtree + + // we found a shortcut leaf in our path + if batch.HasElementAt(iBatch) && batch.HasLeafAt(iBatch) { + key, value := batch.GetLeafKVAt(iBatch) + if bytes.Equal(index, key) { + // we found the searched index + return NewShortcutLeafOp(pos, batch, iBatch, key, value), nil + } + // we found another shortcut leaf on our path so the we index + // we are looking for has never been inserted in the tree + return NewShortcutLeafOp(pos, batch, iBatch, key, nil), nil + } + + var left, right Operation + var err error + + rightPos := pos.Right() + leftPos := pos.Left() + if bytes.Compare(index, rightPos.Index) < 0 { // go to left + left, err = traverse(&leftPos, batch, 2*iBatch+1) + if err != nil { + return nil, err + } + right = NewCollectOp(discardBranch(&rightPos, batch, 2*iBatch+2)) + } else { // go to right + left = NewCollectOp(discardBranch(&leftPos, batch, 2*iBatch+1)) + right, err = traverse(&rightPos, batch, 2*iBatch+2) + if err != nil { + return nil, err + } + } + + return NewInnerHashOp(pos, batch, iBatch, left, right), nil + + } + + root := navigation.NewRootPosition(uint16(len(index) * 8)) + return traverse(&root, nil, 0) +} diff --git a/balloon/hyper2/pruning/search_test.go b/balloon/hyper2/pruning/search_test.go new file mode 100644 index 000000000..ea1cbe7af --- /dev/null +++ b/balloon/hyper2/pruning/search_test.go @@ -0,0 +1,93 @@ +package pruning + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPruneToFind(t *testing.T) { + + testCases := []struct { + index []byte + cachedBatches map[string][]byte + storedBatches map[string][]byte + expectedOp Operation + }{ + { + // search for index=0 on an empty tree + index: []byte{0}, + cachedBatches: map[string][]byte{}, + storedBatches: map[string][]byte{}, + expectedOp: shortcut(pos(0, 8), 0, []byte{0x00, 0x00, 0x00, 0x00}, + []byte{0}, nil, + ), + }, + { + // search for index=0 on a tree with one leaf (index=0, value=0) + index: []byte{0}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, + }, + expectedOp: inner(pos(0, 8), 0, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + inner(pos(0, 7), 1, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + inner(pos(0, 6), 3, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + inner(pos(0, 5), 7, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + leaf(pos(0, 4), 15, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + shortcut(pos(0, 4), 0, []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, + []byte{0}, []byte{0}, + ), + ), + collect(getDefault(pos(16, 4))), + ), + collect(getDefault(pos(32, 5))), + ), + collect(getDefault(pos(64, 6))), + ), + collect(getDefault(pos(128, 7))), + ), + }, + { + // search for key=1 on tree with 1 leaf (index: 0, value: 0) + index: []byte{1}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, + }, + expectedOp: inner(pos(0, 8), 0, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + inner(pos(0, 7), 1, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + inner(pos(0, 6), 3, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + inner(pos(0, 5), 7, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + leaf(pos(0, 4), 15, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + shortcut(pos(0, 4), 0, []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, + []byte{0}, nil, + ), + ), + collect(getDefault(pos(16, 4))), + ), + collect(getDefault(pos(32, 5))), + ), + collect(getDefault(pos(64, 6))), + ), + collect(getDefault(pos(128, 7))), + ), + }, + } + + batchLevels := uint16(1) + cacheHeightLimit := batchLevels * 4 + + for i, c := range testCases { + loader := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) + prunedOp, err := PruneToFind(c.index, loader) + require.NoError(t, err) + assert.Equalf(t, c.expectedOp, prunedOp, "The pruned operation should match for test case %d", i) + } + +} diff --git a/balloon/hyper2/pruning/test_util.go b/balloon/hyper2/pruning/test_util.go index f50f83291..94b593105 100644 --- a/balloon/hyper2/pruning/test_util.go +++ b/balloon/hyper2/pruning/test_util.go @@ -21,7 +21,8 @@ import ( ) func pos(index byte, height uint16) *navigation.Position { - return navigation.NewPosition([]byte{index}, height) + p := navigation.NewPosition([]byte{index}, height) + return &p } func inner(pos *navigation.Position, iBatch int8, batch []byte, left, right Operation) *InnerHashOp { diff --git a/balloon/hyper2/pruning/verify.go b/balloon/hyper2/pruning/verify.go new file mode 100644 index 000000000..bb7688dfd --- /dev/null +++ b/balloon/hyper2/pruning/verify.go @@ -0,0 +1,38 @@ +package pruning + +import ( + "bytes" + + "github.com/bbva/qed/balloon/hyper2/navigation" +) + +func PruneToVerify(key, value []byte, auditPathHeight uint16) Operation { + + var traverse func(pos *navigation.Position) Operation + + traverse = func(pos *navigation.Position) Operation { + + if pos.Height <= auditPathHeight { + // we are at the leaf height + return NewShortcutLeafOp(pos, nil, 0, key, value) + } + + var left, right Operation + rightPos := pos.Right() + leftPos := pos.Left() + if bytes.Compare(key, rightPos.Index) < 0 { // go to left + left = traverse(&leftPos) + right = NewGetDefaultOp(&rightPos) + } else { // go to right + left = NewGetDefaultOp(&leftPos) + right = traverse(&rightPos) + } + + return NewInnerHashOp(pos, nil, 0, left, right) + + } + + root := navigation.NewRootPosition(uint16(len(key) * 8)) + return traverse(&root) + +} diff --git a/balloon/hyper2/pruning/visitor.go b/balloon/hyper2/pruning/visitor.go index 5513e807b..5632917e1 100644 --- a/balloon/hyper2/pruning/visitor.go +++ b/balloon/hyper2/pruning/visitor.go @@ -2,6 +2,7 @@ package pruning import ( "github.com/bbva/qed/balloon/cache" + "github.com/bbva/qed/balloon/hyper2/navigation" "github.com/bbva/qed/hashing" "github.com/bbva/qed/storage" ) @@ -79,3 +80,57 @@ func (v *InsertVisitor) VisitMutateBatchOp(op MutateBatchOp) hashing.Digest { func (v *InsertVisitor) VisitCollectOp(op CollectOp) hashing.Digest { return op.Operation.Accept(v) } + +type AuditPathVisitor struct { + hasher hashing.Hasher + defaultHashes []hashing.Digest + auditPath navigation.AuditPath +} + +func NewAuditPathVisitor(hasher hashing.Hasher, defaultHashes []hashing.Digest) *AuditPathVisitor { + return &AuditPathVisitor{ + hasher: hasher, + defaultHashes: defaultHashes, + auditPath: navigation.NewAuditPath(), + } +} + +func (v AuditPathVisitor) Result() navigation.AuditPath { + return v.auditPath +} + +func (v *AuditPathVisitor) VisitShortcutLeafOp(op ShortcutLeafOp) hashing.Digest { + return nil +} + +func (v *AuditPathVisitor) VisitLeafOp(op LeafOp) hashing.Digest { + return op.Operation.Accept(v) +} + +func (v *AuditPathVisitor) VisitInnerHashOp(op InnerHashOp) hashing.Digest { + op.Left.Accept(v) + op.Right.Accept(v) + return nil +} + +func (v *AuditPathVisitor) VisitGetDefaultOp(op GetDefaultOp) hashing.Digest { + return v.defaultHashes[op.Position().Height] +} + +func (v *AuditPathVisitor) VisitUseProvidedOp(op UseProvidedOp) hashing.Digest { + return op.Batch.GetElementAt(op.Idx) +} + +func (v *AuditPathVisitor) VisitPutBatchOp(op PutBatchOp) hashing.Digest { + return op.Operation.Accept(v) +} + +func (v *AuditPathVisitor) VisitMutateBatchOp(op MutateBatchOp) hashing.Digest { + return op.Operation.Accept(v) +} + +func (v *AuditPathVisitor) VisitCollectOp(op CollectOp) hashing.Digest { + hash := op.Operation.Accept(v) + v.auditPath = append(v.auditPath, hash) + return hash +} diff --git a/balloon/hyper2/pruning2/batch.go b/balloon/hyper2/pruning2/batch.go new file mode 100644 index 000000000..a9c83de53 --- /dev/null +++ b/balloon/hyper2/pruning2/batch.go @@ -0,0 +1,104 @@ +package pruning2 + +import ( + "fmt" + "strings" + + "github.com/bbva/qed/hashing" +) + +type BatchNode struct { + Batch [][]byte + nodeSize int // in bytes +} + +func NewEmptyBatchNode(nodeSize int) *BatchNode { + return &BatchNode{ + nodeSize: nodeSize, + Batch: make([][]byte, 31, 31), + } +} + +func NewBatchNode(nodeSize int, batch [][]byte) *BatchNode { + return &BatchNode{ + nodeSize: nodeSize, + Batch: batch, + } +} + +func (b BatchNode) String() string { + var strs []string + for i, n := range b.Batch { + strs = append(strs, fmt.Sprintf("[%d - %#x]", i, n)) + } + return strings.Join(strs, "\n") +} + +func (b BatchNode) HasLeafAt(i int8) bool { + return b.Batch[i][b.nodeSize] == byte(1) +} + +func (b BatchNode) AddHashAt(i int8, value []byte) { + b.Batch[i] = append(value, byte(0)) +} + +func (b BatchNode) AddLeafAt(i int8, hash hashing.Digest, key, value []byte) { + b.Batch[i] = append(hash, byte(1)) + b.Batch[2*i+1] = append(key, byte(2)) + b.Batch[2*i+2] = append(value, byte(2)) +} + +func (b BatchNode) GetLeafKVAt(i int8) ([]byte, []byte) { + return b.Batch[2*i+1][:b.nodeSize], b.Batch[2*i+2][:b.nodeSize] +} + +func (b BatchNode) HasElementAt(i int8) bool { + return len(b.Batch[i]) > 0 +} + +func (b BatchNode) GetElementAt(i int8) []byte { + return b.Batch[i][:b.nodeSize] +} + +func (b BatchNode) ResetElementAt(i int8) { + b.Batch[i] = nil +} + +func (b BatchNode) Serialize() []byte { + serialized := make([]byte, 4) + for i := 0; i < 31; i++ { + if len(b.Batch[i]) != 0 { + bitSet(serialized, i) + serialized = append(serialized, b.Batch[i]...) + } + } + return serialized +} + +func ParseBatch(nodeSize int, value []byte) [][]byte { + batch := make([][]byte, 31, 31) // 31 nodes (including the root) + bitmap := value[:4] // the first 4 bytes define the bitmap + size := nodeSize + 1 + + j := 0 + for i := 0; i < 31; i++ { + if bitIsSet(bitmap, i) { + batch[i] = value[4+size*j : 4+size*(j+1)] + j++ + } + } + + return batch +} + +func ParseBatchNode(nodeSize int, value []byte) *BatchNode { + return NewBatchNode(nodeSize, ParseBatch(nodeSize, value)) +} + +func bitIsSet(bits []byte, i int) bool { + return bits[i/8]&(1< 0 + }) + + if index > 0 && bytes.Equal(l[index-1].Index, leaf.Index) { + return l + } + + l = append(l, leaf) + copy(l[index+1:], l[index:]) + l[index] = leaf + return l + +} + +func (l Leaves) Split(index []byte) (left, right Leaves) { + // the smallest index i where l[i].Index >= index + splitIndex := sort.Search(len(l), func(i int) bool { + return bytes.Compare(l[i].Index, index) >= 0 + }) + return l[:splitIndex], l[splitIndex:] +} + +type TraverseBatch func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) + +func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches BatchLoader) *OperationsStack { + + var traverse, traverseThroughCache, traverseAfterCache TraverseBatch + + traverse = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) { + + if batch == nil { + batch = batches.Load(pos) + } + + if pos.Height > cacheHeightLimit { + traverseThroughCache(pos, leaves, batch, iBatch, ops) + } else { + traverseAfterCache(pos, leaves, batch, iBatch, ops) + } + } + + traverseThroughCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) { + + if len(leaves) == 0 { // discarded branch + if batch.HasElementAt(iBatch) { + ops.Push(getProvidedHash(pos, iBatch, batch)) + } else { + ops.Push(getDefaultHash(pos)) + } + return + } + + // at the end of a batch tree + if iBatch > 0 && pos.Height%4 == 0 { + traverse(pos, leaves, nil, 0, ops) + ops.Push(putInCache(pos, batch)) + return + } + + // on an internal node of the subtree + + // we found a node in our path + if batch.HasElementAt(iBatch) { + // we found a shortcut leaf in our path + if batch.HasLeafAt(iBatch) { + // push down leaf + key, value := batch.GetLeafKVAt(iBatch) + leaves = leaves.InsertSorted(Leaf{key, value}) + batch.ResetElementAt(iBatch) + batch.ResetElementAt(2*iBatch + 1) + batch.ResetElementAt(2*iBatch + 2) + traverse(pos, leaves, batch, iBatch, ops) + return + } + } + + // on an internal node with more than one leaf + + rightPos := pos.Right() + leftLeaves, rightLeaves := leaves.Split(rightPos.Index) + + traverse(pos.Left(), leftLeaves, batch, 2*iBatch+1, ops) + traverse(rightPos, rightLeaves, batch, 2*iBatch+2, ops) + + ops.PushAll(innerHash(pos), updateBatchNode(pos, iBatch, batch)) + if iBatch == 0 { // it's the root of the batch tree + ops.Push(putInCache(pos, batch)) + } + + } + + traverseAfterCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) { + + if len(leaves) == 0 { // discarded branch + if batch.HasElementAt(iBatch) { + ops.Push(getProvidedHash(pos, iBatch, batch)) + } else { + ops.Push(getDefaultHash(pos)) + } + return + } + + // at the end of the main tree + // this is a special case because we have to mutate even if there exists a previous stored leaf (update scenario) + if pos.IsLeaf() { + if len(leaves) != 1 { + panic("Oops, something went wrong. We cannot have more than one leaf at the end of the main tree") + } + // create or update the leaf with a new shortcut + newBatch := NewEmptyBatchNode(len(pos.Index)) + ops.PushAll( + shortcutHash(pos, leaves[0].Index, leaves[0].Value), + updateBatchLeaf(pos, 0, newBatch, leaves[0].Index, leaves[0].Value), + mutateBatch(pos, newBatch), + updateBatchNode(pos, iBatch, batch), + ) + return + } + + // at the end of a subtree + if iBatch > 0 && pos.Height%4 == 0 { + if len(leaves) > 1 { + // with more than one leaf to insert -> it's impossible to be a shortcut leaf + traverse(pos, leaves, nil, 0, ops) + ops.Push(updateBatchNode(pos, iBatch, batch)) + return + } + // with only one leaf to insert -> add a new shortcut leaf or continue traversing + if batch.HasElementAt(iBatch) { + // continue traversing + traverse(pos, leaves, nil, 0, ops) + ops.Push(updateBatchNode(pos, iBatch, batch)) + return + } + // nil value (no previous node stored) so create a new shortcut batch + newBatch := NewEmptyBatchNode(len(pos.Index)) + ops.PushAll( + shortcutHash(pos, leaves[0].Index, leaves[0].Value), + updateBatchLeaf(pos, 0, newBatch, leaves[0].Index, leaves[0].Value), + mutateBatch(pos, newBatch), + updateBatchNode(pos, iBatch, batch), + ) + return + } + + // on an internal node with only one leaf to insert + + if len(leaves) == 1 { + // we found a nil in our path -> create a shortcut leaf + if !batch.HasElementAt(iBatch) { + ops.PushAll( + shortcutHash(pos, leaves[0].Index, leaves[0].Value), + updateBatchLeaf(pos, iBatch, batch, leaves[0].Index, leaves[0].Value), + ) + if pos.Height%4 == 0 { // at the root or at a leaf of the subtree + ops.Push(mutateBatch(pos, batch)) + } + return + } + + // we found a node in our path + if batch.HasElementAt(iBatch) { + // we found a shortcut leaf in our path + if batch.HasLeafAt(iBatch) { + // push down leaf + key, value := batch.GetLeafKVAt(iBatch) + leaves = leaves.InsertSorted(Leaf{key, value}) + batch.ResetElementAt(iBatch) + batch.ResetElementAt(2*iBatch + 1) + batch.ResetElementAt(2*iBatch + 2) + traverse(pos, leaves, batch, iBatch, ops) + return + } + } + } + + // on an internal node with more than one leaf + rightPos := pos.Right() + leftLeaves, rightLeaves := leaves.Split(rightPos.Index) + + traverse(pos.Left(), leftLeaves, batch, 2*iBatch+1, ops) + traverse(rightPos, rightLeaves, batch, 2*iBatch+2, ops) + + ops.PushAll(innerHash(pos), updateBatchNode(pos, iBatch, batch)) + if iBatch == 0 { // at root node -> mutate batch + ops.Push(mutateBatch(pos, batch)) + } + + } + + ops := NewOperationsStack() + leaves := make(Leaves, 0) + leaves = leaves.InsertSorted(Leaf{index, value}) + traverse(navigation.NewRootPosition(uint16(len(index)*8)), leaves, nil, 0, ops) + return ops +} diff --git a/balloon/hyper2/pruning2/insert_test.go b/balloon/hyper2/pruning2/insert_test.go new file mode 100644 index 000000000..b8bc2e5cc --- /dev/null +++ b/balloon/hyper2/pruning2/insert_test.go @@ -0,0 +1,59 @@ +package pruning2 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPruneToInsert(t *testing.T) { + + testCases := []struct { + index, value []byte + cachedBatches map[string][]byte + storedBatches map[string][]byte + expectedOps []*Operation + }{ + { + // insert index = 0 on empty tree + index: []byte{0}, + value: []byte{0}, + cachedBatches: map[string][]byte{}, + storedBatches: map[string][]byte{}, + expectedOps: []*Operation{ + put(pos(0, 8), []byte{0x00, 0x00, 0x00, 0x00}), + updateNode(pos(0, 8), 0, []byte{0x00, 0x00, 0x00, 0x00}), + inner(pos(0, 8)), + getDefault(pos(128, 7)), + updateNode(pos(0, 7), 1, []byte{0x00, 0x00, 0x00, 0x00}), + inner(pos(0, 7)), + getDefault(pos(64, 6)), + updateNode(pos(0, 6), 3, []byte{0x00, 0x00, 0x00, 0x00}), + inner(pos(0, 6)), + getDefault(pos(32, 5)), + updateNode(pos(0, 5), 7, []byte{0x00, 0x00, 0x00, 0x00}), + inner(pos(0, 5)), + getDefault(pos(16, 4)), + updateNode(pos(0, 4), 15, []byte{0x00, 0x00, 0x00, 0x00}), + mutate(pos(0, 4), []byte{0x00, 0x00, 0x00, 0x00}), + updateLeaf(pos(0, 4), 0, []byte{0x00, 0x00, 0x00, 0x00}, []byte{0}, []byte{0}), + shortcut(pos(0, 4), []byte{0}, []byte{0}), + }, + }, + } + + batchLevels := uint16(1) + cacheHeightLimit := batchLevels * 4 + + for i, c := range testCases { + loader := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) + prunedOps := PruneToInsert(c.index, c.value, cacheHeightLimit, loader).List() + require.Truef(t, len(c.expectedOps) == len(prunedOps), "The size of the pruned ops should match the expected for test case %d", i) + for i := 0; i < len(prunedOps); i++ { + assert.Equalf(t, c.expectedOps[i].Code, prunedOps[i].Code, "The pruned operation's code should match for test case %d", i) + assert.Equalf(t, c.expectedOps[i].Pos, prunedOps[i].Pos, "The pruned operation's position should match for test case %d", i) + } + } + +} diff --git a/balloon/hyper2/pruning2/loader.go b/balloon/hyper2/pruning2/loader.go new file mode 100644 index 000000000..e18608cc8 --- /dev/null +++ b/balloon/hyper2/pruning2/loader.go @@ -0,0 +1,56 @@ +package pruning2 + +import ( + "github.com/bbva/qed/log" + + "github.com/bbva/qed/balloon/cache" + "github.com/bbva/qed/balloon/hyper2/navigation" + "github.com/bbva/qed/storage" +) + +type BatchLoader interface { + Load(pos navigation.Position) *BatchNode +} + +// TODO maybe use a function +type DefaultBatchLoader struct { + cacheHeightLimit uint16 + cache cache.Cache + store storage.Store +} + +func NewDefaultBatchLoader(store storage.Store, cache cache.Cache, cacheHeightLimit uint16) *DefaultBatchLoader { + return &DefaultBatchLoader{ + cacheHeightLimit: cacheHeightLimit, + cache: cache, + store: store, + } +} + +func (l DefaultBatchLoader) Load(pos navigation.Position) *BatchNode { + if pos.Height > l.cacheHeightLimit { + return l.loadBatchFromCache(pos) + } + return l.loadBatchFromStore(pos) +} + +func (l DefaultBatchLoader) loadBatchFromCache(pos navigation.Position) *BatchNode { + value, ok := l.cache.Get(pos.Bytes()) + if !ok { + return NewEmptyBatchNode(len(pos.Index)) + } + batch := ParseBatchNode(len(pos.Index), value) + return batch +} + +func (l DefaultBatchLoader) loadBatchFromStore(pos navigation.Position) *BatchNode { + kv, err := l.store.Get(storage.HyperCachePrefix, pos.Bytes()) + if err != nil { + if err == storage.ErrKeyNotFound { + return NewEmptyBatchNode(len(pos.Index)) + } + log.Fatalf("Oops, something went wrong. Unable to load batch: %v", err) + } + batch := ParseBatchNode(len(pos.Index), kv.Value) + return batch +} diff --git a/balloon/hyper2/pruning2/operation.go b/balloon/hyper2/pruning2/operation.go new file mode 100644 index 000000000..a10885b51 --- /dev/null +++ b/balloon/hyper2/pruning2/operation.go @@ -0,0 +1,141 @@ +package pruning2 + +import ( + "github.com/bbva/qed/balloon/cache" + "github.com/bbva/qed/balloon/hyper2/navigation" + "github.com/bbva/qed/hashing" + "github.com/bbva/qed/storage" +) + +type Context struct { + Hasher hashing.Hasher + Cache cache.ModifiableCache + DefaultHashes []hashing.Digest + Mutations []*storage.Mutation + AuditPath navigation.AuditPath +} + +type OperationCode int + +const ( + ShortcutHashCode OperationCode = iota + InnerHashCode + LeafHashCode + UpdateBatchNodeCode + UpdateBatchLeafCode + DefaultHashCode + GetProvidedHashCode + PutInCacheCode + MutateBatchCode + CollectHashCode +) + +type Interpreter func(ops *OperationsStack, c *Context) hashing.Digest + +type Operation struct { + Code OperationCode + Pos navigation.Position + Interpret Interpreter +} + +func shortcutHash(pos navigation.Position, key, value []byte) *Operation { + return &Operation{ + Code: ShortcutHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + return c.Hasher.Salted(pos.Bytes(), value) + }, + } +} + +func innerHash(pos navigation.Position) *Operation { + return &Operation{ + Code: InnerHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + leftHash := ops.Pop().Interpret(ops, c) + rightHash := ops.Pop().Interpret(ops, c) + return c.Hasher.Salted(pos.Bytes(), leftHash, rightHash) + }, + } +} + +func updateBatchNode(pos navigation.Position, idx int8, batch *BatchNode) *Operation { + return &Operation{ + Code: UpdateBatchNodeCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + batch.AddHashAt(idx, hash) + return hash + }, + } +} + +func updateBatchLeaf(pos navigation.Position, idx int8, batch *BatchNode, key, value []byte) *Operation { + return &Operation{ + Code: UpdateBatchLeafCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + batch.AddLeafAt(idx, hash, key, value) + return hash + }, + } +} + +func getDefaultHash(pos navigation.Position) *Operation { + return &Operation{ + Code: DefaultHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + return c.DefaultHashes[pos.Height] + }, + } +} + +func getProvidedHash(pos navigation.Position, idx int8, batch *BatchNode) *Operation { + return &Operation{ + Code: GetProvidedHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + return batch.GetElementAt(idx) + }, + } +} + +func putInCache(pos navigation.Position, batch *BatchNode) *Operation { + return &Operation{ + Code: PutInCacheCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + c.Cache.Put(pos.Bytes(), batch.Serialize()) + return hash + }, + } +} + +func mutateBatch(pos navigation.Position, batch *BatchNode) *Operation { + return &Operation{ + Code: MutateBatchCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + c.Mutations = append(c.Mutations, storage.NewMutation(storage.HyperCachePrefix, pos.Bytes(), batch.Serialize())) + return hash + }, + } +} + +func collectHash(pos navigation.Position) *Operation { + return &Operation{ + Code: CollectHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + c.AuditPath = append(c.AuditPath, hash) + return hash + }, + } +} diff --git a/balloon/hyper2/pruning2/operation_test.go b/balloon/hyper2/pruning2/operation_test.go new file mode 100644 index 000000000..9f6e9e499 --- /dev/null +++ b/balloon/hyper2/pruning2/operation_test.go @@ -0,0 +1,75 @@ +package pruning2 + +import ( + "testing" + + "github.com/bbva/qed/balloon/cache" + "github.com/bbva/qed/balloon/hyper2/navigation" + "github.com/bbva/qed/hashing" + "github.com/bbva/qed/storage" + "github.com/stretchr/testify/assert" +) + +func TestInsertInterpretation(t *testing.T) { + + testCases := []struct { + index, value []byte + cachedBatches map[string][]byte + storedBatches map[string][]byte + expectedMutations []*storage.Mutation + expectedElements []*cachedElement + }{ + { + // insert index = 0 on empty tree + index: []byte{0}, + value: []byte{0}, + cachedBatches: map[string][]byte{}, + storedBatches: map[string][]byte{}, + expectedMutations: []*storage.Mutation{ + { + Prefix: storage.HyperCachePrefix, + Key: pos(0, 4).Bytes(), + Value: []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, + }, + }, + expectedElements: []*cachedElement{ + newCachedElement(pos(0, 8), []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}), + }, + }, + } + + batchLevels := uint16(1) + cacheHeightLimit := batchLevels * 4 + defaultHashes := []hashing.Digest{{0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}} + + for i, c := range testCases { + cache := cache.NewFakeCache([]byte{0x0}) + batches := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) + + ops := PruneToInsert(c.index, c.value, cacheHeightLimit, batches) + ctx := &Context{ + Hasher: hashing.NewFakeXorHasher(), + Cache: cache, + DefaultHashes: defaultHashes, + Mutations: make([]*storage.Mutation, 0), + } + + ops.Pop().Interpret(ops, ctx) + + assert.ElementsMatchf(t, c.expectedMutations, ctx.Mutations, "Mutation error in test case %d", i) + for _, e := range c.expectedElements { + v, _ := cache.Get(e.Pos.Bytes()) + assert.Equalf(t, e.Value, v, "The cached element %v should be cached in test case %d", e, i) + } + } + +} + +type cachedElement struct { + Pos navigation.Position + Value []byte +} + +func newCachedElement(pos navigation.Position, value []byte) *cachedElement { + return &cachedElement{pos, value} +} diff --git a/balloon/hyper2/pruning2/search.go b/balloon/hyper2/pruning2/search.go new file mode 100644 index 000000000..64095c87e --- /dev/null +++ b/balloon/hyper2/pruning2/search.go @@ -0,0 +1,82 @@ +package pruning2 + +import ( + "bytes" + + "github.com/bbva/qed/balloon/hyper2/navigation" +) + +func PruneToFind(index []byte, batches BatchLoader) *OperationsStack { + + var traverse, traverseBatch, discardBranch func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) + + traverse = func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) { + + if batch == nil { + batch = batches.Load(pos) + } + + traverseBatch(pos, batch, iBatch, ops) + } + + discardBranch = func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) { + if batch.HasElementAt(iBatch) { + ops.Push(getProvidedHash(pos, iBatch, batch)) + } else { + ops.Push(getDefaultHash(pos)) + } + } + + traverseBatch = func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) { + + // We found a nil value. That means there is no previous node stored on the current + // path so we stop traversing because the index does no exist in the tree. + // We return a new shortcut without mutating. + if !batch.HasElementAt(iBatch) { + ops.Push(shortcutHash(pos, index, nil)) // TODO shall i return nothing? + return + } + + // at the end of the batch tree + if iBatch > 0 && pos.Height%4 == 0 { + traverse(pos, nil, 0, ops) + ops.Push(getProvidedHash(pos, iBatch, batch)) + return + } + + // on an internal node of the subtree + + // we found a shortcut leaf in our path + if batch.HasElementAt(iBatch) && batch.HasLeafAt(iBatch) { + key, value := batch.GetLeafKVAt(iBatch) + if bytes.Equal(index, key) { + // we found the searched index + ops.Push(shortcutHash(pos, key, value)) + return + } + // we found another shortcut leaf on our path so the index + // we are looking for has never been inserted in the tree + ops.Push(shortcutHash(pos, key, nil)) + return + } + + rightPos := pos.Right() + leftPos := pos.Left() + if bytes.Compare(index, rightPos.Index) < 0 { // go to left + traverse(pos.Left(), batch, 2*iBatch+1, ops) + discardBranch(rightPos, batch, 2*iBatch+2, ops) + ops.Push(collectHash(rightPos)) + } else { // go to right + discardBranch(leftPos, batch, 2*iBatch+1, ops) + ops.Push(collectHash(leftPos)) + traverse(rightPos, batch, 2*iBatch+2, ops) + } + + ops.Push(innerHash(pos)) + + } + + ops := NewOperationsStack() + traverse(navigation.NewRootPosition(uint16(len(index)*8)), nil, 0, ops) + return ops +} diff --git a/balloon/hyper2/pruning2/stack.go b/balloon/hyper2/pruning2/stack.go new file mode 100644 index 000000000..5606de1ac --- /dev/null +++ b/balloon/hyper2/pruning2/stack.go @@ -0,0 +1,40 @@ +package pruning2 + +type OperationsStack []*Operation + +func NewOperationsStack() *OperationsStack { + return new(OperationsStack) +} + +func (s *OperationsStack) Len() int { + return len(*s) +} + +func (s OperationsStack) Peek() (op *Operation) { + return s[len(s)-1] +} + +func (s *OperationsStack) Pop() (op *Operation) { + i := s.Len() - 1 + op = (*s)[i] + *s = (*s)[:i] + return +} + +func (s *OperationsStack) Push(op *Operation) { + *s = append(*s, op) +} + +func (s *OperationsStack) PushAll(values ...*Operation) { + for _, value := range values { + s.Push(value) + } +} + +func (s *OperationsStack) List() []*Operation { + l := make([]*Operation, 0) + for s.Len() > 0 { + l = append(l, s.Pop()) + } + return l +} diff --git a/balloon/hyper2/pruning2/test_util.go b/balloon/hyper2/pruning2/test_util.go new file mode 100644 index 000000000..ae6807626 --- /dev/null +++ b/balloon/hyper2/pruning2/test_util.go @@ -0,0 +1,97 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package pruning2 + +import ( + "github.com/bbva/qed/balloon/hyper2/navigation" +) + +func pos(index byte, height uint16) navigation.Position { + return navigation.NewPosition([]byte{index}, height) +} + +func shortcut(pos navigation.Position, key, value []byte) *Operation { + return shortcutHash(pos, key, value) +} + +func inner(pos navigation.Position) *Operation { + return innerHash(pos) +} + +func updateNode(pos navigation.Position, iBatch int8, batch []byte) *Operation { + return updateBatchNode(pos, iBatch, ParseBatchNode(1, batch)) +} + +func updateLeaf(pos navigation.Position, iBatch int8, batch, key, value []byte) *Operation { + return updateBatchLeaf(pos, iBatch, ParseBatchNode(1, batch), key, value) +} + +func getDefault(pos navigation.Position) *Operation { + return getDefaultHash(pos) +} + +func getProvided(pos navigation.Position, iBatch int8, batch []byte) *Operation { + return getProvidedHash(pos, iBatch, ParseBatchNode(1, batch)) +} + +func put(pos navigation.Position, batch []byte) *Operation { + return putInCache(pos, ParseBatchNode(1, batch)) +} + +func mutate(pos navigation.Position, batch []byte) *Operation { + return mutateBatch(pos, ParseBatchNode(1, batch)) +} + +func collect(pos navigation.Position) *Operation { + return collectHash(pos) +} + +type FakeBatchLoader struct { + cacheHeightLimit uint16 + cached map[string]*BatchNode + stored map[string]*BatchNode +} + +func NewFakeBatchLoader(cached map[string][]byte, stored map[string][]byte, cacheHeightLimit uint16) *FakeBatchLoader { + loader := &FakeBatchLoader{ + cacheHeightLimit: cacheHeightLimit, + cached: make(map[string]*BatchNode, 0), + stored: make(map[string]*BatchNode, 0), + } + for k, v := range cached { + loader.cached[k] = ParseBatchNode(1, v) + } + for k, v := range stored { + loader.stored[k] = ParseBatchNode(1, v) + } + return loader +} + +func (l *FakeBatchLoader) Load(pos navigation.Position) *BatchNode { + if pos.Height > l.cacheHeightLimit { + batch, ok := l.cached[pos.StringId()] + if !ok { + return NewEmptyBatchNode(len(pos.Index)) + } + return batch + } + batch, ok := l.stored[pos.StringId()] + if !ok { + return NewEmptyBatchNode(len(pos.Index)) + } + return batch +} diff --git a/balloon/hyper2/tree.go b/balloon/hyper2/tree.go index 91e6865a3..b295eda49 100644 --- a/balloon/hyper2/tree.go +++ b/balloon/hyper2/tree.go @@ -3,8 +3,10 @@ package hyper2 import ( "sync" + "github.com/bbva/qed/balloon/hyper2/navigation" + "github.com/bbva/qed/balloon/cache" - "github.com/bbva/qed/balloon/hyper2/pruning" + "github.com/bbva/qed/balloon/hyper2/pruning2" "github.com/bbva/qed/hashing" "github.com/bbva/qed/storage" "github.com/bbva/qed/util" @@ -18,7 +20,7 @@ type HyperTree struct { hasher hashing.Hasher cacheHeightLimit uint16 defaultHashes []hashing.Digest - batchLoader pruning.BatchLoader + batchLoader pruning2.BatchLoader sync.RWMutex } @@ -36,7 +38,7 @@ func NewHyperTree(hasherF func() hashing.Hasher, store storage.Store, cache cach hasher: hasher, cacheHeightLimit: cacheHeightLimit, defaultHashes: make([]hashing.Digest, numBits), - batchLoader: pruning.NewDefaultBatchLoader(store, cache, cacheHeightLimit), + batchLoader: pruning2.NewDefaultBatchLoader(store, cache, cacheHeightLimit), } tree.defaultHashes[0] = tree.hasher.Do([]byte{0x0}, []byte{0x0}) @@ -56,18 +58,41 @@ func (t *HyperTree) Add(eventDigest hashing.Digest, version uint64) (hashing.Dig //log.Debugf("Adding new event digest %x with version %d", eventDigest, version) - // build a visitable pruned tree and then visit it to generate the root hash versionAsBytes := util.Uint64AsPaddedBytes(version, len(eventDigest)) versionAsBytes = versionAsBytes[len(versionAsBytes)-len(eventDigest):] - visitor := pruning.NewInsertVisitor(t.hasher, t.cache, t.defaultHashes) - op, err := pruning.PruneToInsert(eventDigest, versionAsBytes, t.cacheHeightLimit, t.batchLoader) - if err != nil { - return nil, nil, err + + // build a stack of operations and then interpret it to generate the root hash + ops := pruning2.PruneToInsert(eventDigest, versionAsBytes, t.cacheHeightLimit, t.batchLoader) + ctx := &pruning2.Context{ + Hasher: t.hasher, + Cache: t.cache, + DefaultHashes: t.defaultHashes, + Mutations: make([]*storage.Mutation, 0), + } + + rh := ops.Pop().Interpret(ops, ctx) + + return rh, ctx.Mutations, nil +} + +func (t *HyperTree) QueryMembership(eventDigest hashing.Digest, version []byte) (proof *QueryProof, err error) { + t.Lock() + defer t.Unlock() + + //log.Debugf("Proving membership for index %d with version %d", eventDigest, version) + + // build a stack of operations and then interpret it to generate the audit path + ops := pruning2.PruneToFind(eventDigest, t.batchLoader) + ctx := &pruning2.Context{ + Hasher: t.hasher, + Cache: t.cache, + DefaultHashes: t.defaultHashes, + AuditPath: make(navigation.AuditPath, 0), } - rh := op.Accept(visitor) + ops.Pop().Interpret(ops, ctx) - return rh, visitor.Result(), nil + return NewQueryProof(eventDigest, version, ctx.AuditPath, t.hasherF()), nil } func min(x, y uint16) uint16 { diff --git a/balloon/hyper2/tree_test.go b/balloon/hyper2/tree_test.go index bfc829160..ae8eb7c2a 100644 --- a/balloon/hyper2/tree_test.go +++ b/balloon/hyper2/tree_test.go @@ -60,7 +60,7 @@ func BenchmarkAdd(b *testing.B) { tree := NewHyperTree(hashing.NewSha256Hasher, store, freeCache) b.ResetTimer() - b.N = 10000000 + b.N = 1000000 for i := 0; i < b.N; i++ { index := make([]byte, 8) binary.LittleEndian.PutUint64(index, uint64(i))