Skip to content

Commit

Permalink
Flatten hyper tree package
Browse files Browse the repository at this point in the history
  • Loading branch information
aalda committed Feb 25, 2019
1 parent 9b16e9e commit 6a8838b
Show file tree
Hide file tree
Showing 26 changed files with 781 additions and 868 deletions.
13 changes: 6 additions & 7 deletions api/apihttp/apihttp_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@ import (

"github.com/bbva/qed/balloon"
"github.com/bbva/qed/balloon/history"
historynav "github.com/bbva/qed/balloon/history/navigation"
"github.com/bbva/qed/balloon/history/navigation"
"github.com/bbva/qed/balloon/hyper"
hypernav "github.com/bbva/qed/balloon/hyper/navigation"
"github.com/bbva/qed/hashing"
"github.com/bbva/qed/protocol"
"github.com/bbva/qed/raftwal"
Expand All @@ -57,8 +56,8 @@ func (b fakeRaftBalloon) Join(nodeID, addr string, metadata map[string]string) e
func (b fakeRaftBalloon) QueryDigestMembership(keyDigest hashing.Digest, version uint64) (*balloon.MembershipProof, error) {
return &balloon.MembershipProof{
Exists: true,
HyperProof: hyper.NewQueryProof([]byte{0x0}, []byte{0x0}, hypernav.AuditPath{}, nil),
HistoryProof: history.NewMembershipProof(0, 0, historynav.AuditPath{}, nil),
HyperProof: hyper.NewQueryProof([]byte{0x0}, []byte{0x0}, hyper.AuditPath{}, nil),
HistoryProof: history.NewMembershipProof(0, 0, navigation.AuditPath{}, nil),
CurrentVersion: 1,
QueryVersion: 1,
ActualVersion: 2,
Expand All @@ -71,8 +70,8 @@ func (b fakeRaftBalloon) QueryMembership(event []byte, version uint64) (*balloon
hasher := hashing.NewFakeXorHasher()
return &balloon.MembershipProof{
Exists: true,
HyperProof: hyper.NewQueryProof([]byte{0x0}, []byte{0x0}, hypernav.AuditPath{}, nil),
HistoryProof: history.NewMembershipProof(0, 0, historynav.AuditPath{}, nil),
HyperProof: hyper.NewQueryProof([]byte{0x0}, []byte{0x0}, hyper.AuditPath{}, nil),
HistoryProof: history.NewMembershipProof(0, 0, navigation.AuditPath{}, nil),
CurrentVersion: 1,
QueryVersion: 1,
ActualVersion: 2,
Expand All @@ -86,7 +85,7 @@ func (b fakeRaftBalloon) QueryConsistency(start, end uint64) (*balloon.Increment
ip := balloon.IncrementalProof{
Start: 2,
End: 8,
AuditPath: historynav.AuditPath{pathKey: hashing.Digest{0x00}},
AuditPath: navigation.AuditPath{pathKey: hashing.Digest{0x00}},
Hasher: hashing.NewFakeXorHasher(),
}
return &ip, nil
Expand Down
116 changes: 116 additions & 0 deletions balloon/hyper/batch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*
Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package hyper

import (
"fmt"
"strings"

"github.com/bbva/qed/hashing"
)

type batchNode struct {
batch [][]byte
nodeSize int // in bytes
}

func newEmptyBatchNode(nodeSize int) *batchNode {
return &batchNode{
nodeSize: nodeSize,
batch: make([][]byte, 31, 31),
}
}

func newBatchNode(nodeSize int, batch [][]byte) *batchNode {
return &batchNode{
nodeSize: nodeSize,
batch: batch,
}
}

func (b batchNode) String() string {
var strs []string
for i, n := range b.batch {
strs = append(strs, fmt.Sprintf("[%d - %#x]", i, n))
}
return strings.Join(strs, "\n")
}

func (b batchNode) HasLeafAt(i int8) bool {
return len(b.batch[i]) > 0 && b.batch[i][b.nodeSize] == byte(1)
}

func (b batchNode) AddHashAt(i int8, value []byte) {
b.batch[i] = append(value, byte(0))
}

func (b batchNode) AddLeafAt(i int8, hash hashing.Digest, key, value []byte) {
b.batch[i] = append(hash, byte(1))
b.batch[2*i+1] = append(key, byte(2))
b.batch[2*i+2] = append(value, byte(2))
}

func (b batchNode) GetLeafKVAt(i int8) ([]byte, []byte) {
return b.batch[2*i+1][:b.nodeSize], b.batch[2*i+2][:b.nodeSize]
}

func (b batchNode) HasElementAt(i int8) bool {
return len(b.batch[i]) > 0
}

func (b batchNode) GetElementAt(i int8) []byte {
return b.batch[i][:b.nodeSize]
}

func (b batchNode) ResetElementAt(i int8) {
b.batch[i] = nil
}

func (b batchNode) Serialize() []byte {
serialized := make([]byte, 4)
for i := uint16(0); i < 31; i++ {
if len(b.batch[i]) != 0 {
bitSet(serialized, i)
serialized = append(serialized, b.batch[i]...)
}
}
return serialized
}

func parseBatch(nodeSize int, value []byte) [][]byte {
batch := make([][]byte, 31, 31) // 31 nodes (including the root)
bitmap := value[:4] // the first 4 bytes define the bitmap
size := nodeSize + 1

j := 0
for i := 0; i < 31; i++ {
if bitIsSet(bitmap, i) {
batch[i] = value[4+size*j : 4+size*(j+1)]
j++
}
}

return batch
}

func parseBatchNode(nodeSize int, value []byte) *batchNode {
return newBatchNode(nodeSize, parseBatch(nodeSize, value))
}

func bitIsSet(bits []byte, i int) bool {
return bits[i/8]&(1<<uint(7-i%8)) != 0
}
63 changes: 31 additions & 32 deletions balloon/hyper/pruning/insert.go → balloon/hyper/insert.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,22 @@
limitations under the License.
*/

package pruning
package hyper

import (
"bytes"
"sort"

"github.com/bbva/qed/balloon/hyper/navigation"
"github.com/bbva/qed/util"
)

type Leaf struct {
type leaf struct {
Index, Value []byte
}

type Leaves []Leaf
type leaves []leaf

func (l Leaves) InsertSorted(leaf Leaf) Leaves {
func (l leaves) InsertSorted(leaf leaf) leaves {

if len(l) == 0 {
l = append(l, leaf)
Expand All @@ -52,21 +51,21 @@ func (l Leaves) InsertSorted(leaf Leaf) Leaves {

}

func (l Leaves) Split(index []byte) (left, right Leaves) {
func (l leaves) Split(index []byte) (left, right leaves) {
// the smallest index i where l[i].Index >= index
splitIndex := sort.Search(len(l), func(i int) bool {
return bytes.Compare(l[i].Index, index) >= 0
})
return l[:splitIndex], l[splitIndex:]
}

type TraverseBatch func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack)
type traverseBatch func(pos position, leaves leaves, batch *batchNode, iBatch int8, ops *operationsStack)

func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches BatchLoader) *OperationsStack {
func pruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches batchLoader) *operationsStack {

var traverse, traverseThroughCache, traverseAfterCache TraverseBatch
var traverse, traverseThroughCache, traverseAfterCache traverseBatch

traverse = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) {
traverse = func(pos position, leaves leaves, batch *batchNode, iBatch int8, ops *operationsStack) {
if batch == nil {
batch = batches.Load(pos)
}
Expand All @@ -77,7 +76,7 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
}
}

traverseThroughCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) {
traverseThroughCache = func(pos position, leaves leaves, batch *batchNode, iBatch int8, ops *operationsStack) {

if len(leaves) == 0 { // discarded branch
if batch.HasElementAt(iBatch) {
Expand All @@ -91,26 +90,26 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
// at the end of a batch tree
if iBatch > 0 && pos.Height%4 == 0 {
traverse(pos, leaves, nil, 0, ops)
ops.Push(updateBatchNode(pos, iBatch, batch))
ops.Push(updatebatchNode(pos, iBatch, batch))
return
}

// on an internal node with more than one leaf

rightPos := pos.Right()
leftLeaves, rightLeaves := leaves.Split(rightPos.Index)
leftleaves, rightleaves := leaves.Split(rightPos.Index)

traverseThroughCache(pos.Left(), leftLeaves, batch, 2*iBatch+1, ops)
traverseThroughCache(rightPos, rightLeaves, batch, 2*iBatch+2, ops)
traverseThroughCache(pos.Left(), leftleaves, batch, 2*iBatch+1, ops)
traverseThroughCache(rightPos, rightleaves, batch, 2*iBatch+2, ops)

ops.PushAll(innerHash(pos), updateBatchNode(pos, iBatch, batch))
ops.PushAll(innerHash(pos), updatebatchNode(pos, iBatch, batch))
if iBatch == 0 { // it's the root of the batch tree
ops.Push(putInCache(pos, batch))
}

}

traverseAfterCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) {
traverseAfterCache = func(pos position, leaves leaves, batch *batchNode, iBatch int8, ops *operationsStack) {

if len(leaves) == 0 { // discarded branch
if batch.HasElementAt(iBatch) {
Expand All @@ -128,12 +127,12 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
panic("Oops, something went wrong. We cannot have more than one leaf at the end of the main tree")
}
// create or update the leaf with a new shortcut
newBatch := NewEmptyBatchNode(len(pos.Index))
newBatch := newEmptyBatchNode(len(pos.Index))
ops.PushAll(
leafHash(pos, leaves[0].Value),
updateBatchShortcut(pos, 0, newBatch, leaves[0].Index, leaves[0].Value),
mutateBatch(pos, newBatch),
updateBatchNode(pos, iBatch, batch),
updatebatchNode(pos, iBatch, batch),
)
return
}
Expand All @@ -143,22 +142,22 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
if len(leaves) > 1 {
// with more than one leaf to insert -> it's impossible to be a shortcut leaf
traverse(pos, leaves, nil, 0, ops)
ops.Push(updateBatchNode(pos, iBatch, batch))
ops.Push(updatebatchNode(pos, iBatch, batch))
return
}
// with only one leaf to insert -> continue traversing
if batch.HasElementAt(iBatch) {
traverse(pos, leaves, nil, 0, ops)
ops.Push(updateBatchNode(pos, iBatch, batch))
ops.Push(updatebatchNode(pos, iBatch, batch))
return
}
// nil value (no previous node stored) so create a new shortcut batch
newBatch := NewEmptyBatchNode(len(pos.Index))
newBatch := newEmptyBatchNode(len(pos.Index))
ops.PushAll(
leafHash(pos, leaves[0].Value),
updateBatchShortcut(pos, 0, newBatch, leaves[0].Index, leaves[0].Value),
mutateBatch(pos, newBatch),
updateBatchNode(pos, iBatch, batch),
updatebatchNode(pos, iBatch, batch),
)
return
}
Expand All @@ -182,7 +181,7 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
if batch.HasLeafAt(iBatch) {
// push down leaf
key, value := batch.GetLeafKVAt(iBatch)
leaves = leaves.InsertSorted(Leaf{key, value})
leaves = leaves.InsertSorted(leaf{key, value})
batch.ResetElementAt(iBatch)
batch.ResetElementAt(2*iBatch + 1)
batch.ResetElementAt(2*iBatch + 2)
Expand All @@ -193,23 +192,23 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches

// on an internal node with more than one leaf
rightPos := pos.Right()
leftLeaves, rightLeaves := leaves.Split(rightPos.Index)
leftleaves, rightleaves := leaves.Split(rightPos.Index)

traverseAfterCache(pos.Left(), leftLeaves, batch, 2*iBatch+1, ops)
traverseAfterCache(rightPos, rightLeaves, batch, 2*iBatch+2, ops)
traverseAfterCache(pos.Left(), leftleaves, batch, 2*iBatch+1, ops)
traverseAfterCache(rightPos, rightleaves, batch, 2*iBatch+2, ops)

ops.PushAll(innerHash(pos), updateBatchNode(pos, iBatch, batch))
ops.PushAll(innerHash(pos), updatebatchNode(pos, iBatch, batch))
if iBatch == 0 { // at root node -> mutate batch
ops.Push(mutateBatch(pos, batch))
}

}

ops := NewOperationsStack()
ops := newOperationsStack()
version := util.AddPaddingToBytes(value, len(index))
version = version[len(version)-len(index):] // TODO GET RID OF THIS: used only to pass tests
leaves := make(Leaves, 0)
leaves = leaves.InsertSorted(Leaf{index, version})
traverse(navigation.NewRootPosition(uint16(len(index))), leaves, nil, 0, ops)
leaves := make(leaves, 0)
leaves = leaves.InsertSorted(leaf{index, version})
traverse(newRootPosition(uint16(len(index))), leaves, nil, 0, ops)
return ops
}
Loading

0 comments on commit 6a8838b

Please sign in to comment.