Skip to content

Commit

Permalink
Merge pull request #72 from aalda/flatten_hyper
Browse files Browse the repository at this point in the history
Flatten hyper tree package
  • Loading branch information
aalda authored Feb 25, 2019
2 parents 9b16e9e + 7916680 commit a4d04a9
Show file tree
Hide file tree
Showing 26 changed files with 765 additions and 852 deletions.
13 changes: 6 additions & 7 deletions api/apihttp/apihttp_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@ import (

"github.com/bbva/qed/balloon"
"github.com/bbva/qed/balloon/history"
historynav "github.com/bbva/qed/balloon/history/navigation"
"github.com/bbva/qed/balloon/history/navigation"
"github.com/bbva/qed/balloon/hyper"
hypernav "github.com/bbva/qed/balloon/hyper/navigation"
"github.com/bbva/qed/hashing"
"github.com/bbva/qed/protocol"
"github.com/bbva/qed/raftwal"
Expand All @@ -57,8 +56,8 @@ func (b fakeRaftBalloon) Join(nodeID, addr string, metadata map[string]string) e
func (b fakeRaftBalloon) QueryDigestMembership(keyDigest hashing.Digest, version uint64) (*balloon.MembershipProof, error) {
return &balloon.MembershipProof{
Exists: true,
HyperProof: hyper.NewQueryProof([]byte{0x0}, []byte{0x0}, hypernav.AuditPath{}, nil),
HistoryProof: history.NewMembershipProof(0, 0, historynav.AuditPath{}, nil),
HyperProof: hyper.NewQueryProof([]byte{0x0}, []byte{0x0}, hyper.AuditPath{}, nil),
HistoryProof: history.NewMembershipProof(0, 0, navigation.AuditPath{}, nil),
CurrentVersion: 1,
QueryVersion: 1,
ActualVersion: 2,
Expand All @@ -71,8 +70,8 @@ func (b fakeRaftBalloon) QueryMembership(event []byte, version uint64) (*balloon
hasher := hashing.NewFakeXorHasher()
return &balloon.MembershipProof{
Exists: true,
HyperProof: hyper.NewQueryProof([]byte{0x0}, []byte{0x0}, hypernav.AuditPath{}, nil),
HistoryProof: history.NewMembershipProof(0, 0, historynav.AuditPath{}, nil),
HyperProof: hyper.NewQueryProof([]byte{0x0}, []byte{0x0}, hyper.AuditPath{}, nil),
HistoryProof: history.NewMembershipProof(0, 0, navigation.AuditPath{}, nil),
CurrentVersion: 1,
QueryVersion: 1,
ActualVersion: 2,
Expand All @@ -86,7 +85,7 @@ func (b fakeRaftBalloon) QueryConsistency(start, end uint64) (*balloon.Increment
ip := balloon.IncrementalProof{
Start: 2,
End: 8,
AuditPath: historynav.AuditPath{pathKey: hashing.Digest{0x00}},
AuditPath: navigation.AuditPath{pathKey: hashing.Digest{0x00}},
Hasher: hashing.NewFakeXorHasher(),
}
return &ip, nil
Expand Down
116 changes: 116 additions & 0 deletions balloon/hyper/batch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*
Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package hyper

import (
"fmt"
"strings"

"github.com/bbva/qed/hashing"
)

type batchNode struct {
batch [][]byte
nodeSize int // in bytes
}

func newEmptyBatchNode(nodeSize int) *batchNode {
return &batchNode{
nodeSize: nodeSize,
batch: make([][]byte, 31, 31),
}
}

func newBatchNode(nodeSize int, batch [][]byte) *batchNode {
return &batchNode{
nodeSize: nodeSize,
batch: batch,
}
}

func (b batchNode) String() string {
var strs []string
for i, n := range b.batch {
strs = append(strs, fmt.Sprintf("[%d - %#x]", i, n))
}
return strings.Join(strs, "\n")
}

func (b batchNode) HasLeafAt(i int8) bool {
return len(b.batch[i]) > 0 && b.batch[i][b.nodeSize] == byte(1)
}

func (b batchNode) AddHashAt(i int8, value []byte) {
b.batch[i] = append(value, byte(0))
}

func (b batchNode) AddLeafAt(i int8, hash hashing.Digest, key, value []byte) {
b.batch[i] = append(hash, byte(1))
b.batch[2*i+1] = append(key, byte(2))
b.batch[2*i+2] = append(value, byte(2))
}

func (b batchNode) GetLeafKVAt(i int8) ([]byte, []byte) {
return b.batch[2*i+1][:b.nodeSize], b.batch[2*i+2][:b.nodeSize]
}

func (b batchNode) HasElementAt(i int8) bool {
return len(b.batch[i]) > 0
}

func (b batchNode) GetElementAt(i int8) []byte {
return b.batch[i][:b.nodeSize]
}

func (b batchNode) ResetElementAt(i int8) {
b.batch[i] = nil
}

func (b batchNode) Serialize() []byte {
serialized := make([]byte, 4)
for i := uint16(0); i < 31; i++ {
if len(b.batch[i]) != 0 {
bitSet(serialized, i)
serialized = append(serialized, b.batch[i]...)
}
}
return serialized
}

func parseBatch(nodeSize int, value []byte) [][]byte {
batch := make([][]byte, 31, 31) // 31 nodes (including the root)
bitmap := value[:4] // the first 4 bytes define the bitmap
size := nodeSize + 1

j := 0
for i := 0; i < 31; i++ {
if bitIsSet(bitmap, i) {
batch[i] = value[4+size*j : 4+size*(j+1)]
j++
}
}

return batch
}

func parseBatchNode(nodeSize int, value []byte) *batchNode {
return newBatchNode(nodeSize, parseBatch(nodeSize, value))
}

func bitIsSet(bits []byte, i int) bool {
return bits[i/8]&(1<<uint(7-i%8)) != 0
}
37 changes: 18 additions & 19 deletions balloon/hyper/pruning/insert.go → balloon/hyper/insert.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,22 @@
limitations under the License.
*/

package pruning
package hyper

import (
"bytes"
"sort"

"github.com/bbva/qed/balloon/hyper/navigation"
"github.com/bbva/qed/util"
)

type Leaf struct {
type leaf struct {
Index, Value []byte
}

type Leaves []Leaf
type leaves []leaf

func (l Leaves) InsertSorted(leaf Leaf) Leaves {
func (l leaves) InsertSorted(leaf leaf) leaves {

if len(l) == 0 {
l = append(l, leaf)
Expand All @@ -52,21 +51,21 @@ func (l Leaves) InsertSorted(leaf Leaf) Leaves {

}

func (l Leaves) Split(index []byte) (left, right Leaves) {
func (l leaves) Split(index []byte) (left, right leaves) {
// the smallest index i where l[i].Index >= index
splitIndex := sort.Search(len(l), func(i int) bool {
return bytes.Compare(l[i].Index, index) >= 0
})
return l[:splitIndex], l[splitIndex:]
}

type TraverseBatch func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack)
type traverseBatch func(pos position, leaves leaves, batch *batchNode, iBatch int8, ops *operationsStack)

func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches BatchLoader) *OperationsStack {
func pruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches batchLoader) *operationsStack {

var traverse, traverseThroughCache, traverseAfterCache TraverseBatch
var traverse, traverseThroughCache, traverseAfterCache traverseBatch

traverse = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) {
traverse = func(pos position, leaves leaves, batch *batchNode, iBatch int8, ops *operationsStack) {
if batch == nil {
batch = batches.Load(pos)
}
Expand All @@ -77,7 +76,7 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
}
}

traverseThroughCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) {
traverseThroughCache = func(pos position, leaves leaves, batch *batchNode, iBatch int8, ops *operationsStack) {

if len(leaves) == 0 { // discarded branch
if batch.HasElementAt(iBatch) {
Expand Down Expand Up @@ -110,7 +109,7 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches

}

traverseAfterCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) {
traverseAfterCache = func(pos position, leaves leaves, batch *batchNode, iBatch int8, ops *operationsStack) {

if len(leaves) == 0 { // discarded branch
if batch.HasElementAt(iBatch) {
Expand All @@ -128,7 +127,7 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
panic("Oops, something went wrong. We cannot have more than one leaf at the end of the main tree")
}
// create or update the leaf with a new shortcut
newBatch := NewEmptyBatchNode(len(pos.Index))
newBatch := newEmptyBatchNode(len(pos.Index))
ops.PushAll(
leafHash(pos, leaves[0].Value),
updateBatchShortcut(pos, 0, newBatch, leaves[0].Index, leaves[0].Value),
Expand All @@ -153,7 +152,7 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
return
}
// nil value (no previous node stored) so create a new shortcut batch
newBatch := NewEmptyBatchNode(len(pos.Index))
newBatch := newEmptyBatchNode(len(pos.Index))
ops.PushAll(
leafHash(pos, leaves[0].Value),
updateBatchShortcut(pos, 0, newBatch, leaves[0].Index, leaves[0].Value),
Expand Down Expand Up @@ -182,7 +181,7 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches
if batch.HasLeafAt(iBatch) {
// push down leaf
key, value := batch.GetLeafKVAt(iBatch)
leaves = leaves.InsertSorted(Leaf{key, value})
leaves = leaves.InsertSorted(leaf{key, value})
batch.ResetElementAt(iBatch)
batch.ResetElementAt(2*iBatch + 1)
batch.ResetElementAt(2*iBatch + 2)
Expand All @@ -205,11 +204,11 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches

}

ops := NewOperationsStack()
ops := newOperationsStack()
version := util.AddPaddingToBytes(value, len(index))
version = version[len(version)-len(index):] // TODO GET RID OF THIS: used only to pass tests
leaves := make(Leaves, 0)
leaves = leaves.InsertSorted(Leaf{index, version})
traverse(navigation.NewRootPosition(uint16(len(index))), leaves, nil, 0, ops)
leaves := make(leaves, 0)
leaves = leaves.InsertSorted(leaf{index, version})
traverse(newRootPosition(uint16(len(index))), leaves, nil, 0, ops)
return ops
}
Loading

0 comments on commit a4d04a9

Please sign in to comment.