From 46e42d005d4f6ee3bff7845429682aa507804bdf Mon Sep 17 00:00:00 2001 From: Alvaro Alda Date: Wed, 28 Nov 2018 15:40:22 +0100 Subject: [PATCH] Replace freecache with fastcache --- balloon/balloon.go | 2 +- balloon/cache/fast.go | 76 ++++++++++++++++++++++++++++++++++++++ balloon/cache/fast_test.go | 73 ++++++++++++++++++++++++++++++++++++ balloon/hyper/tree_test.go | 4 +- go.mod | 2 +- go.sum | 2 + 6 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 balloon/cache/fast.go create mode 100644 balloon/cache/fast_test.go diff --git a/balloon/balloon.go b/balloon/balloon.go index cfbe3be6f..a693fdec3 100644 --- a/balloon/balloon.go +++ b/balloon/balloon.go @@ -47,7 +47,7 @@ type Balloon struct { func NewBalloon(store storage.Store, hasherF func() hashing.Hasher) (*Balloon, error) { // create caches - hyperCache := cache.NewFreeCache(int(hyper.CacheSize)) + hyperCache := cache.NewFastCache(hyper.CacheSize) // create trees historyTree := history.NewHistoryTree(hasherF, store, 30) diff --git a/balloon/cache/fast.go b/balloon/cache/fast.go new file mode 100644 index 000000000..62865b08d --- /dev/null +++ b/balloon/cache/fast.go @@ -0,0 +1,76 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cache + +import ( + "github.com/VictoriaMetrics/fastcache" + "github.com/bbva/qed/balloon/navigator" + "github.com/bbva/qed/hashing" + "github.com/bbva/qed/storage" +) + +type FastCache struct { + cached *fastcache.Cache +} + +func NewFastCache(maxBytes int64) *FastCache { + cache := fastcache.New(int(maxBytes)) + return &FastCache{cached: cache} +} + +func (c FastCache) Get(pos navigator.Position) (hashing.Digest, bool) { + value := c.cached.Get(nil, pos.Bytes()) + if value == nil { + return nil, false + } + return value, true +} + +func (c *FastCache) Put(pos navigator.Position, value hashing.Digest) { + c.cached.Set(pos.Bytes(), value) +} + +func (c *FastCache) Fill(r storage.KVPairReader) (err error) { + defer r.Close() + for { + entries := make([]*storage.KVPair, 100) + n, err := r.Read(entries) + if err != nil || n == 0 { + break + } + for _, entry := range entries { + if entry != nil { + c.cached.Set(entry.Key, entry.Value) + } + } + } + return nil +} + +func (c FastCache) Size() int { + var s fastcache.Stats + c.cached.UpdateStats(&s) + return int(s.EntriesCount) +} + +func (c FastCache) Equal(o *FastCache) bool { + // can only check size and entries count + var stats, oStats fastcache.Stats + c.cached.UpdateStats(&stats) + o.cached.UpdateStats(&oStats) + return stats.BytesSize == oStats.BytesSize && stats.EntriesCount == oStats.EntriesCount +} diff --git a/balloon/cache/fast_test.go b/balloon/cache/fast_test.go new file mode 100644 index 000000000..87dbbec7a --- /dev/null +++ b/balloon/cache/fast_test.go @@ -0,0 +1,73 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package cache + +import ( + "testing" + + "github.com/bbva/qed/balloon/navigator" + "github.com/bbva/qed/hashing" + "github.com/bbva/qed/util" + "github.com/stretchr/testify/require" +) + +func TestFastCache(t *testing.T) { + + testCases := []struct { + pos navigator.Position + value hashing.Digest + cached bool + }{ + {&navigator.FakePosition{[]byte{0x0}, 0}, hashing.Digest{0x1}, true}, + {&navigator.FakePosition{[]byte{0x1}, 0}, hashing.Digest{0x2}, true}, + {&navigator.FakePosition{[]byte{0x2}, 0}, hashing.Digest{0x3}, false}, + } + + cache := NewFastCache(100 * 1024) + + for i, c := range testCases { + if c.cached { + cache.Put(c.pos, c.value) + } + + cachedValue, ok := cache.Get(c.pos) + + if c.cached { + require.Truef(t, ok, "The key should exists in cache in test case %d", i) + require.Equalf(t, c.value, cachedValue, "The cached value should be equal to stored value in test case %d", i) + } else { + require.Falsef(t, ok, "The key should not exist in cache in test case %d", i) + } + } +} + +func TestFillFastCache(t *testing.T) { + + numElems := uint64(10000) + cache := NewFastCache(10000 * 1024) + reader := NewFakeKVPairReader(numElems) + + err := cache.Fill(reader) + + require.NoError(t, err) + require.Truef(t, reader.Remaining == 0, "All elements should be cached. Remaining: %d", reader.Remaining) + + for i := uint64(0); i < numElems; i++ { + pos := &navigator.FakePosition{util.Uint64AsBytes(i), 0} + _, ok := cache.Get(pos) + require.Truef(t, ok, "The element in position %v should be in cache", pos) + } +} diff --git a/balloon/hyper/tree_test.go b/balloon/hyper/tree_test.go index 46c9ffe45..42ea66c55 100644 --- a/balloon/hyper/tree_test.go +++ b/balloon/hyper/tree_test.go @@ -289,8 +289,8 @@ func BenchmarkAdd(b *testing.B) { defer closeF() hasher := hashing.NewSha256Hasher() - freeCache := cache.NewFreeCache(int(CacheSize)) - tree := NewHyperTree(hashing.NewSha256Hasher, store, freeCache) + fastCache := cache.NewFastCache(CacheSize) + tree := NewHyperTree(hashing.NewSha256Hasher, store, fastCache) b.ResetTimer() b.N = 100000 diff --git a/go.mod b/go.mod index 2dd8c4dd9..5d6dda8f2 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/bbva/qed require ( + github.com/VictoriaMetrics/fastcache v1.0.2 github.com/bbva/raft-badger v0.1.1 github.com/boltdb/bolt v1.3.1 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/coocood/freecache v1.0.1 github.com/coreos/bbolt v1.3.0 github.com/dgraph-io/badger v1.5.4 diff --git a/go.sum b/go.sum index 9c27c3a68..71cff7062 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20170702084017-28f7e881ca57/go.mod h1:bOv github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/VictoriaMetrics/fastcache v1.0.2/go.mod h1:4zLf+tCNHtaJzQfIkJ+vBXS1O98gXA/KbKtUv/W1Tdo= +github.com/allegro/bigcache v1.1.0/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78 h1:mdRSArcFLfW0VoL34LZAKSz6LkkK4jFxVx2xYavACMg= github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=