From cc3c5cd9d6253bf55dedd714032def21bbffb81b Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Tue, 17 Mar 2020 00:58:53 +0530 Subject: [PATCH] Update badger to commit 91c31ebe8c22 (#4935) Important changes ``` - Changes to overlap check in compaction. - Remove 'this entry should've been caught' log. - Changes to write stalling on levels 0 and 1. - Compression is disabled by default in Badger. - Bloom filter caching in a separate ristretto cache. - Compression/Encryption in background. - Disable cache by default in badger. ``` The following new changes are being added from badger `git log ab4352b00a17...91c31ebe8c22` ``` 91c31eb Disable cache by default (#1257) eaf64c0 Add separate cache for bloom filters (#1260) 1bcbefc Add BypassDirLock option (#1243) c6c1e5e Add support for watching nil prefix in subscribe API (#1246) b13b927 Compress/Encrypt Blocks in the background (#1227) bdb2b13 fix changelog for v2.0.2 (#1244) 8dbc982 Add Dkron to README (#1241) 3d95b94 Remove coveralls from Travis Build(#1219) 5b4c0a6 Fix ValueThreshold for in-memory mode (#1235) 617ed7c Initialize vlog before starting compactions in db.Open (#1226) e908818 Update CHANGELOG for Badger 2.0.2 release. (#1230) bce069c Fix int overflow for 32bit (#1216) e029e93 Remove ExampleDB_Subscribe Test (#1214) 8734e3a Add missing package to README for badger.NewEntry (#1223) 78d405a Replace t.Fatal with require.NoError in tests (#1213) c51748e Fix flaky TestPageBufferReader2 test (#1210) eee1602 Change else-if statements to idiomatic switch statements. (#1207) 3e25d77 Rework concurrency semantics of valueLog.maxFid (#1184) (#1187) 4676ca9 Add support for caching bloomfilters (#1204) c3333a5 Disable compression and set ZSTD Compression Level to 1 (#1191) 0acb3f6 Fix L0/L1 stall test (#1201) 7e5a956 Support disabling the cache completely. (#1183) (#1185) 82381ac Update ristretto to version 8f368f2 (#1195) 3747be5 Improve write stalling on level 0 and 1 5870b7b Run all tests on CI (#1189) 01a00cb Add Jaegar to list of projects (#1192) 9d6512b Use fastRand instead of locked-rand in skiplist (#1173) 2698bfc Avoid sync in inmemory mode (#1190) 2a90c66 Remove the 'this entry should've caught' log from value.go (#1170) 0a06173 Fix checkOverlap in compaction (#1166) 0f2e629 Fix windows build (#1177) 03af216 Fix commit sha for WithInMemory in CHANGELOG. (#1172) 23a73cd Update CHANGELOG for v2.0.1 release. (#1181) 465f28a Cast sz to uint32 to fix compilation on 32 bit (#1175) ea01d38 Rename option builder from WithInmemory to WithInMemory. (#1169) df99253 Remove ErrGCInMemoryMode in CHANGELOG. (#1171) 8dfdd6d Adding changes for 2.0.1 so far (#1168) ``` --- dgraph/cmd/live/run.go | 6 ++++-- dgraph/cmd/zero/run.go | 5 ++--- go.mod | 3 +-- go.sum | 9 ++++----- worker/server_state.go | 9 +++++++-- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/dgraph/cmd/live/run.go b/dgraph/cmd/live/run.go index b147d71192d..97b35367575 100644 --- a/dgraph/cmd/live/run.go +++ b/dgraph/cmd/live/run.go @@ -337,8 +337,10 @@ func setup(opts batchMutationOptions, dc *dgo.Dgraph) *loader { var err error db, err = badger.Open(badger.DefaultOptions(opt.clientDir). WithTableLoadingMode(bopt.MemoryMap). - // TODO(Ibrahim): Remove compression level once badger is updated. - WithSyncWrites(false).WithZSTDCompressionLevel(1)) + WithCompression(bopt.ZSTD). + WithSyncWrites(false). + WithLoadBloomsOnOpen(false). + WithZSTDCompressionLevel(3)) x.Checkf(err, "Error while creating badger KV posting store") } diff --git a/dgraph/cmd/zero/run.go b/dgraph/cmd/zero/run.go index b6cf6a49c43..0cad468c495 100644 --- a/dgraph/cmd/zero/run.go +++ b/dgraph/cmd/zero/run.go @@ -217,10 +217,9 @@ func run() { // Open raft write-ahead log and initialize raft node. x.Checkf(os.MkdirAll(opts.w, 0700), "Error while creating WAL dir.") kvOpt := badger.LSMOnlyOptions(opts.w).WithSyncWrites(false).WithTruncate(true). - WithValueLogFileSize(64 << 20).WithMaxCacheSize(10 << 20) + WithValueLogFileSize(64 << 20).WithMaxCacheSize(10 << 20).WithLoadBloomsOnOpen(false) - // TOOD(Ibrahim): Remove this once badger is updated. - kvOpt.ZSTDCompressionLevel = 1 + kvOpt.ZSTDCompressionLevel = 3 kv, err := badger.Open(kvOpt) x.Checkf(err, "Error while opening WAL store") diff --git a/go.mod b/go.mod index 6b7df9e1414..5b2d6a9a948 100644 --- a/go.mod +++ b/go.mod @@ -16,9 +16,8 @@ require ( github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f // indirect github.com/blevesearch/snowballstem v0.0.0-20180110192139-26b06a2c243d // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd - github.com/dgraph-io/badger/v2 v2.0.1-0.20191220102048-ab4352b00a17 + github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200316175624-91c31ebe8c22 github.com/dgraph-io/dgo/v2 v2.2.0 - github.com/dgraph-io/ristretto v0.0.1 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgryski/go-farm v0.0.0-20191112170834-c2139c5d712b github.com/dgryski/go-groupvarint v0.0.0-20190318181831-5ce5df8ca4e1 diff --git a/go.sum b/go.sum index a49abf7b547..b6f3c495231 100644 --- a/go.sum +++ b/go.sum @@ -76,13 +76,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger/v2 v2.0.1-0.20191220102048-ab4352b00a17 h1:BxXd8isFVcup+jlLnnEN22xhB+UwLMgAgk56kDxy9AY= -github.com/dgraph-io/badger/v2 v2.0.1-0.20191220102048-ab4352b00a17/go.mod h1:YoRSIp1LmAJ7zH7tZwRvjNMUYLxB4wl3ebYkaIruZ04= +github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200316175624-91c31ebe8c22 h1:X2g/STOldw/iNkdP9BHI/8zHe4N/Lkp8Z4jLyywtnCI= +github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200316175624-91c31ebe8c22/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= github.com/dgraph-io/dgo/v2 v2.2.0 h1:qYbm6mEF3wuKiRpgNOldk6PmPbBJFwj6vL7I7dTSdyc= github.com/dgraph-io/dgo/v2 v2.2.0/go.mod h1:LJCkLxm5fUMcU+yb8gHFjHt7ChgNuz3YnQQ6MQkmscI= -github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs= -github.com/dgraph-io/ristretto v0.0.1 h1:cJwdnj42uV8Jg4+KLrYovLiCgIfz9wtWm6E6KA+1tLs= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= diff --git a/worker/server_state.go b/worker/server_state.go index b230454618e..71373168032 100644 --- a/worker/server_state.go +++ b/worker/server_state.go @@ -131,8 +131,9 @@ func (s *ServerState) initStorage() { glog.Infof("Opening write-ahead log BadgerDB with options: %+v\n", opt) opt.EncryptionKey = key - // TODO(Ibrahim): Remove this once badger is updated. - opt.ZSTDCompressionLevel = 1 + opt.ZSTDCompressionLevel = 3 + opt.Compression = options.ZSTD + opt.LoadBloomsOnOpen = false s.WALstore, err = badger.Open(opt) x.Checkf(err, "Error while creating badger KV WAL store") @@ -153,6 +154,10 @@ func (s *ServerState) initStorage() { glog.Infof("Opening postings BadgerDB with options: %+v\n", opt) opt.EncryptionKey = key + opt.Compression = options.ZSTD + opt.ZSTDCompressionLevel = 3 + opt.LoadBloomsOnOpen = false + s.Pstore, err = badger.OpenManaged(opt) x.Checkf(err, "Error while creating badger KV posting store")