From 25ee10f8031d8ac21cc6a2c44b60cbabc1fada95 Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Sat, 14 Sep 2019 14:00:54 +0100 Subject: [PATCH 1/6] Add EventLogging option This introduces an option to disable or enable trace.EventLog logging of events. For backwards compatability this defaults to true. The reason for making this configurable is that under some workloads the cost of event logging is significant and so can cause performance issues. Fixes #938 --- db.go | 7 ++++++- nil_event_log.go | 15 +++++++++++++++ options.go | 12 ++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 nil_event_log.go diff --git a/db.go b/db.go index 6483df439..49c0d4bb8 100644 --- a/db.go +++ b/db.go @@ -270,13 +270,18 @@ func Open(opt Options) (db *DB, err error) { } }() + elog := noEventLog + if opt.EventLogging { + elog = trace.NewEventLog("Badger", "DB") + } + db = &DB{ imm: make([]*skl.Skiplist, 0, opt.NumMemtables), flushChan: make(chan flushTask, opt.NumMemtables), writeCh: make(chan *request, kvWriteChCapacity), opt: opt, manifest: manifestFile, - elog: trace.NewEventLog("Badger", "DB"), + elog: elog, dirLockGuard: dirLockGuard, valueDirGuard: valueDirLockGuard, orc: newOracle(opt), diff --git a/nil_event_log.go b/nil_event_log.go new file mode 100644 index 000000000..4cd42fd60 --- /dev/null +++ b/nil_event_log.go @@ -0,0 +1,15 @@ +package badger + +import "golang.org/x/net/trace" + +var ( + noEventLog trace.EventLog = nilEventLog{} +) + +type nilEventLog struct{} + +func (nel nilEventLog) Printf(format string, a ...interface{}) {} + +func (nel nilEventLog) Errorf(format string, a ...interface{}) {} + +func (nel nilEventLog) Finish() {} diff --git a/options.go b/options.go index 3a4f6cfdf..2835b1443 100644 --- a/options.go +++ b/options.go @@ -44,6 +44,7 @@ type Options struct { ReadOnly bool Truncate bool Logger Logger + EventLogging bool // Fine tuning options. @@ -114,6 +115,7 @@ func DefaultOptions(path string) Options { ValueThreshold: 32, Truncate: false, Logger: defaultLogger, + EventLogging: true, LogRotatesToFlush: 2, } } @@ -237,6 +239,16 @@ func (opt Options) WithLogger(val Logger) Options { return opt } +// WithEventLogging returns a new Options value with EventLogging set to the given value. +// +// EventLogging provides a way to enable or disable trace.EventLog logging. +// +// The default value of EventLogging is true. +func (opt Options) WithEventLogging(enabled bool) Options { + opt.EventLogging = enabled + return opt +} + // WithMaxTableSize returns a new Options value with MaxTableSize set to the given value. // // MaxTableSize sets the maximum size in bytes for each LSM table or file. From 9b56efb537324f60a877e5a3f1aa6b5a4749b3c6 Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Mon, 16 Sep 2019 09:14:12 +0100 Subject: [PATCH 2/6] Rename file and add licence text --- event_log.go | 31 +++++++++++++++++++++++++++++++ nil_event_log.go | 15 --------------- 2 files changed, 31 insertions(+), 15 deletions(-) create mode 100644 event_log.go delete mode 100644 nil_event_log.go diff --git a/event_log.go b/event_log.go new file mode 100644 index 000000000..e95311d87 --- /dev/null +++ b/event_log.go @@ -0,0 +1,31 @@ +/* + * Copyright 2019 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package badger + +import "golang.org/x/net/trace" + +var ( + noEventLog trace.EventLog = nilEventLog{} +) + +type nilEventLog struct{} + +func (nel nilEventLog) Printf(format string, a ...interface{}) {} + +func (nel nilEventLog) Errorf(format string, a ...interface{}) {} + +func (nel nilEventLog) Finish() {} diff --git a/nil_event_log.go b/nil_event_log.go deleted file mode 100644 index 4cd42fd60..000000000 --- a/nil_event_log.go +++ /dev/null @@ -1,15 +0,0 @@ -package badger - -import "golang.org/x/net/trace" - -var ( - noEventLog trace.EventLog = nilEventLog{} -) - -type nilEventLog struct{} - -func (nel nilEventLog) Printf(format string, a ...interface{}) {} - -func (nel nilEventLog) Errorf(format string, a ...interface{}) {} - -func (nel nilEventLog) Finish() {} From cb7f35cce94168f314dffab0e028809cd2ccbfe1 Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Mon, 16 Sep 2019 21:44:46 +0100 Subject: [PATCH 3/6] Apply event logging config to value log This also applies the event logging config option to the value log. --- db.go | 2 +- value.go | 7 +++++-- value_test.go | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/db.go b/db.go index 49c0d4bb8..ca687ec96 100644 --- a/db.go +++ b/db.go @@ -324,7 +324,7 @@ func Open(opt Options) (db *DB, err error) { replayCloser := y.NewCloser(1) go db.doWrites(replayCloser) - if err = db.vlog.open(db, vptr, db.replayFunction()); err != nil { + if err = db.vlog.open(db, vptr, db.replayFunction(), opt.EventLogging); err != nil { return db, y.Wrapf(err, "During db.vlog.open") } replayCloser.SignalAndWait() // Wait for replay to be applied first. diff --git a/value.go b/value.go index 2478f1935..4a914f2bc 100644 --- a/value.go +++ b/value.go @@ -787,12 +787,15 @@ func (vlog *valueLog) replayLog(lf *logFile, offset uint32, replayFn logEntry) e return nil } -func (vlog *valueLog) open(db *DB, ptr valuePointer, replayFn logEntry) error { +func (vlog *valueLog) open(db *DB, ptr valuePointer, replayFn logEntry, eventLogging bool) error { opt := db.opt vlog.opt = opt vlog.dirPath = opt.ValueDir vlog.db = db - vlog.elog = trace.NewEventLog("Badger", "Valuelog") + vlog.elog = noEventLog + if eventLogging { + vlog.elog = trace.NewEventLog("Badger", "Valuelog") + } vlog.garbageCh = make(chan struct{}, 1) // Only allow one GC at a time. vlog.lfDiscardStats = &lfDiscardStats{m: make(map[uint32]int64)} if err := vlog.populateFilesMap(); err != nil { diff --git a/value_test.go b/value_test.go index ef5674ba4..29389edbe 100644 --- a/value_test.go +++ b/value_test.go @@ -405,7 +405,7 @@ func TestValueGC4(t *testing.T) { err = kv.vlog.Close() require.NoError(t, err) - err = kv.vlog.open(kv, valuePointer{Fid: 2}, kv.replayFunction()) + err = kv.vlog.open(kv, valuePointer{Fid: 2}, kv.replayFunction(), true) require.NoError(t, err) for i := 0; i < 8; i++ { @@ -628,7 +628,7 @@ func TestPartialAppendToValueLog(t *testing.T) { // Replay value log from beginning, badger head is past k2. require.NoError(t, kv.vlog.Close()) require.NoError(t, - kv.vlog.open(kv, valuePointer{Fid: 0}, kv.replayFunction())) + kv.vlog.open(kv, valuePointer{Fid: 0}, kv.replayFunction(), true)) require.NoError(t, kv.Close()) } From 7bb38a5586ab272850309771ce8aa7fc787d731e Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Tue, 17 Sep 2019 11:45:27 +0100 Subject: [PATCH 4/6] Avoid passing extra parameter Remove the additional eventLogging parameter because it's already available as db.opts. --- db.go | 2 +- value.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db.go b/db.go index ca687ec96..49c0d4bb8 100644 --- a/db.go +++ b/db.go @@ -324,7 +324,7 @@ func Open(opt Options) (db *DB, err error) { replayCloser := y.NewCloser(1) go db.doWrites(replayCloser) - if err = db.vlog.open(db, vptr, db.replayFunction(), opt.EventLogging); err != nil { + if err = db.vlog.open(db, vptr, db.replayFunction()); err != nil { return db, y.Wrapf(err, "During db.vlog.open") } replayCloser.SignalAndWait() // Wait for replay to be applied first. diff --git a/value.go b/value.go index 4a914f2bc..346f010d8 100644 --- a/value.go +++ b/value.go @@ -787,13 +787,13 @@ func (vlog *valueLog) replayLog(lf *logFile, offset uint32, replayFn logEntry) e return nil } -func (vlog *valueLog) open(db *DB, ptr valuePointer, replayFn logEntry, eventLogging bool) error { +func (vlog *valueLog) open(db *DB, ptr valuePointer, replayFn logEntry) error { opt := db.opt vlog.opt = opt vlog.dirPath = opt.ValueDir vlog.db = db vlog.elog = noEventLog - if eventLogging { + if opt.EventLogging { vlog.elog = trace.NewEventLog("Badger", "Valuelog") } vlog.garbageCh = make(chan struct{}, 1) // Only allow one GC at a time. From 011015a08daa88e2a93ee1e9499ef0b8bd4e8178 Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Wed, 18 Sep 2019 08:48:04 +0100 Subject: [PATCH 5/6] Move event_log.go to `y` package It is going to need to be used from there and elsewhere, so move into `y` --- db.go | 2 +- value.go | 2 +- event_log.go => y/event_log.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename event_log.go => y/event_log.go (93%) diff --git a/db.go b/db.go index 49c0d4bb8..59137dd43 100644 --- a/db.go +++ b/db.go @@ -270,7 +270,7 @@ func Open(opt Options) (db *DB, err error) { } }() - elog := noEventLog + elog := y.NoEventLog if opt.EventLogging { elog = trace.NewEventLog("Badger", "DB") } diff --git a/value.go b/value.go index 346f010d8..494227cb9 100644 --- a/value.go +++ b/value.go @@ -792,7 +792,7 @@ func (vlog *valueLog) open(db *DB, ptr valuePointer, replayFn logEntry) error { vlog.opt = opt vlog.dirPath = opt.ValueDir vlog.db = db - vlog.elog = noEventLog + vlog.elog = y.NoEventLog if opt.EventLogging { vlog.elog = trace.NewEventLog("Badger", "Valuelog") } diff --git a/event_log.go b/y/event_log.go similarity index 93% rename from event_log.go rename to y/event_log.go index e95311d87..ba9dcb1f6 100644 --- a/event_log.go +++ b/y/event_log.go @@ -14,12 +14,12 @@ * limitations under the License. */ -package badger +package y import "golang.org/x/net/trace" var ( - noEventLog trace.EventLog = nilEventLog{} + NoEventLog trace.EventLog = nilEventLog{} ) type nilEventLog struct{} From bb52b9d89a6999abe3e50fae29413e68dcfc4297 Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Wed, 18 Sep 2019 08:51:40 +0100 Subject: [PATCH 6/6] Apply event logging option to watermark The option to enable/disable event logging now also works for watermarks, which was previously missed. --- txn.go | 4 ++-- y/watermark.go | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/txn.go b/txn.go index 79840cc0f..7cb2299f7 100644 --- a/txn.go +++ b/txn.go @@ -70,8 +70,8 @@ func newOracle(opt Options) *oracle { txnMark: &y.WaterMark{Name: "badger.TxnTimestamp"}, closer: y.NewCloser(2), } - orc.readMark.Init(orc.closer) - orc.txnMark.Init(orc.closer) + orc.readMark.Init(orc.closer, opt.EventLogging) + orc.txnMark.Init(orc.closer, opt.EventLogging) return orc } diff --git a/y/watermark.go b/y/watermark.go index d25a58c7d..2ff70b38a 100644 --- a/y/watermark.go +++ b/y/watermark.go @@ -68,9 +68,13 @@ type WaterMark struct { } // Init initializes a WaterMark struct. MUST be called before using it. -func (w *WaterMark) Init(closer *Closer) { +func (w *WaterMark) Init(closer *Closer, eventLogging bool) { w.markCh = make(chan mark, 100) - w.elog = trace.NewEventLog("Watermark", w.Name) + if eventLogging { + w.elog = trace.NewEventLog("Watermark", w.Name) + } else { + w.elog = NoEventLog + } go w.process(closer) }