Convert to bbolt

cmd/grpc.ext/grpc.go

@@ -9,7 +9,6 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/boltdb/bolt"
 	"github.com/go-kit/kit/log/level"
 	"github.com/kolide/kit/env"
 	"github.com/kolide/kit/logutil"
@@ -21,6 +20,7 @@ import (
 	"github.com/kolide/osquery-go/plugin/distributed"
 	osquery_logger "github.com/kolide/osquery-go/plugin/logger"
 	"github.com/pkg/errors"
+	"go.etcd.io/bbolt"
 )
 
 func main() {
@@ -84,7 +84,7 @@ func main() {
 		LoggingInterval: loggingInterval,
 	}
 
-	db, err := bolt.Open(filepath.Join(rootDirectory, "launcher.db"), 0600, nil)
+	db, err := bbolt.Open(filepath.Join(rootDirectory, "launcher.db"), 0600, nil)
 	if err != nil {
 		logutil.Fatal(logger, "err", errors.Wrap(err, "open local store"), "stack", fmt.Sprintf("%+v", err))
 	}

cmd/launcher/control.go

@@ -5,16 +5,16 @@ package main
 import (
 	"context"
 
-	"github.com/boltdb/bolt"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/kolide/kit/actor"
 	"github.com/kolide/launcher/pkg/control"
 	"github.com/kolide/launcher/pkg/launcher"
 	"github.com/pkg/errors"
+	"go.etcd.io/bbolt"
 )
 
-func createControl(ctx context.Context, db *bolt.DB, logger log.Logger, opts *launcher.Options) (*actor.Actor, error) {
+func createControl(ctx context.Context, db *bbolt.DB, logger log.Logger, opts *launcher.Options) (*actor.Actor, error) {
 	level.Debug(logger).Log("msg", "creating control client")
 
 	controlOpts := []control.Option{

cmd/launcher/control_windows.go

@@ -5,16 +5,16 @@ package main
 import (
 	"context"
 
-	"github.com/boltdb/bolt"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/kolide/kit/actor"
 	"github.com/kolide/launcher/pkg/launcher"
+	"go.etcd.io/bbolt"
 )
 
 // createControl creates a no-op actor, as the control server isn't
 // yet supported on windows.
-func createControl(ctx context.Context, db *bolt.DB, logger log.Logger, opts *launcher.Options) (*actor.Actor, error) {
+func createControl(ctx context.Context, db *bbolt.DB, logger log.Logger, opts *launcher.Options) (*actor.Actor, error) {
 	level.Info(logger).Log("msg", "Cannot create control channel for windows, ignoring")
 
 	return nil, nil

cmd/launcher/extension.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io/ioutil"
 
-	"github.com/boltdb/bolt"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/kolide/kit/actor"
@@ -22,11 +21,12 @@ import (
 	"github.com/kolide/osquery-go/plugin/distributed"
 	osquerylogger "github.com/kolide/osquery-go/plugin/logger"
 	"github.com/pkg/errors"
+	"go.etcd.io/bbolt"
 )
 
 // TODO: the extension, runtime, and client are all kind of entangled
 // here. Untangle the underlying libraries and separate into units
-func createExtensionRuntime(ctx context.Context, db *bolt.DB, launcherClient service.KolideService, opts *launcher.Options) (
+func createExtensionRuntime(ctx context.Context, db *bbolt.DB, launcherClient service.KolideService, opts *launcher.Options) (
 	run *actor.Actor,
 	restart func() error, // restart osqueryd runner
 	shutdown func() error, // shutdown osqueryd runner

cmd/launcher/launcher.go

@@ -13,7 +13,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/boltdb/bolt"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/kolide/kit/fs"
@@ -26,6 +25,7 @@ import (
 	"github.com/kolide/launcher/pkg/service"
 	"github.com/oklog/run"
 	"github.com/pkg/errors"
+	"go.etcd.io/bbolt"
 )
 
 // runLauncher is the entry point into running launcher. It creates a
@@ -80,8 +80,8 @@ func runLauncher(ctx context.Context, cancel func(), opts *launcher.Options) err
 	// this. Note that the timeout is documented as failing
 	// unimplemented on windows, though empirically it seems to
 	// work.
-	boltOptions := &bolt.Options{Timeout: time.Duration(30) * time.Second}
-	db, err := bolt.Open(filepath.Join(rootDirectory, "launcher.db"), 0600, boltOptions)
+	boltOptions := &bbolt.Options{Timeout: time.Duration(30) * time.Second}
+	db, err := bbolt.Open(filepath.Join(rootDirectory, "launcher.db"), 0600, boltOptions)
 	if err != nil {
 		return errors.Wrap(err, "open launcher db")
 	}

cmd/launcher/query_target_updater.go

@@ -3,15 +3,15 @@ package main
 import (
 	"context"
 
-	"github.com/boltdb/bolt"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/kolide/kit/actor"
 	"github.com/kolide/launcher/pkg/querytarget"
+	"go.etcd.io/bbolt"
 	"google.golang.org/grpc"
 )
 
-func createQueryTargetUpdater(logger log.Logger, db *bolt.DB, grpcConn *grpc.ClientConn) *actor.Actor {
+func createQueryTargetUpdater(logger log.Logger, db *bbolt.DB, grpcConn *grpc.ClientConn) *actor.Actor {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	updater := querytarget.NewQueryTargeter(logger, db, grpcConn)

go.mod

@@ -1,7 +1,7 @@
 module github.com/kolide/launcher
 
 require (
-	cloud.google.com/go v0.43.0 // indirect
+	cloud.google.com/go v0.43.0
 	github.com/Masterminds/semver v1.4.2
 	github.com/Microsoft/go-winio v0.4.11 // indirect
 	github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
@@ -11,7 +11,6 @@ require (
 	github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect
 	github.com/bitly/go-simplejson v0.5.0 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/boltdb/bolt v1.3.1
 	github.com/bugsnag/bugsnag-go v1.3.2 // indirect
 	github.com/bugsnag/panicwrap v1.2.0 // indirect
 	github.com/cenkalti/backoff v2.0.0+incompatible // indirect
@@ -60,7 +59,6 @@ require (
 	github.com/onsi/gomega v1.4.3 // indirect
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/pelletier/go-toml v1.6.0
 	github.com/peterbourgon/ff/v3 v3.0.0
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.9.2 // indirect
@@ -72,6 +70,7 @@ require (
 	github.com/stretchr/testify v1.5.1
 	github.com/theupdateframework/notary v0.6.1
 	github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9
+	go.etcd.io/bbolt v1.3.5
 	go.opencensus.io v0.22.1
 	golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5
 	golang.org/x/image v0.0.0-20190227222117-0694c2d4d067

go.sum

@@ -31,8 +31,6 @@ github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkN
 github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bugsnag/bugsnag-go v1.3.2 h1:8bcRylldQKQiAx9/KPu9+1iLZwgK1eN1Ib3SROSXfIY=
 github.com/bugsnag/bugsnag-go v1.3.2/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
 github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA=
@@ -256,6 +254,8 @@ github.com/theupdateframework/notary v0.6.1 h1:7wshjstgS9x9F5LuB1L5mBI2xNMObWqjz
 github.com/theupdateframework/notary v0.6.1/go.mod h1:MOfgIfmox8s7/7fduvB2xyPPMJCrjRLRizA8OFwpnKY=
 github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9 h1:vY5WqiEon0ZSTGM3ayVVi+twaHKHDFUVloaQ/wug9/c=
 github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9/go.mod h1:q+QjxYvZ+fpjMXqs+XEriussHjSYqeXVnAdSV1tkMYk=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -317,6 +317,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803150936-fd5f0c170ac3 h1:Q8sHd8ZmZZguDQWkUYcD4pIRG5dG+euDEmNyknTRbGs=
 golang.org/x/sys v0.0.0-20200803150936-fd5f0c170ac3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=

pkg/agent/dbdump.go

@@ -0,0 +1,60 @@
+package agent
+
+import (
+	"github.com/pkg/errors"
+	"go.etcd.io/bbolt"
+)
+
+// KeyN is the number of keys
+// LeafAlloc is pretty close the number of bytes uses
+
+type bucketStatsHolder struct {
+	Stats        bbolt.BucketStats
+	FillPercent  float64
+	NumberOfKeys int
+	Size         int
+}
+
+type dbStatsHolder struct {
+	Stats bbolt.TxStats
+	Size  int64
+}
+
+type Stats struct {
+	DB      dbStatsHolder
+	Buckets map[string]bucketStatsHolder
+}
+
+func GetStats(db *bbolt.DB) (*Stats, error) {
+	stats := &Stats{
+		Buckets: make(map[string]bucketStatsHolder),
+	}
+
+	if err := db.View(func(tx *bbolt.Tx) error {
+		stats.DB.Stats = tx.Stats()
+		stats.DB.Size = tx.Size()
+
+		if err := tx.ForEach(bucketStatsFunc(stats)); err != nil {
+			return errors.Wrap(err, "dumping bucket")
+		}
+		return nil
+	}); err != nil {
+		return nil, errors.Wrap(err, "creating view tx")
+	}
+
+	return stats, nil
+}
+
+func bucketStatsFunc(stats *Stats) func([]byte, *bbolt.Bucket) error {
+	return func(name []byte, b *bbolt.Bucket) error {
+		bstats := b.Stats()
+		stats.Buckets[string(name)] = bucketStatsHolder{
+			Stats:        bstats,
+			FillPercent:  b.FillPercent,
+			NumberOfKeys: bstats.KeyN,
+			Size:         bstats.LeafAlloc,
+		}
+
+		return nil
+	}
+}

pkg/control/control.go

@@ -7,23 +7,23 @@ import (
 	"net/http"
 	"net/url"
 
-	"github.com/boltdb/bolt"
 	"github.com/go-kit/kit/log"
 	"github.com/pkg/errors"
+	"go.etcd.io/bbolt"
 )
 
 type Client struct {
 	addr       string
 	baseURL    *url.URL
 	cancel     context.CancelFunc
 	client     *http.Client
-	db         *bolt.DB
+	db         *bbolt.DB
 	insecure   bool
 	disableTLS bool
 	logger     log.Logger
 }
 
-func NewControlClient(db *bolt.DB, addr string, opts ...Option) (*Client, error) {
+func NewControlClient(db *bbolt.DB, addr string, opts ...Option) (*Client, error) {
 	baseURL, err := url.Parse("https://" + addr)
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing URL")