From 07da75507bff7345d99fde8afcf7f56551d91022 Mon Sep 17 00:00:00 2001 From: seph Date: Sat, 20 Feb 2021 00:37:39 -0500 Subject: [PATCH 1/3] Upgrade to bbolt --- cmd/grpc.ext/grpc.go | 4 +- cmd/launcher/control.go | 4 +- cmd/launcher/control_windows.go | 4 +- cmd/launcher/extension.go | 4 +- cmd/launcher/launcher.go | 6 +-- cmd/launcher/query_target_updater.go | 4 +- go.mod | 5 +-- go.sum | 5 ++- pkg/control/control.go | 6 +-- pkg/osquery/extension.go | 42 +++++++++---------- pkg/osquery/extension_test.go | 10 ++--- pkg/osquery/table/kolide_target_membership.go | 8 ++-- pkg/osquery/table/launcher_config.go | 6 +-- pkg/osquery/table/launcher_identifier.go | 6 +-- pkg/osquery/table/table.go | 4 +- pkg/querytarget/query_target.go | 8 ++-- 16 files changed, 63 insertions(+), 63 deletions(-) diff --git a/cmd/grpc.ext/grpc.go b/cmd/grpc.ext/grpc.go index 103d520bc..eb637fbac 100644 --- a/cmd/grpc.ext/grpc.go +++ b/cmd/grpc.ext/grpc.go @@ -9,7 +9,7 @@ import ( "path/filepath" "time" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/env" "github.com/kolide/kit/logutil" @@ -84,7 +84,7 @@ func main() { LoggingInterval: loggingInterval, } - db, err := bolt.Open(filepath.Join(rootDirectory, "launcher.db"), 0600, nil) + db, err := bbolt.Open(filepath.Join(rootDirectory, "launcher.db"), 0600, nil) if err != nil { logutil.Fatal(logger, "err", errors.Wrap(err, "open local store"), "stack", fmt.Sprintf("%+v", err)) } diff --git a/cmd/launcher/control.go b/cmd/launcher/control.go index dd5c58063..ed030a40c 100644 --- a/cmd/launcher/control.go +++ b/cmd/launcher/control.go @@ -5,16 +5,16 @@ package main import ( "context" - "github.com/boltdb/bolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/actor" "github.com/kolide/launcher/pkg/control" "github.com/kolide/launcher/pkg/launcher" "github.com/pkg/errors" + "go.etcd.io/bbolt" ) -func createControl(ctx context.Context, db *bolt.DB, logger log.Logger, opts *launcher.Options) (*actor.Actor, error) { +func createControl(ctx context.Context, db *bbolt.DB, logger log.Logger, opts *launcher.Options) (*actor.Actor, error) { level.Debug(logger).Log("msg", "creating control client") controlOpts := []control.Option{ diff --git a/cmd/launcher/control_windows.go b/cmd/launcher/control_windows.go index fb571c5d0..8062aeba6 100644 --- a/cmd/launcher/control_windows.go +++ b/cmd/launcher/control_windows.go @@ -5,7 +5,7 @@ package main import ( "context" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/actor" @@ -14,7 +14,7 @@ import ( // createControl creates a no-op actor, as the control server isn't // yet supported on windows. -func createControl(ctx context.Context, db *bolt.DB, logger log.Logger, opts *launcher.Options) (*actor.Actor, error) { +func createControl(ctx context.Context, db *bbolt.DB, logger log.Logger, opts *launcher.Options) (*actor.Actor, error) { level.Info(logger).Log("msg", "Cannot create control channel for windows, ignoring") return nil, nil diff --git a/cmd/launcher/extension.go b/cmd/launcher/extension.go index 05abd261b..39c8a44d0 100644 --- a/cmd/launcher/extension.go +++ b/cmd/launcher/extension.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" - "github.com/boltdb/bolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/actor" @@ -22,11 +21,12 @@ import ( "github.com/kolide/osquery-go/plugin/distributed" osquerylogger "github.com/kolide/osquery-go/plugin/logger" "github.com/pkg/errors" + "go.etcd.io/bbolt" ) // TODO: the extension, runtime, and client are all kind of entangled // here. Untangle the underlying libraries and separate into units -func createExtensionRuntime(ctx context.Context, db *bolt.DB, launcherClient service.KolideService, opts *launcher.Options) ( +func createExtensionRuntime(ctx context.Context, db *bbolt.DB, launcherClient service.KolideService, opts *launcher.Options) ( run *actor.Actor, restart func() error, // restart osqueryd runner shutdown func() error, // shutdown osqueryd runner diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index df5bb233d..f3b4e65b9 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -13,7 +13,7 @@ import ( "strconv" "time" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/fs" @@ -80,8 +80,8 @@ func runLauncher(ctx context.Context, cancel func(), opts *launcher.Options) err // this. Note that the timeout is documented as failing // unimplemented on windows, though empirically it seems to // work. - boltOptions := &bolt.Options{Timeout: time.Duration(30) * time.Second} - db, err := bolt.Open(filepath.Join(rootDirectory, "launcher.db"), 0600, boltOptions) + boltOptions := &bbolt.Options{Timeout: time.Duration(30) * time.Second} + db, err := bbolt.Open(filepath.Join(rootDirectory, "launcher.db"), 0600, boltOptions) if err != nil { return errors.Wrap(err, "open launcher db") } diff --git a/cmd/launcher/query_target_updater.go b/cmd/launcher/query_target_updater.go index 525220caa..d62c0ed24 100644 --- a/cmd/launcher/query_target_updater.go +++ b/cmd/launcher/query_target_updater.go @@ -3,7 +3,7 @@ package main import ( "context" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/actor" @@ -11,7 +11,7 @@ import ( "google.golang.org/grpc" ) -func createQueryTargetUpdater(logger log.Logger, db *bolt.DB, grpcConn *grpc.ClientConn) *actor.Actor { +func createQueryTargetUpdater(logger log.Logger, db *bbolt.DB, grpcConn *grpc.ClientConn) *actor.Actor { ctx, cancel := context.WithCancel(context.Background()) updater := querytarget.NewQueryTargeter(logger, db, grpcConn) diff --git a/go.mod b/go.mod index 5a8ef22f5..814cd9b2e 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/kolide/launcher require ( - cloud.google.com/go v0.43.0 // indirect + cloud.google.com/go v0.43.0 github.com/Masterminds/semver v1.4.2 github.com/Microsoft/go-winio v0.4.11 // indirect github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect @@ -11,7 +11,6 @@ require ( github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect - github.com/boltdb/bolt v1.3.1 github.com/bugsnag/bugsnag-go v1.3.2 // indirect github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.0.0+incompatible // indirect @@ -60,7 +59,6 @@ require ( github.com/onsi/gomega v1.4.3 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/pelletier/go-toml v1.6.0 github.com/peterbourgon/ff/v3 v3.0.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.2 // indirect @@ -72,6 +70,7 @@ require ( github.com/stretchr/testify v1.5.1 github.com/theupdateframework/notary v0.6.1 github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9 + go.etcd.io/bbolt v1.3.5 go.opencensus.io v0.22.1 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 diff --git a/go.sum b/go.sum index 4f58107e0..389b33810 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,6 @@ github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkN github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bugsnag/bugsnag-go v1.3.2 h1:8bcRylldQKQiAx9/KPu9+1iLZwgK1eN1Ib3SROSXfIY= github.com/bugsnag/bugsnag-go v1.3.2/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA= @@ -256,6 +254,8 @@ github.com/theupdateframework/notary v0.6.1 h1:7wshjstgS9x9F5LuB1L5mBI2xNMObWqjz github.com/theupdateframework/notary v0.6.1/go.mod h1:MOfgIfmox8s7/7fduvB2xyPPMJCrjRLRizA8OFwpnKY= github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9 h1:vY5WqiEon0ZSTGM3ayVVi+twaHKHDFUVloaQ/wug9/c= github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9/go.mod h1:q+QjxYvZ+fpjMXqs+XEriussHjSYqeXVnAdSV1tkMYk= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -317,6 +317,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803150936-fd5f0c170ac3 h1:Q8sHd8ZmZZguDQWkUYcD4pIRG5dG+euDEmNyknTRbGs= golang.org/x/sys v0.0.0-20200803150936-fd5f0c170ac3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= diff --git a/pkg/control/control.go b/pkg/control/control.go index c6e5d08f5..92b47d65e 100644 --- a/pkg/control/control.go +++ b/pkg/control/control.go @@ -7,7 +7,7 @@ import ( "net/http" "net/url" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/pkg/errors" ) @@ -17,13 +17,13 @@ type Client struct { baseURL *url.URL cancel context.CancelFunc client *http.Client - db *bolt.DB + db *bbolt.DB insecure bool disableTLS bool logger log.Logger } -func NewControlClient(db *bolt.DB, addr string, opts ...Option) (*Client, error) { +func NewControlClient(db *bbolt.DB, addr string, opts ...Option) (*Client, error) { baseURL, err := url.Parse("https://" + addr) if err != nil { return nil, errors.Wrap(err, "parsing URL") diff --git a/pkg/osquery/extension.go b/pkg/osquery/extension.go index 844904d34..d825f25ee 100644 --- a/pkg/osquery/extension.go +++ b/pkg/osquery/extension.go @@ -11,7 +11,7 @@ import ( "sync" "time" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/google/uuid" @@ -31,7 +31,7 @@ import ( type Extension struct { NodeKey string Opts ExtensionOpts - db *bolt.DB + db *bbolt.DB serviceClient service.KolideService enrollMutex sync.Mutex done chan struct{} @@ -117,7 +117,7 @@ type ExtensionOpts struct { // NewExtension creates a new Extension from the provided service.KolideService // implementation. The background routines should be started by calling // Start(). -func NewExtension(client service.KolideService, db *bolt.DB, opts ExtensionOpts) (*Extension, error) { +func NewExtension(client service.KolideService, db *bbolt.DB, opts ExtensionOpts) (*Extension, error) { // bucketNames contains the names of buckets that should be created when the // extension opens the DB. It should be treated as a constant. var bucketNames = []string{configBucket, statusLogsBucket, resultLogsBucket, initialResultsBucket, ServerProvidedDataBucket} @@ -148,7 +148,7 @@ func NewExtension(client service.KolideService, db *bolt.DB, opts ExtensionOpts) } // Create Bolt buckets as necessary - err := db.Update(func(tx *bolt.Tx) error { + err := db.Update(func(tx *bbolt.Tx) error { for _, name := range bucketNames { _, err := tx.CreateBucketIfNotExists([]byte(name)) if err != nil { @@ -206,9 +206,9 @@ func (e *Extension) getHostIdentifier() (string, error) { // IdentifierFromDB returns the built-in launcher identifier from the config bucket. // The function is exported to allow for building the kolide_launcher_identifier table. -func IdentifierFromDB(db *bolt.DB) (string, error) { +func IdentifierFromDB(db *bbolt.DB) (string, error) { var identifier string - err := db.Update(func(tx *bolt.Tx) error { + err := db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(configBucket)) uuidBytes := b.Get([]byte(uuidKey)) gotID, err := uuid.ParseBytes(uuidBytes) @@ -239,13 +239,13 @@ func IdentifierFromDB(db *bolt.DB) (string, error) { } // NodeKeyFromDB returns the device node key from a local bolt DB -func NodeKeyFromDB(db *bolt.DB) (string, error) { +func NodeKeyFromDB(db *bbolt.DB) (string, error) { if db == nil { return "", errors.New("received a nil db") } var key []byte - err := db.View(func(tx *bolt.Tx) error { + err := db.View(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(configBucket)) key = b.Get([]byte(nodeKeyKey)) return nil @@ -261,13 +261,13 @@ func NodeKeyFromDB(db *bolt.DB) (string, error) { } // ConfigFromDB returns the device config from a local bolt DB -func ConfigFromDB(db *bolt.DB) (string, error) { +func ConfigFromDB(db *bbolt.DB) (string, error) { if db == nil { return "", errors.New("received a nil db") } var key []byte - err := db.View(func(tx *bolt.Tx) error { + err := db.View(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(configBucket)) key = b.Get([]byte(configKey)) return nil @@ -348,7 +348,7 @@ func (e *Extension) Enroll(ctx context.Context) (string, bool, error) { } // Save newly acquired node key if successful - err = e.db.Update(func(tx *bolt.Tx) error { + err = e.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(configBucket)) return b.Put([]byte(nodeKeyKey), []byte(keyString)) }) @@ -367,7 +367,7 @@ func (e *Extension) RequireReenroll(ctx context.Context) { defer e.enrollMutex.Unlock() // Clear the node key such that reenrollment is required. e.NodeKey = "" - e.db.Update(func(tx *bolt.Tx) error { + e.db.Update(func(tx *bbolt.Tx) error { tx.Bucket([]byte(configBucket)).Delete([]byte(nodeKeyKey)) return nil }) @@ -386,7 +386,7 @@ func (e *Extension) GenerateConfigs(ctx context.Context) (map[string]string, err ) // Try to use cached config var confBytes []byte - e.db.View(func(tx *bolt.Tx) error { + e.db.View(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(configBucket)) confBytes = b.Get([]byte(configKey)) return nil @@ -398,7 +398,7 @@ func (e *Extension) GenerateConfigs(ctx context.Context) (map[string]string, err config = string(confBytes) } else { // Store good config - e.db.Update(func(tx *bolt.Tx) error { + e.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(configBucket)) return b.Put([]byte(configKey), []byte(config)) }) @@ -537,7 +537,7 @@ func (e *Extension) writeBufferedLogsForType(typ logger.LogType) error { // Collect up logs to be sent var logs []string var logIDs [][]byte - err = e.db.View(func(tx *bolt.Tx) error { + err = e.db.View(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketName)) c := b.Cursor() @@ -585,7 +585,7 @@ func (e *Extension) writeBufferedLogsForType(typ logger.LogType) error { } // Delete logs that were successfully sent - err = e.db.Update(func(tx *bolt.Tx) error { + err = e.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketName)) for _, k := range logIDs { b.Delete(k) @@ -636,7 +636,7 @@ func (e *Extension) purgeBufferedLogsForType(typ logger.LogType) error { if err != nil { return err } - err = e.db.Update(func(tx *bolt.Tx) error { + err = e.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketName)) logCount := b.Stats().KeyN @@ -688,7 +688,7 @@ func (e *Extension) LogString(ctx context.Context, typ logger.LogType, logText s } // Buffer the log for sending later in a batch - err = e.db.Update(func(tx *bolt.Tx) error { + err = e.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketName)) // Log keys are generated with the auto-incrementing sequence @@ -861,7 +861,7 @@ type initialRunner struct { enabled bool identifier string client Querier - db *bolt.DB + db *bbolt.DB } func (i *initialRunner) Execute(configBlob string, writeFn func(ctx context.Context, l logger.LogType, results []string, reeenroll bool) error) error { @@ -952,7 +952,7 @@ func (i *initialRunner) Execute(configBlob string, writeFn func(ctx context.Cont func (i *initialRunner) queriesToRun(allFromConfig []string) (map[string]struct{}, error) { known := make(map[string]struct{}) - err := i.db.View(func(tx *bolt.Tx) error { + err := i.db.View(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(initialResultsBucket)) for _, q := range allFromConfig { knownQuery := b.Get([]byte(q)) @@ -968,7 +968,7 @@ func (i *initialRunner) queriesToRun(allFromConfig []string) (map[string]struct{ } func (i *initialRunner) cacheRanQueries(known map[string]struct{}) error { - err := i.db.Update(func(tx *bolt.Tx) error { + err := i.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(initialResultsBucket)) for q := range known { if err := b.Put([]byte(q), []byte(q)); err != nil { diff --git a/pkg/osquery/extension_test.go b/pkg/osquery/extension_test.go index 1724a9546..f1150512a 100644 --- a/pkg/osquery/extension_test.go +++ b/pkg/osquery/extension_test.go @@ -12,7 +12,7 @@ import ( "testing/quick" "time" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/kolide/kit/testutil" "github.com/kolide/launcher/pkg/service" "github.com/kolide/launcher/pkg/service/mock" @@ -23,13 +23,13 @@ import ( "github.com/stretchr/testify/require" ) -func makeTempDB(t *testing.T) (db *bolt.DB, cleanup func()) { +func makeTempDB(t *testing.T) (db *bbolt.DB, cleanup func()) { file, err := ioutil.TempFile("", "kolide_launcher_test") if err != nil { t.Fatalf("creating temp file: %s", err.Error()) } - db, err = bolt.Open(file.Name(), 0600, nil) + db, err = bbolt.Open(file.Name(), 0600, nil) if err != nil { t.Fatalf("opening bolt DB: %s", err.Error()) } @@ -57,7 +57,7 @@ func TestNewExtensionDatabaseError(t *testing.T) { db.Close() // Open read-only DB - db, err = bolt.Open(path, 0600, &bolt.Options{ReadOnly: true}) + db, err = bbolt.Open(path, 0600, &bbolt.Options{ReadOnly: true}) if err != nil { t.Fatalf("opening bolt DB: %s", err.Error()) } @@ -105,7 +105,7 @@ func TestGetHostIdentifierCorruptedData(t *testing.T) { require.Nil(t, err) // Put garbage UUID in DB - err = db.Update(func(tx *bolt.Tx) error { + err = db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(configBucket)) return b.Put([]byte(uuidKey), []byte("garbage_uuid")) }) diff --git a/pkg/osquery/table/kolide_target_membership.go b/pkg/osquery/table/kolide_target_membership.go index 6b25be905..5e944a90b 100644 --- a/pkg/osquery/table/kolide_target_membership.go +++ b/pkg/osquery/table/kolide_target_membership.go @@ -3,7 +3,7 @@ package table import ( "context" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/gogo/protobuf/proto" "github.com/kolide/launcher/pkg/osquery" qt "github.com/kolide/launcher/pkg/pb/querytarget" @@ -14,7 +14,7 @@ import ( const TargetMembershipKey = "target_membership" const targetMembershipTableName = "kolide_target_membership" -func TargetMembershipTable(db *bolt.DB) *table.Plugin { +func TargetMembershipTable(db *bbolt.DB) *table.Plugin { columns := []table.ColumnDefinition{ table.TextColumn("id"), } @@ -22,11 +22,11 @@ func TargetMembershipTable(db *bolt.DB) *table.Plugin { return table.NewPlugin(targetMembershipTableName, columns, generateTargetMembershipTable(db)) } -func generateTargetMembershipTable(db *bolt.DB) table.GenerateFunc { +func generateTargetMembershipTable(db *bbolt.DB) table.GenerateFunc { return func(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { var targetRespBytes []byte - if err := db.View(func(tx *bolt.Tx) error { + if err := db.View(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(osquery.ServerProvidedDataBucket)) targetRespBytes = b.Get([]byte(TargetMembershipKey)) diff --git a/pkg/osquery/table/launcher_config.go b/pkg/osquery/table/launcher_config.go index 1d6e96a64..8250ed658 100644 --- a/pkg/osquery/table/launcher_config.go +++ b/pkg/osquery/table/launcher_config.go @@ -3,19 +3,19 @@ package table import ( "context" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/kolide/launcher/pkg/osquery" "github.com/kolide/osquery-go/plugin/table" ) -func LauncherConfigTable(db *bolt.DB) *table.Plugin { +func LauncherConfigTable(db *bbolt.DB) *table.Plugin { columns := []table.ColumnDefinition{ table.TextColumn("config"), } return table.NewPlugin("kolide_launcher_config", columns, generateLauncherConfig(db)) } -func generateLauncherConfig(db *bolt.DB) table.GenerateFunc { +func generateLauncherConfig(db *bbolt.DB) table.GenerateFunc { return func(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { config, err := osquery.ConfigFromDB(db) if err != nil { diff --git a/pkg/osquery/table/launcher_identifier.go b/pkg/osquery/table/launcher_identifier.go index e80224d2d..44499dba0 100644 --- a/pkg/osquery/table/launcher_identifier.go +++ b/pkg/osquery/table/launcher_identifier.go @@ -3,19 +3,19 @@ package table import ( "context" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/kolide/launcher/pkg/osquery" "github.com/kolide/osquery-go/plugin/table" ) -func LauncherIdentifierTable(db *bolt.DB) *table.Plugin { +func LauncherIdentifierTable(db *bbolt.DB) *table.Plugin { columns := []table.ColumnDefinition{ table.TextColumn("identifier"), } return table.NewPlugin("kolide_launcher_identifier", columns, generateLauncherIdentifier(db)) } -func generateLauncherIdentifier(db *bolt.DB) table.GenerateFunc { +func generateLauncherIdentifier(db *bbolt.DB) table.GenerateFunc { return func(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { identifier, err := osquery.IdentifierFromDB(db) if err != nil { diff --git a/pkg/osquery/table/table.go b/pkg/osquery/table/table.go index 9c3b3cb6f..89e5a0b29 100644 --- a/pkg/osquery/table/table.go +++ b/pkg/osquery/table/table.go @@ -5,14 +5,14 @@ import ( "github.com/kolide/launcher/pkg/osquery/tables/dataflattentable" "github.com/kolide/launcher/pkg/osquery/tables/zfs" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/go-kit/kit/log" osquery "github.com/kolide/osquery-go" "github.com/kolide/osquery-go/plugin/table" ) // LauncherTables returns launcher-specific tables -func LauncherTables(db *bolt.DB, opts *launcher.Options) []osquery.OsqueryPlugin { +func LauncherTables(db *bbolt.DB, opts *launcher.Options) []osquery.OsqueryPlugin { return []osquery.OsqueryPlugin{ LauncherConfigTable(db), LauncherIdentifierTable(db), diff --git a/pkg/querytarget/query_target.go b/pkg/querytarget/query_target.go index 588b0485e..7527c53cc 100644 --- a/pkg/querytarget/query_target.go +++ b/pkg/querytarget/query_target.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/boltdb/bolt" + "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/gogo/protobuf/proto" @@ -19,11 +19,11 @@ import ( type QueryTargetUpdater struct { logger log.Logger - db *bolt.DB + db *bbolt.DB targetClient qt.QueryTargetClient } -func NewQueryTargeter(logger log.Logger, db *bolt.DB, grpcConn *grpc.ClientConn) QueryTargetUpdater { +func NewQueryTargeter(logger log.Logger, db *bbolt.DB, grpcConn *grpc.ClientConn) QueryTargetUpdater { return QueryTargetUpdater{ logger: logger, db: db, @@ -47,7 +47,7 @@ func (qtu *QueryTargetUpdater) updateTargetMemberships(ctx context.Context) erro return errors.Wrap(err, "marshaling targets to bytes") } - if err := qtu.db.Update(func(tx *bolt.Tx) error { + if err := qtu.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(osquery.ServerProvidedDataBucket)) err := b.Put([]byte(table.TargetMembershipKey), targetRespBytes) From cb082b1fcbe8b382ab5bd6ec4bf548ca23f090fc Mon Sep 17 00:00:00 2001 From: seph Date: Sat, 20 Feb 2021 00:39:50 -0500 Subject: [PATCH 2/3] gofmt --- cmd/grpc.ext/grpc.go | 2 +- cmd/launcher/control_windows.go | 2 +- cmd/launcher/launcher.go | 2 +- cmd/launcher/query_target_updater.go | 2 +- pkg/control/control.go | 2 +- pkg/osquery/extension.go | 2 +- pkg/osquery/extension_test.go | 2 +- pkg/osquery/table/kolide_target_membership.go | 2 +- pkg/osquery/table/launcher_config.go | 2 +- pkg/osquery/table/launcher_identifier.go | 2 +- pkg/osquery/table/table.go | 2 +- pkg/querytarget/query_target.go | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/grpc.ext/grpc.go b/cmd/grpc.ext/grpc.go index eb637fbac..63f25f462 100644 --- a/cmd/grpc.ext/grpc.go +++ b/cmd/grpc.ext/grpc.go @@ -9,7 +9,6 @@ import ( "path/filepath" "time" - "go.etcd.io/bbolt" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/env" "github.com/kolide/kit/logutil" @@ -21,6 +20,7 @@ import ( "github.com/kolide/osquery-go/plugin/distributed" osquery_logger "github.com/kolide/osquery-go/plugin/logger" "github.com/pkg/errors" + "go.etcd.io/bbolt" ) func main() { diff --git a/cmd/launcher/control_windows.go b/cmd/launcher/control_windows.go index 8062aeba6..ad6c0b876 100644 --- a/cmd/launcher/control_windows.go +++ b/cmd/launcher/control_windows.go @@ -5,11 +5,11 @@ package main import ( "context" - "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/actor" "github.com/kolide/launcher/pkg/launcher" + "go.etcd.io/bbolt" ) // createControl creates a no-op actor, as the control server isn't diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index f3b4e65b9..c52549e81 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -13,7 +13,6 @@ import ( "strconv" "time" - "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/fs" @@ -26,6 +25,7 @@ import ( "github.com/kolide/launcher/pkg/service" "github.com/oklog/run" "github.com/pkg/errors" + "go.etcd.io/bbolt" ) // runLauncher is the entry point into running launcher. It creates a diff --git a/cmd/launcher/query_target_updater.go b/cmd/launcher/query_target_updater.go index d62c0ed24..b7b08ba04 100644 --- a/cmd/launcher/query_target_updater.go +++ b/cmd/launcher/query_target_updater.go @@ -3,11 +3,11 @@ package main import ( "context" - "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/kolide/kit/actor" "github.com/kolide/launcher/pkg/querytarget" + "go.etcd.io/bbolt" "google.golang.org/grpc" ) diff --git a/pkg/control/control.go b/pkg/control/control.go index 92b47d65e..f5da19b16 100644 --- a/pkg/control/control.go +++ b/pkg/control/control.go @@ -7,9 +7,9 @@ import ( "net/http" "net/url" - "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/pkg/errors" + "go.etcd.io/bbolt" ) type Client struct { diff --git a/pkg/osquery/extension.go b/pkg/osquery/extension.go index d825f25ee..c64ff63ea 100644 --- a/pkg/osquery/extension.go +++ b/pkg/osquery/extension.go @@ -11,7 +11,6 @@ import ( "sync" "time" - "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/google/uuid" @@ -22,6 +21,7 @@ import ( "github.com/kolide/osquery-go/plugin/logger" "github.com/mixer/clock" "github.com/pkg/errors" + "go.etcd.io/bbolt" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/pkg/osquery/extension_test.go b/pkg/osquery/extension_test.go index f1150512a..c73bac02e 100644 --- a/pkg/osquery/extension_test.go +++ b/pkg/osquery/extension_test.go @@ -12,7 +12,6 @@ import ( "testing/quick" "time" - "go.etcd.io/bbolt" "github.com/kolide/kit/testutil" "github.com/kolide/launcher/pkg/service" "github.com/kolide/launcher/pkg/service/mock" @@ -21,6 +20,7 @@ import ( "github.com/mixer/clock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" ) func makeTempDB(t *testing.T) (db *bbolt.DB, cleanup func()) { diff --git a/pkg/osquery/table/kolide_target_membership.go b/pkg/osquery/table/kolide_target_membership.go index 5e944a90b..63865f175 100644 --- a/pkg/osquery/table/kolide_target_membership.go +++ b/pkg/osquery/table/kolide_target_membership.go @@ -3,12 +3,12 @@ package table import ( "context" - "go.etcd.io/bbolt" "github.com/gogo/protobuf/proto" "github.com/kolide/launcher/pkg/osquery" qt "github.com/kolide/launcher/pkg/pb/querytarget" "github.com/kolide/osquery-go/plugin/table" "github.com/pkg/errors" + "go.etcd.io/bbolt" ) const TargetMembershipKey = "target_membership" diff --git a/pkg/osquery/table/launcher_config.go b/pkg/osquery/table/launcher_config.go index 8250ed658..05b25badf 100644 --- a/pkg/osquery/table/launcher_config.go +++ b/pkg/osquery/table/launcher_config.go @@ -3,9 +3,9 @@ package table import ( "context" - "go.etcd.io/bbolt" "github.com/kolide/launcher/pkg/osquery" "github.com/kolide/osquery-go/plugin/table" + "go.etcd.io/bbolt" ) func LauncherConfigTable(db *bbolt.DB) *table.Plugin { diff --git a/pkg/osquery/table/launcher_identifier.go b/pkg/osquery/table/launcher_identifier.go index 44499dba0..ae8ed0755 100644 --- a/pkg/osquery/table/launcher_identifier.go +++ b/pkg/osquery/table/launcher_identifier.go @@ -3,9 +3,9 @@ package table import ( "context" - "go.etcd.io/bbolt" "github.com/kolide/launcher/pkg/osquery" "github.com/kolide/osquery-go/plugin/table" + "go.etcd.io/bbolt" ) func LauncherIdentifierTable(db *bbolt.DB) *table.Plugin { diff --git a/pkg/osquery/table/table.go b/pkg/osquery/table/table.go index 89e5a0b29..aa1f363ad 100644 --- a/pkg/osquery/table/table.go +++ b/pkg/osquery/table/table.go @@ -5,10 +5,10 @@ import ( "github.com/kolide/launcher/pkg/osquery/tables/dataflattentable" "github.com/kolide/launcher/pkg/osquery/tables/zfs" - "go.etcd.io/bbolt" "github.com/go-kit/kit/log" osquery "github.com/kolide/osquery-go" "github.com/kolide/osquery-go/plugin/table" + "go.etcd.io/bbolt" ) // LauncherTables returns launcher-specific tables diff --git a/pkg/querytarget/query_target.go b/pkg/querytarget/query_target.go index 7527c53cc..25de05324 100644 --- a/pkg/querytarget/query_target.go +++ b/pkg/querytarget/query_target.go @@ -4,7 +4,6 @@ import ( "context" "time" - "go.etcd.io/bbolt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/gogo/protobuf/proto" @@ -12,6 +11,7 @@ import ( "github.com/kolide/launcher/pkg/osquery/table" qt "github.com/kolide/launcher/pkg/pb/querytarget" "github.com/pkg/errors" + "go.etcd.io/bbolt" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" From 55266dd1672e3558fd883fafebf38df9cadd18d2 Mon Sep 17 00:00:00 2001 From: seph Date: Mon, 22 Feb 2021 00:57:46 -0500 Subject: [PATCH 3/3] db stats table --- pkg/agent/dbdump.go | 60 +++++++++++++++++++++++++++ pkg/osquery/table/launcher_db_info.go | 56 +++++++++++++++++++++++++ pkg/osquery/table/table.go | 1 + 3 files changed, 117 insertions(+) create mode 100644 pkg/agent/dbdump.go create mode 100644 pkg/osquery/table/launcher_db_info.go diff --git a/pkg/agent/dbdump.go b/pkg/agent/dbdump.go new file mode 100644 index 000000000..0ce8df888 --- /dev/null +++ b/pkg/agent/dbdump.go @@ -0,0 +1,60 @@ +package agent + +import ( + "github.com/pkg/errors" + "go.etcd.io/bbolt" +) + +// KeyN is the number of keys +// LeafAlloc is pretty close the number of bytes uses + +type bucketStatsHolder struct { + Stats bbolt.BucketStats + FillPercent float64 + NumberOfKeys int + Size int +} + +type dbStatsHolder struct { + Stats bbolt.TxStats + Size int64 +} + +type Stats struct { + DB dbStatsHolder + Buckets map[string]bucketStatsHolder +} + +func GetStats(db *bbolt.DB) (*Stats, error) { + stats := &Stats{ + Buckets: make(map[string]bucketStatsHolder), + } + + if err := db.View(func(tx *bbolt.Tx) error { + stats.DB.Stats = tx.Stats() + stats.DB.Size = tx.Size() + + if err := tx.ForEach(bucketStatsFunc(stats)); err != nil { + return errors.Wrap(err, "dumping bucket") + } + return nil + }); err != nil { + return nil, errors.Wrap(err, "creating view tx") + } + + return stats, nil +} + +func bucketStatsFunc(stats *Stats) func([]byte, *bbolt.Bucket) error { + return func(name []byte, b *bbolt.Bucket) error { + bstats := b.Stats() + stats.Buckets[string(name)] = bucketStatsHolder{ + Stats: bstats, + FillPercent: b.FillPercent, + NumberOfKeys: bstats.KeyN, + Size: bstats.LeafAlloc, + } + + return nil + } +} diff --git a/pkg/osquery/table/launcher_db_info.go b/pkg/osquery/table/launcher_db_info.go new file mode 100644 index 000000000..05f40769e --- /dev/null +++ b/pkg/osquery/table/launcher_db_info.go @@ -0,0 +1,56 @@ +package table + +import ( + "context" + "encoding/json" + "strings" + + "github.com/kolide/launcher/pkg/agent" + "github.com/kolide/launcher/pkg/dataflatten" + "github.com/kolide/launcher/pkg/osquery/tables/dataflattentable" + "github.com/kolide/launcher/pkg/osquery/tables/tablehelpers" + "github.com/kolide/osquery-go/plugin/table" + "github.com/pkg/errors" + "go.etcd.io/bbolt" +) + +func LauncherDbInfo(db *bbolt.DB) *table.Plugin { + columns := dataflattentable.Columns() + return table.NewPlugin("kolide_launcher_db_info", columns, generateLauncherDbInfo(db)) +} + +func generateLauncherDbInfo(db *bbolt.DB) table.GenerateFunc { + return func(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + stats, err := agent.GetStats(db) + if err != nil { + return nil, err + } + + statsJson, err := json.Marshal(stats) + if err != nil { + return nil, err + } + + var results []map[string]string + + for _, dataQuery := range tablehelpers.GetConstraints(queryContext, "query", tablehelpers.WithDefaults("*")) { + flattenOpts := []dataflatten.FlattenOpts{ + dataflatten.WithQuery(strings.Split(dataQuery, "/")), + } + + rows, err := dataflatten.Json(statsJson, flattenOpts...) + + if err != nil { + // In other places, we'd log the error + // and continue. We don't have logs + // here, so we'll just return it. + return results, errors.Wrapf(err, "flattening with query %s", dataQuery) + + } + + results = append(results, dataflattentable.ToMap(rows, dataQuery, nil)...) + } + + return results, nil + } +} diff --git a/pkg/osquery/table/table.go b/pkg/osquery/table/table.go index aa1f363ad..dd664a374 100644 --- a/pkg/osquery/table/table.go +++ b/pkg/osquery/table/table.go @@ -15,6 +15,7 @@ import ( func LauncherTables(db *bbolt.DB, opts *launcher.Options) []osquery.OsqueryPlugin { return []osquery.OsqueryPlugin{ LauncherConfigTable(db), + LauncherDbInfo(db), LauncherIdentifierTable(db), TargetMembershipTable(db), LauncherAutoupdateConfigTable(opts),