From 932ac454c542221bdb061271b4505b688b078da0 Mon Sep 17 00:00:00 2001 From: Shashank Sinha Date: Mon, 7 Nov 2022 15:38:33 +0530 Subject: [PATCH 1/4] PMM-10072 Filter out profiler collection --- exporter/common.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/exporter/common.go b/exporter/common.go index 353722e1f..9b8261148 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -29,7 +29,10 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -var systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals +var ( + systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals + systemCollections = []string{"system.profile"} +) func listCollections(ctx context.Context, client *mongo.Client, database string, filterInNamespaces []string) ([]string, error) { filter := bson.D{} // Default=empty -> list all collections @@ -63,7 +66,15 @@ func listCollections(ctx context.Context, client *mongo.Client, database string, return nil, errors.Wrap(err, "cannot get the list of collections for discovery") } - return collections, nil + filteredCollections := []string{} + for _, collection := range collections { + if collection == systemCollections[0] { + continue + } + filteredCollections = append(filteredCollections, collection) + } + + return filteredCollections, nil } // databases returns the list of databases matching the filters. From 1002fa32ec27135f500f4df00f3c413bea382d58 Mon Sep 17 00:00:00 2001 From: Shashank Sinha Date: Wed, 7 Dec 2022 14:52:28 +0530 Subject: [PATCH 2/4] Update system collection filter logic --- exporter/common.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/exporter/common.go b/exporter/common.go index 9b8261148..05bf82dc4 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -29,10 +29,7 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -var ( - systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals - systemCollections = []string{"system.profile"} -) +var systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals func listCollections(ctx context.Context, client *mongo.Client, database string, filterInNamespaces []string) ([]string, error) { filter := bson.D{} // Default=empty -> list all collections @@ -66,14 +63,7 @@ func listCollections(ctx context.Context, client *mongo.Client, database string, return nil, errors.Wrap(err, "cannot get the list of collections for discovery") } - filteredCollections := []string{} - for _, collection := range collections { - if collection == systemCollections[0] { - continue - } - filteredCollections = append(filteredCollections, collection) - } - + filteredCollections := filterSystemCollections(collections) return filteredCollections, nil } @@ -212,6 +202,21 @@ func listAllCollections(ctx context.Context, client *mongo.Client, filterInNames return namespaces, nil } +func filterSystemCollections(cols []string) []string { + systemCollections := map[string]bool{ + "system.profile": true, + } + + filtered := make([]string, 0, len(cols)) + for _, col := range cols { + if systemCollections[col] { + continue + } + filtered = append(filtered, col) + } + return filtered +} + func nonSystemCollectionsCount(ctx context.Context, client *mongo.Client, includeNamespaces []string, filterInCollections []string) (int, error) { databases, err := databases(ctx, client, includeNamespaces, systemDBs) if err != nil { From d5634f91df0c6b80994da9ff3b71b4b8e81aece8 Mon Sep 17 00:00:00 2001 From: Shashank Sinha Date: Wed, 7 Dec 2022 14:54:41 +0530 Subject: [PATCH 3/4] Enable test for system collection filtering --- docker-compose.yml | 6 +++--- exporter/dbstats_collector_test.go | 31 +++++++++--------------------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 11193b8ee..9f8eda66d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: image: ${TEST_MONGODB_IMAGE:-mongo:4.2} ports: - "${TEST_MONGODB_S1_PRIMARY_PORT:-17001}:27017" - command: mongod --replSet rs1 --shardsvr --port 27017 --oplogSize 16 + command: mongod --replSet rs1 --shardsvr --port 27017 --oplogSize 16 --profile 2 links: - mongo-1-2:mongo-1-2 - mongo-1-3:mongo-1-3 @@ -15,14 +15,14 @@ services: image: ${TEST_MONGODB_IMAGE:-mongo:4.2} ports: - "${TEST_MONGODB_S1_SECONDARY1_PORT:-17002}:27017" - command: mongod --replSet rs1 --shardsvr --port 27017 --oplogSize 16 + command: mongod --replSet rs1 --shardsvr --port 27017 --oplogSize 16 --profile 2 mongo-1-3: container_name: "mongo-1-3" image: ${TEST_MONGODB_IMAGE:-mongo:4.2} ports: - "${TEST_MONGODB_S1_SECONDARY2_PORT:-17003}:27017" - command: mongod --replSet rs1 --shardsvr --port 27017 --oplogSize 16 + command: mongod --replSet rs1 --shardsvr --port 27017 --oplogSize 16 --profile 2 mongo-1-arbiter: container_name: "mongo-1-arbiter" diff --git a/exporter/dbstats_collector_test.go b/exporter/dbstats_collector_test.go index d43db7c45..90028c8b8 100644 --- a/exporter/dbstats_collector_test.go +++ b/exporter/dbstats_collector_test.go @@ -18,7 +18,6 @@ package exporter import ( "context" - "fmt" "strings" "testing" "time" @@ -26,7 +25,6 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "go.mongodb.org/mongo-driver/bson" "github.com/percona/mongodb_exporter/internal/tu" ) @@ -40,36 +38,25 @@ func TestDBStatsCollector(t *testing.T) { defer cancel() client := tu.DefaultTestClient(ctx, t) - - database := client.Database(dbName) - database.Drop(ctx) //nolint - - defer func() { - err := database.Drop(ctx) - assert.NoError(t, err) - }() - - for i := 0; i < 3; i++ { - coll := fmt.Sprintf("testcol_%02d", i) - for j := 0; j < 10; j++ { - _, err := database.Collection(coll).InsertOne(ctx, bson.M{"f1": j, "f2": "2"}) - assert.NoError(t, err) - } - } + setupDB(ctx, t, client) + defer cleanupDB(ctx, client) ti := labelsGetterMock{} - c := newDBStatsCollector(ctx, client, logrus.New(), false, ti, []string{dbName}) + c := newDBStatsCollector(ctx, client, logrus.New(), false, ti, testDBs) expected := strings.NewReader(` # HELP mongodb_dbstats_collections dbstats. # TYPE mongodb_dbstats_collections untyped - mongodb_dbstats_collections{database="testdb"} 3 + mongodb_dbstats_collections{database="testdb01"} 5 + mongodb_dbstats_collections{database="testdb02"} 5 # HELP mongodb_dbstats_indexes dbstats. # TYPE mongodb_dbstats_indexes untyped - mongodb_dbstats_indexes{database="testdb"} 3 + mongodb_dbstats_indexes{database="testdb01"} 4 + mongodb_dbstats_indexes{database="testdb02"} 4 # HELP mongodb_dbstats_objects dbstats. # TYPE mongodb_dbstats_objects untyped - mongodb_dbstats_objects{database="testdb"} 30` + "\n") + mongodb_dbstats_objects{database="testdb01"} 80 + mongodb_dbstats_objects{database="testdb02"} 80` + "\n") // Only look at metrics created by our activity filters := []string{ From 742f1b9dd8ba54645c7b6da2ba6d587e4b33be9a Mon Sep 17 00:00:00 2001 From: Shashank Sinha Date: Wed, 7 Dec 2022 15:03:34 +0530 Subject: [PATCH 4/4] Fix linter issues --- exporter/common.go | 2 ++ exporter/dbstats_collector_test.go | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/exporter/common.go b/exporter/common.go index 05bf82dc4..e6c943c0e 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -64,6 +64,7 @@ func listCollections(ctx context.Context, client *mongo.Client, database string, } filteredCollections := filterSystemCollections(collections) + return filteredCollections, nil } @@ -214,6 +215,7 @@ func filterSystemCollections(cols []string) []string { } filtered = append(filtered, col) } + return filtered } diff --git a/exporter/dbstats_collector_test.go b/exporter/dbstats_collector_test.go index 90028c8b8..10e431f59 100644 --- a/exporter/dbstats_collector_test.go +++ b/exporter/dbstats_collector_test.go @@ -29,10 +29,6 @@ import ( "github.com/percona/mongodb_exporter/internal/tu" ) -const ( - dbName = "testdb" -) - func TestDBStatsCollector(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel()