PMM-13477 Support MongoDB 8.0

.github/workflows/go.yml

@@ -22,8 +22,15 @@ jobs:
         image:
           - mongo:4.4
           - mongo:5.0
+          - mongo:6.0
+          - mongo:7.0
+          - mongo:8.0
+          - mongo:latest
           - percona/percona-server-mongodb:4.4
           - percona/percona-server-mongodb:5.0
+          - percona/percona-server-mongodb:6.0
+          - percona/percona-server-mongodb:7.0
+          - percona/percona-server-mongodb:latest
 
     runs-on: ubuntu-latest
 
@@ -42,3 +49,17 @@ jobs:
           sleep 10
           make test-race
           make test-cluster-clean
+
+      - name: Run debug commands on failure
+        if: ${{ failure() }}
+        run: |
+          echo "--- Environment variables ---"
+          env | sort
+          echo "--- GO Environment ---"
+          go env | sort
+          echo "--- Git status ---"
+          git status
+          echo "--- Docker logs ---"
+          docker compose logs
+          echo "--- Docker ps ---"
+          docker compose ps -a

exporter/diagnostic_data_collector.go

@@ -35,12 +35,14 @@ type diagnosticDataCollector struct {
 	ctx  context.Context
 	base *baseCollector
 
+	buildInfo buildInfo
+
 	compatibleMode bool
 	topologyInfo   labelsGetter
 }
 
 // newDiagnosticDataCollector creates a collector for diagnostic information.
-func newDiagnosticDataCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, compatible bool, topology labelsGetter) *diagnosticDataCollector {
+func newDiagnosticDataCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, compatible bool, topology labelsGetter, buildInfo buildInfo) *diagnosticDataCollector {
 	nodeType, err := getNodeType(ctx, client)
 	if err != nil {
 		logger.WithFields(logrus.Fields{
@@ -57,6 +59,8 @@ func newDiagnosticDataCollector(ctx context.Context, client *mongo.Client, logge
 		ctx:  ctx,
 		base: newBaseCollector(client, logger.WithFields(logrus.Fields{"collector": "diagnostic_data"})),
 
+		buildInfo: buildInfo,
+
 		compatibleMode: compatible,
 		topologyInfo:   topology,
 	}
@@ -108,6 +112,9 @@ func (d *diagnosticDataCollector) collect(ch chan<- prometheus.Metric) {
 			err = errors.Wrapf(errUnexpectedDataType, "%T for data field", m["data"])
 			logger.Errorf("cannot decode getDiagnosticData: %s", err)
 		}
+		if c, ok := m["common"].(bson.M); ok {
+			m = c
+		}
 
 		logger.Debug("getDiagnosticData result")
 		debugResult(logger, m)
@@ -132,12 +139,7 @@ func (d *diagnosticDataCollector) collect(ch chan<- prometheus.Metric) {
 	}
 
 	if d.compatibleMode {
-		buildInfo, err := retrieveMongoDBBuildInfo(d.ctx, client, logger)
-		if err != nil {
-			logger.Errorf("cannot retrieve MongoDB buildInfo: %s", err)
-		}
-
-		metrics = append(metrics, serverVersion(buildInfo))
+		metrics = append(metrics, serverVersion(d.buildInfo))
 
 		if nodeType == typeArbiter {
 			if hm := arbiterMetrics(d.ctx, client, logger); hm != nil {

exporter/diagnostic_data_collector_test.go

@@ -46,31 +46,36 @@ func TestDiagnosticDataCollector(t *testing.T) {
 	logger := logrus.New()
 	ti := labelsGetterMock{}
 
-	c := newDiagnosticDataCollector(ctx, client, logger, false, ti)
+	dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, logger.WithField("component", "test"))
+	require.NoError(t, err)
+
+	c := newDiagnosticDataCollector(ctx, client, logger, false, ti, dbBuildInfo)
+
+	prefix := "local.oplog.rs.stats.storageStats.wiredTiger"
+	if dbBuildInfo.VersionArray[0] < 7 {
+		prefix = "local.oplog.rs.stats.wiredTiger"
+	}
 
 	// The last \n at the end of this string is important
-	expected := strings.NewReader(`
-	# HELP mongodb_oplog_stats_ok local.oplog.rs.stats.
-	# TYPE mongodb_oplog_stats_ok untyped
-	mongodb_oplog_stats_ok 1
-	# HELP mongodb_oplog_stats_wt_btree_fixed_record_size local.oplog.rs.stats.wiredTiger.btree.
+	expectedString := fmt.Sprintf(`
+	# HELP mongodb_oplog_stats_wt_btree_fixed_record_size %s.btree.
 	# TYPE mongodb_oplog_stats_wt_btree_fixed_record_size untyped
 	mongodb_oplog_stats_wt_btree_fixed_record_size 0
-	# HELP mongodb_oplog_stats_wt_transaction_update_conflicts local.oplog.rs.stats.wiredTiger.transaction.
+	# HELP mongodb_oplog_stats_wt_transaction_update_conflicts %s.transaction.
 	# TYPE mongodb_oplog_stats_wt_transaction_update_conflicts untyped
-	mongodb_oplog_stats_wt_transaction_update_conflicts 0` + "\n")
+	mongodb_oplog_stats_wt_transaction_update_conflicts 0`, prefix, prefix)
+	expected := strings.NewReader(expectedString + "\n")
 
 	// Filter metrics for 2 reasons:
 	// 1. The result is huge
 	// 2. We need to check against know values. Don't use metrics that return counters like uptime
 	//    or counters like the number of transactions because they won't return a known value to compare
 	filter := []string{
-		"mongodb_oplog_stats_ok",
 		"mongodb_oplog_stats_wt_btree_fixed_record_size",
 		"mongodb_oplog_stats_wt_transaction_update_conflicts",
 	}
 
-	err := testutil.CollectAndCompare(c, expected, filter...)
+	err = testutil.CollectAndCompare(c, expected, filter...)
 	assert.NoError(t, err)
 }
 
@@ -188,7 +193,10 @@ func TestCollectorWithCompatibleMode(t *testing.T) {
 			logger := logrus.New()
 			ti := labelsGetterMock{}
 
-			c := newDiagnosticDataCollector(ctx, client, logger, true, ti)
+			dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, logger.WithField("component", "test"))
+			require.NoError(t, err)
+
+			c := newDiagnosticDataCollector(ctx, client, logger, true, ti, dbBuildInfo)
 
 			err = testutil.CollectAndCompare(c, tt.expectedMetrics(), tt.metricsFilter...)
 			assert.NoError(t, err)
@@ -202,12 +210,16 @@ func TestAllDiagnosticDataCollectorMetrics(t *testing.T) {
 
 	client := tu.DefaultTestClient(ctx, t)
 
-	ti := newTopologyInfo(ctx, client, logrus.New())
+	logger := logrus.New()
+	ti := newTopologyInfo(ctx, client, logger)
 
-	c := newDiagnosticDataCollector(ctx, client, logrus.New(), true, ti)
+	dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, logger.WithField("component", "test"))
+	require.NoError(t, err)
+
+	c := newDiagnosticDataCollector(ctx, client, logger, true, ti, dbBuildInfo)
 
 	reg := prometheus.NewRegistry()
-	err := reg.Register(c)
+	err = reg.Register(c)
 	require.NoError(t, err)
 	metrics := helpers.CollectMetrics(c)
 	actualMetrics := helpers.ReadMetrics(metrics)
@@ -281,7 +293,11 @@ func TestDiagnosticDataErrors(t *testing.T) {
 
 			logger, hook := logrustest.NewNullLogger()
 			ti := newTopologyInfo(ctx, client, logger)
-			c := newDiagnosticDataCollector(ctx, client, logger, true, ti)
+
+			dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, logger.WithField("component", "test"))
+			require.NoError(t, err)
+
+			c := newDiagnosticDataCollector(ctx, client, logger, true, ti, dbBuildInfo)
 
 			reg := prometheus.NewRegistry()
 			err = reg.Register(c)
@@ -318,19 +334,23 @@ func TestContextTimeout(t *testing.T) {
 
 	client := tu.DefaultTestClient(ctx, t)
 
-	ti := newTopologyInfo(ctx, client, logrus.New())
+	logger := logrus.New()
+	ti := newTopologyInfo(ctx, client, logger)
+
+	dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, logger.WithField("component", "test"))
+	require.NoError(t, err)
 
 	dbCount := 100
 
-	err := addTestData(ctx, client, dbCount)
+	err = addTestData(ctx, client, dbCount)
 	assert.NoError(t, err)
 
 	defer cleanTestData(ctx, client, dbCount) //nolint:errcheck
 
 	cctx, ccancel := context.WithCancel(context.Background())
 	ccancel()
 
-	c := newDiagnosticDataCollector(cctx, client, logrus.New(), true, ti)
+	c := newDiagnosticDataCollector(cctx, client, logger, true, ti, dbBuildInfo)
 	// it should not panic
 	helpers.CollectMetrics(c)
 }
@@ -403,7 +423,7 @@ func cleanTestData(ctx context.Context, client *mongo.Client, count int) error {
 }
 
 func TestDisconnectedDiagnosticDataCollector(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
 	client := tu.DefaultTestClient(ctx, t)
@@ -415,7 +435,10 @@ func TestDisconnectedDiagnosticDataCollector(t *testing.T) {
 
 	ti := labelsGetterMock{}
 
-	c := newDiagnosticDataCollector(ctx, client, logger, true, ti)
+	dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, logger.WithField("component", "test"))
+	require.Error(t, err)
+
+	c := newDiagnosticDataCollector(ctx, client, logger, true, ti, dbBuildInfo)
 
 	// The last \n at the end of this string is important
 	expected := strings.NewReader(`

exporter/encryption_info_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/percona/mongodb_exporter/internal/tu"
 )
@@ -45,7 +46,10 @@ func TestGetEncryptionInfo(t *testing.T) {
 
 	ti := labelsGetterMock{}
 
-	c := newDiagnosticDataCollector(ctx, client, logger, true, ti)
+	dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, logger.WithField("component", "test"))
+	require.NoError(t, err)
+
+	c := newDiagnosticDataCollector(ctx, client, logger, true, ti, dbBuildInfo)
 
 	// The last \n at the end of this string is important
 	expected := strings.NewReader(`
@@ -61,6 +65,6 @@ func TestGetEncryptionInfo(t *testing.T) {
 		"mongodb_version_info",
 	}
 
-	err := testutil.CollectAndCompare(c, expected, filter...)
+	err = testutil.CollectAndCompare(c, expected, filter...)
 	assert.NoError(t, err)
 }

exporter/exporter.go

@@ -134,7 +134,12 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 
 	nodeType, err := getNodeType(ctx, client)
 	if err != nil {
-		e.logger.Errorf("Registry - Cannot get node type to check if this is a mongos : %s", err)
+		e.logger.Errorf("Registry - Cannot get node type : %s", err)
+	}
+
+	dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, e.logger.WithField("component", "buildInfo"))
+	if err != nil {
+		e.logger.Warnf("Registry - Cannot get MongoDB buildInfo: %s", err)
 	}
 
 	gc := newGeneralCollector(ctx, client, nodeType, e.opts.Logger)
@@ -203,7 +208,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 
 	if e.opts.EnableDiagnosticData && requestOpts.EnableDiagnosticData {
 		ddc := newDiagnosticDataCollector(ctx, client, e.opts.Logger,
-			e.opts.CompatibleMode, topologyInfo)
+			e.opts.CompatibleMode, topologyInfo, dbBuildInfo)
 		registry.MustRegister(ddc)
 	}
 
@@ -234,7 +239,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 	// replSetGetStatus is not supported through mongos.
 	if e.opts.EnableReplicasetStatus && nodeType != typeMongos && requestOpts.EnableReplicasetStatus {
 		rsgsc := newReplicationSetStatusCollector(ctx, client, e.opts.Logger,
-			e.opts.CompatibleMode, topologyInfo)
+			e.opts.CompatibleMode, topologyInfo, dbBuildInfo)
 		registry.MustRegister(rsgsc)
 	}
 

exporter/exporter_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/percona/mongodb_exporter/internal/tu"
 )
@@ -187,8 +188,9 @@ func TestMongoS(t *testing.T) {
 
 		e := New(exporterOpts)
 
-		rsgsc := newReplicationSetStatusCollector(ctx, client, e.opts.Logger,
-			e.opts.CompatibleMode, new(labelsGetterMock))
+		version, err := retrieveMongoDBBuildInfo(ctx, client, exporterOpts.Logger.WithField("component", "test"))
+		require.NoError(t, err)
+		rsgsc := newReplicationSetStatusCollector(ctx, client, e.opts.Logger, e.opts.CompatibleMode, new(labelsGetterMock), version)
 
 		r := e.makeRegistry(ctx, client, new(labelsGetterMock), *e.opts)
 

exporter/feature_compatibility_version_collector_test.go

@@ -55,6 +55,14 @@ func TestFCVCollector(t *testing.T) {
 		mversion = "4.4"
 	case mmv == "4.4":
 		mversion = "4.2"
+	case mmv == "6.0":
+		mversion = "5.0"
+	case mmv == "7.0":
+		mversion = "6.0"
+	case mmv == "8.0":
+		mversion = "7.0"
+	default:
+		mversion = mmv
 	}
 
 	// The last \n at the end of this string is important

exporter/metrics.go

@@ -52,11 +52,14 @@ var (
 	prefixes = [][]string{
 		{"serverStatus.wiredTiger.transaction", "ss_wt_txn"},
 		{"serverStatus.wiredTiger", "ss_wt"},
+		{"serverStatus.queues.execution", "ss_wt_concurrentTransactions"},
 		{"serverStatus", "ss"},
 		{"replSetGetStatus", "rs"},
 		{"systemMetrics", "sys"},
 		{"local.oplog.rs.stats.wiredTiger", "oplog_stats_wt"},
+		{"local.oplog.rs.stats.storageStats.wiredTiger", "oplog_stats_wt"},
 		{"local.oplog.rs.stats", "oplog_stats"},
+		{"local.oplog.rs.stats.storageStats", "oplog_stats"},
 		{"collstats_storage.wiredTiger", "collstats_storage_wt"},
 		{"collstats_storage.indexDetails", "collstats_storage_idx"},
 		{"collStats.storageStats", "collstats_storage"},
@@ -105,6 +108,7 @@ var (
 		"serverStatus.opcountersRepl.":                    "legacy_op_type",
 		"serverStatus.transactions.commitTypes.":          "commit_type",
 		"serverStatus.wiredTiger.concurrentTransactions.": "txn_rw_type",
+		"serverStatus.queues.execution.":                  "txn_rw_type",
 		"serverStatus.wiredTiger.perf.":                   "perf_bucket",
 		"systemMetrics.disks.":                            "device_name",
 	}

exporter/replset_status_collector.go

@@ -17,6 +17,7 @@ package exporter
 
 import (
 	"context"
+	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
@@ -35,16 +36,19 @@ type replSetGetStatusCollector struct {
 
 	compatibleMode bool
 	topologyInfo   labelsGetter
+
+	version buildInfo
 }
 
 // newReplicationSetStatusCollector creates a collector for statistics on replication set.
-func newReplicationSetStatusCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, compatible bool, topology labelsGetter) *replSetGetStatusCollector {
+func newReplicationSetStatusCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, compatible bool, topology labelsGetter, version buildInfo) *replSetGetStatusCollector {
 	return &replSetGetStatusCollector{
 		ctx:  ctx,
 		base: newBaseCollector(client, logger.WithFields(logrus.Fields{"collector": "replset_status"})),
 
 		compatibleMode: compatible,
 		topologyInfo:   topology,
+		version:        version,
 	}
 }
 
@@ -84,6 +88,19 @@ func (d *replSetGetStatusCollector) collect(ch chan<- prometheus.Metric) {
 	for _, metric := range makeMetrics("", m, d.topologyInfo.baseLabels(), d.compatibleMode) {
 		ch <- metric
 	}
+
+	if strings.HasPrefix(d.version.Version, "8.") {
+		for _, metric := range makeMetrics("rs", m, d.topologyInfo.baseLabels(), d.compatibleMode) {
+			ch <- metric
+		}
+		if d.compatibleMode {
+			logger.Infof("collecting compatibility metrics for version %s", d.version.Version)
+			metrics := replSetMetrics(m, logger)
+			for _, metric := range metrics {
+				ch <- metric
+			}
+		}
+	}
 }
 
 var _ prometheus.Collector = (*replSetGetStatusCollector)(nil)