prometheusreceiver: Don't ignore instance labels for internal metrics

.chloggen/prom_federated_internal_metrics.yaml

@@ -0,0 +1,4 @@
+change_type: bug_fix
+component: prometheusreceiver
+note: Don't ignore instance, job, or metric path labels for internal metrics
+issues: [14453]

receiver/prometheusreceiver/internal/metricfamily.go

@@ -49,6 +49,7 @@ type metricFamily struct {
 // simple types like counter and gauge, each data point is a group of itself
 type metricGroup struct {
 	mtype        pmetric.MetricType
+	name         string
 	ts           int64
 	ls           labels.Labels
 	count        float64
@@ -141,7 +142,7 @@ func (mg *metricGroup) toDistributionPoint(dest pmetric.HistogramDataPointSlice)
 	tsNanos := timestampFromMs(mg.ts)
 	point.SetStartTimestamp(tsNanos) // metrics_adjuster adjusts the startTimestamp to the initial scrape timestamp
 	point.SetTimestamp(tsNanos)
-	populateAttributes(pmetric.MetricTypeHistogram, mg.ls, point.Attributes())
+	populateAttributes(pmetric.MetricTypeHistogram, mg.name, mg.ls, point.Attributes())
 	mg.setExemplars(point.Exemplars())
 }
 
@@ -195,7 +196,7 @@ func (mg *metricGroup) toSummaryPoint(dest pmetric.SummaryDataPointSlice) {
 	tsNanos := timestampFromMs(mg.ts)
 	point.SetTimestamp(tsNanos)
 	point.SetStartTimestamp(tsNanos) // metrics_adjuster adjusts the startTimestamp to the initial scrape timestamp
-	populateAttributes(pmetric.MetricTypeSummary, mg.ls, point.Attributes())
+	populateAttributes(pmetric.MetricTypeSummary, mg.name, mg.ls, point.Attributes())
 }
 
 func (mg *metricGroup) toNumberDataPoint(dest pmetric.NumberDataPointSlice) {
@@ -211,13 +212,13 @@ func (mg *metricGroup) toNumberDataPoint(dest pmetric.NumberDataPointSlice) {
 	} else {
 		point.SetDoubleValue(mg.value)
 	}
-	populateAttributes(pmetric.MetricTypeGauge, mg.ls, point.Attributes())
+	populateAttributes(pmetric.MetricTypeGauge, mg.name, mg.ls, point.Attributes())
 	mg.setExemplars(point.Exemplars())
 }
 
-func populateAttributes(mType pmetric.MetricType, ls labels.Labels, dest pcommon.Map) {
+func populateAttributes(mType pmetric.MetricType, mName string, ls labels.Labels, dest pcommon.Map) {
 	dest.EnsureCapacity(ls.Len())
-	names := getSortedNotUsefulLabels(mType)
+	names := getSortedNotUsefulLabels(mType, mName)
 	j := 0
 	for i := range ls {
 		for j < len(names) && names[j] < ls[i].Name {
@@ -239,6 +240,7 @@ func (mf *metricFamily) loadMetricGroupOrCreate(groupKey uint64, ls labels.Label
 	if !ok {
 		mg = &metricGroup{
 			mtype:     mf.mtype,
+			name:      mf.name,
 			ts:        ts,
 			ls:        ls,
 			exemplars: pmetric.NewExemplarSlice(),

receiver/prometheusreceiver/internal/metricfamily_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/textparse"
 	"github.com/prometheus/prometheus/model/value"
@@ -187,7 +188,7 @@ func TestMetricGroupData_toDistributionUnitTest(t *testing.T) {
 				} else {
 					lbls = tt.labels.Copy()
 				}
-				sRef, _ := getSeriesRef(nil, lbls, mp.mtype)
+				sRef, _ := getSeriesRef(nil, lbls, mp.mtype, mp.name)
 				err := mp.addSeries(sRef, tv.metric, lbls, tv.at, tv.value)
 				if tt.wantErr {
 					if i != 0 {
@@ -400,7 +401,7 @@ func TestMetricGroupData_toSummaryUnitTest(t *testing.T) {
 			for _, lbs := range tt.labelsScrapes {
 				for i, scrape := range lbs.scrapes {
 					lb := lbs.labels.Copy()
-					sRef, _ := getSeriesRef(nil, lb, mp.mtype)
+					sRef, _ := getSeriesRef(nil, lb, mp.mtype, mp.name)
 					err := mp.addSeries(sRef, scrape.metric, lb, scrape.at, scrape.value)
 					if tt.wantErr {
 						// The first scrape won't have an error
@@ -496,7 +497,7 @@ func TestMetricGroupData_toNumberDataUnitTest(t *testing.T) {
 			mp := newMetricFamily(tt.metricKind, mc, zap.NewNop())
 			for _, tv := range tt.scrapes {
 				lb := tt.labels.Copy()
-				sRef, _ := getSeriesRef(nil, lb, mp.mtype)
+				sRef, _ := getSeriesRef(nil, lb, mp.mtype, mp.name)
 				require.NoError(t, mp.addSeries(sRef, tv.metric, lb, tv.at, tv.value))
 			}
 
@@ -518,3 +519,61 @@ func TestMetricGroupData_toNumberDataUnitTest(t *testing.T) {
 		})
 	}
 }
+
+func TestMetricGroupData_toInternalMetric(t *testing.T) {
+	tests := []struct{ name string }{
+		{name: "up"},
+		{name: "scrape_duration_seconds"},
+		{name: "scrape_samples_scraped"},
+		{name: "scrape_series_added"},
+		{name: "scrape_samples_post_metric_relabeling"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mp := newMetricFamily(tt.name, mc, zap.NewNop())
+
+			lb := labels.FromMap(map[string]string{
+				"a": "A", "b": "B",
+				model.MetricsPathLabel: "a.path",
+				model.MetricNameLabel:  "scrape_duration_seconds",
+				model.SchemeLabel:      "a.scheme",
+				model.InstanceLabel:    "an.instance",
+				model.JobLabel:         "a.job",
+			})
+
+			sRef, _ := getSeriesRef(nil, lb, mp.mtype, mp.name)
+			require.NoError(t, mp.addSeries(sRef, "value", lb, 28, 99.9))
+
+			require.Len(t, mp.groups, 1)
+
+			sl := pmetric.NewMetricSlice()
+			mp.appendMetric(sl)
+
+			require.Equal(t, 1, sl.Len(), "Exactly one metric expected")
+			metric := sl.At(0)
+			require.Equal(t, internalMetricMetadata[tt.name].Help, metric.Description(), "Expected help metadata in metric description")
+			require.Equal(t, internalMetricMetadata[tt.name].Unit, metric.Unit(), "Expected unit metadata in metric")
+
+			ndpL := metric.Gauge().DataPoints()
+			require.Equal(t, 1, ndpL.Len(), "Exactly one point expected")
+			got := ndpL.At(0)
+			want := pmetric.NewNumberDataPoint()
+			want.SetDoubleValue(99.9)
+			want.SetTimestamp(pcommon.Timestamp(28 * time.Millisecond))
+			want.Attributes().FromRaw(
+				map[string]interface{}{
+					"a": "A", "b": "B",
+					model.MetricsPathLabel: "a.path",
+					model.InstanceLabel:    "an.instance",
+					model.JobLabel:         "a.job",
+				})
+
+			// require.Equal doesn't work well w/ *[]otlpcommon.KeyValue
+			// so assert the attribute maps are the same before clearing
+			require.Equal(t, want.Attributes().AsRaw(), got.Attributes().AsRaw())
+			got.Attributes().Clear()
+			want.Attributes().Clear()
+			require.EqualValues(t, want, got, "Expected the points to be equal")
+		})
+	}
+}

receiver/prometheusreceiver/internal/transaction.go

@@ -129,7 +129,7 @@ func (t *transaction) Append(ref storage.SeriesRef, ls labels.Labels, atMs int64
 
 	curMF := t.getOrCreateMetricFamily(metricName)
 
-	return 0, curMF.addSeries(t.getSeriesRef(ls, curMF.mtype), metricName, ls, atMs, val)
+	return 0, curMF.addSeries(t.getSeriesRef(ls, curMF.mtype, metricName), metricName, ls, atMs, val)
 }
 
 func (t *transaction) getOrCreateMetricFamily(mn string) *metricFamily {
@@ -174,14 +174,14 @@ func (t *transaction) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e e
 	}
 
 	mf := t.getOrCreateMetricFamily(mn)
-	mf.addExemplar(t.getSeriesRef(l, mf.mtype), e)
+	mf.addExemplar(t.getSeriesRef(l, mf.mtype, mf.name), e)
 
 	return 0, nil
 }
 
-func (t *transaction) getSeriesRef(ls labels.Labels, mtype pmetric.MetricType) uint64 {
+func (t *transaction) getSeriesRef(ls labels.Labels, mtype pmetric.MetricType, mname string) uint64 {
 	var hash uint64
-	hash, t.bufBytes = getSeriesRef(t.bufBytes, ls, mtype)
+	hash, t.bufBytes = getSeriesRef(t.bufBytes, ls, mtype, mname)
 	return hash
 }
 
@@ -273,6 +273,6 @@ func (t *transaction) AddTargetInfo(labels labels.Labels) error {
 	return nil
 }
 
-func getSeriesRef(bytes []byte, ls labels.Labels, mtype pmetric.MetricType) (uint64, []byte) {
-	return ls.HashWithoutLabels(bytes, getSortedNotUsefulLabels(mtype)...)
+func getSeriesRef(bytes []byte, ls labels.Labels, mtype pmetric.MetricType, name string) (uint64, []byte) {
+	return ls.HashWithoutLabels(bytes, getSortedNotUsefulLabels(mtype, name)...)
 }

receiver/prometheusreceiver/internal/util.go

@@ -53,14 +53,22 @@ var (
 	notUsefulLabelsHistogram = sortString([]string{model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel, model.BucketLabel})
 	notUsefulLabelsSummary   = sortString([]string{model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel, model.QuantileLabel})
 	notUsefulLabelsOther     = sortString([]string{model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel})
+	notUsefulLabelsInternal  = sortString([]string{model.MetricNameLabel, model.SchemeLabel})
 )
 
 func sortString(strs []string) []string {
 	sort.Strings(strs)
 	return strs
 }
 
-func getSortedNotUsefulLabels(mType pmetric.MetricType) []string {
+func getSortedNotUsefulLabels(mType pmetric.MetricType, mName string) []string {
+	if _, ok := internalMetricMetadata[mName]; ok {
+		// For internal time series don't filter instance, job, or metrics path labels as these
+		// differentiate federated instances in cases of otherwise equivalent labels.
+		// Internal metrics are all gauges.
+		return notUsefulLabelsInternal
+	}
+
 	switch mType {
 	case pmetric.MetricTypeHistogram:
 		return notUsefulLabelsHistogram