Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Prometheus Remote Write Exporter supporting Cortex - conversion and export for Summary OTLP metrics #1649

Closed
wants to merge 25 commits into from
+573 −108
Closed

Add Prometheus Remote Write Exporter supporting Cortex - conversion and export for Summary OTLP metrics #1649

Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
086e84c
implement prw exporter
huyan0 Jul 30, 2020
2bd3936
built success
huyan0 Aug 24, 2020
f0b8d77
remove supoprt for summary
huyan0 Aug 27, 2020
af058eb
remove supoprt for summary
huyan0 Aug 27, 2020
5028bf7
fix lint
huyan0 Aug 27, 2020
876d1be
rebase should work
huyan0 Aug 28, 2020
4beab20
rebase should work
huyan0 Aug 28, 2020
ff07141
add comments and double check
huyan0 Aug 28, 2020
af56e3f
increase coverage
huyan0 Aug 28, 2020
3d6cf0f
increase coverage
huyan0 Aug 28, 2020
2776235
increase coverage
huyan0 Aug 28, 2020
f2c76d5
increase coverage
huyan0 Aug 28, 2020
73f05c5
update test coverage
huyan0 Aug 28, 2020
860ae77
improve coverage
huyan0 Aug 28, 2020
d22c6b4
improve coverage
huyan0 Aug 28, 2020
5ae7599
fix go.mod
huyan0 Aug 28, 2020
f832ebe
fix dropped metric count
huyan0 Aug 28, 2020
757b82f
change exporter methods
huyan0 Aug 28, 2020
ebbe7b2
fix lint
huyan0 Aug 28, 2020
c4cd9d4
fix lint
huyan0 Aug 28, 2020
4c7aac0
fix test failing
huyan0 Aug 28, 2020
1407206
implement prw exporter
huyan0 Jul 30, 2020
18883d1
message
huyan0 Aug 28, 2020
a464e30
save progress
huyan0 Aug 30, 2020
d07f56c
temp test
huyan0 Aug 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
increase coverage
@huyan0
huyan0 committed Aug 28, 2020
commit af56e3f967e5a9aadf0e79867bd44d8cf780d5f0
5 changes: 4 additions & 1 deletion exporter/prometheusremotewriteexporter/exporter.go
View file
Original file line number Diff line number Diff line change
@@ -124,13 +124,16 @@ func (prwe *prwExporter) pushMetrics(ctx context.Context, md pdata.Metrics) (int
if err := prwe.handleHistogramMetric(tsMap, metric); err != nil {
errs = append(errs, err)
}
default:
errs = append(errs, fmt.Errorf("Unsupported metric type"))
}
}
}
}

if err := prwe.export(ctx, tsMap); err != nil {
return pdatautil.MetricCount(md), err
dropped = pdatautil.MetricCount(md)
errs = append(errs, err)
}

if dropped != 0 {
37 changes: 37 additions & 0 deletions exporter/prometheusremotewriteexporter/exporter_test.go
View file
Original file line number Diff line number Diff line change
@@ -81,6 +81,18 @@ func Test_handleScalarMetric(t *testing.T) {
true,
map[string]*prompb.TimeSeries{},
},
{
"invalid_metric_type",
&otlp.Metric{
MetricDescriptor: getDescriptor("invalid_nil_array", histogramComb, validCombinations),
Int64DataPoints: nil,
DoubleDataPoints: nil,
HistogramDataPoints: nil,
SummaryDataPoints: nil,
},
true,
map[string]*prompb.TimeSeries{},
},
{
"same_ts_int_points",
&otlp.Metric{
@@ -483,6 +495,23 @@ func Test_pushMetrics(t *testing.T) {
require.Nil(t, ok)
assert.EqualValues(t, expected, len(wr.Timeseries))
}

summary := dataold.MetricDataToOtlp(testdataold.GenerateMetricDataOneMetric())
summary[0].InstrumentationLibraryMetrics[0].Metrics[0] = &otlp.Metric{
MetricDescriptor: getDescriptor("summary_test", summaryComb, validCombinations),
SummaryDataPoints: []*otlp.SummaryDataPoint{getSummaryDataPoint(
lbs1,
time1,
floatVal1,
uint64(intVal1),
[]float64{floatVal1},
[]float64{floatVal2},
),
},
}
summaryBatch := pdatautil.MetricsFromOldInternalMetrics(dataold.MetricDataFromOtlp(summary))


tests := []struct {
name string
md *pdata.Metrics
@@ -572,6 +601,14 @@ func Test_pushMetrics(t *testing.T) {
0,
false,
},
{"summary_case",
&summaryBatch,
checkFunc,
3,
http.StatusAccepted,
0,
false,
},
}

for _, tt := range tests {
125 changes: 125 additions & 0 deletions exporter/prometheusremotewriteexporter/out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
mode: set
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:52.99,54.19 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:58.2,59.16 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:63.2,69.8 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:54.19,56.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:59.16,61.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:74.58,78.2 3 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:83.90,86.9 3 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:87.24,88.75 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:89.10,95.50 5 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:134.3,134.49 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:139.3,139.19 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:143.3,143.16 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:95.50,96.29 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:100.4,100.88 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:96.29,97.13 1 0
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:100.88,101.38 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:105.5,105.59 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:101.38,102.14 1 0
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:105.59,106.23 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:110.6,110.61 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:116.6,116.52 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:106.23,107.15 1 0
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:110.61,113.15 3 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:118.76,119.68 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:123.43,124.71 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:127.14,128.65 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:119.68,122.8 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:124.71,126.8 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:134.49,137.4 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:139.19,141.4 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:150.109,154.15 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:195.2,195.75 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:156.74,157.36 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:161.3,161.45 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:174.3,174.13 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:177.76,178.37 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:181.3,181.46 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:193.3,193.13 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:157.36,159.4 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:161.45,173.4 4 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:178.37,180.4 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:181.46,192.4 4 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:201.112,203.39 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:207.2,207.48 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:254.2,254.12 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:203.39,205.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:207.48,235.39 11 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:247.3,252.48 3 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:235.39,245.4 5 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:258.97,261.16 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:266.2,267.16 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:270.2,275.16 4 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:281.2,291.16 8 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:295.2,295.34 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:304.2,304.12 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:261.16,263.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:267.16,269.3 1 0
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:275.16,277.3 1 0
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:291.16,293.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:295.34,298.21 3 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:301.3,302.28 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/exporter.go:298.21,300.4 1 0
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/factory.go:32.45,37.2 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/factory.go:40.64,43.9 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/factory.go:47.2,49.16 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/factory.go:53.2,55.16 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/factory.go:59.2,68.20 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/factory.go:43.9,45.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/factory.go:49.16,51.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/factory.go:55.16,57.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/factory.go:71.50,95.2 3 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:46.42,46.59 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:47.42,47.74 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:48.42,48.69 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:51.56,53.17 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:57.2,57.24 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:65.2,65.14 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:53.17,55.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:59.66,60.67 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:61.65,62.14 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:71.33,73.52 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:77.2,80.8 3 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:73.52,75.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:80.8,82.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:82.8,88.3 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:95.87,101.29 4 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:108.2,108.19 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:101.29,106.3 4 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:114.87,119.28 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:126.2,126.38 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:145.2,147.23 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:151.2,151.10 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:119.28,124.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:126.38,127.25 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:130.3,131.12 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:135.3,136.73 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:139.3,142.4 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:127.25,128.9 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:131.12,133.4 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:136.73,138.4 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:147.23,149.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:156.71,158.17 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:162.2,169.17 4 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:172.2,183.30 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:187.2,187.29 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:158.17,160.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:169.17,171.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:183.30,186.3 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:192.88,193.21 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:196.2,197.26 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:200.2,204.22 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:193.21,195.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:197.26,199.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:208.47,210.2 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:214.32,215.17 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:222.2,223.33 2 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:226.2,226.17 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:229.2,229.10 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:215.17,217.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:223.33,225.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:226.17,228.3 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:234.32,235.47 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:239.2,239.12 1 1
go.opentelemetry.io/collector/exporter/prometheusremotewriteexporter/helper.go:235.47,237.3 1 1
18 changes: 18 additions & 0 deletions exporter/prometheusremotewriteexporter/testutil_test.go
View file
Original file line number Diff line number Diff line change
@@ -175,6 +175,24 @@ func getHistogramDataPoint(labels []*commonpb.StringKeyValue, ts uint64, sum flo
}
}

func getSummaryDataPoint(labels []*commonpb.StringKeyValue, ts uint64, sum float64, count uint64, pcts []float64, values []float64) *otlp.SummaryDataPoint {
pcs := []*otlp.SummaryDataPoint_ValueAtPercentile{}
for i, v := range values {
pcs = append(pcs, &otlp.SummaryDataPoint_ValueAtPercentile{
Percentile: pcts[i],
Value: v,
})
}
return &otlp.SummaryDataPoint{
Labels: labels,
StartTimeUnixNano: 0,
TimeUnixNano: ts,
Count: count,
Sum: sum,
PercentileValues: pcs,
}
}

// Prometheus TimeSeries
func getPromLabels(lbs ...string) []prompb.Label {
pbLbs := prompb.Labels{