Initial gmp exporter (#342)

Add GMP exporter

exporter/collector/config.go

@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"time"
 
+	"go.opentelemetry.io/collector/model/pdata"
 	"google.golang.org/api/option"
+	monitoredrespb "google.golang.org/genproto/googleapis/api/monitoredres"
 )
 
 const (
@@ -106,6 +108,18 @@ type MetricConfig struct {
 	// so we don't send it by default, and don't expose it to users. For some uses, it is
 	// expected, however.
 	EnableSumOfSquaredDeviation bool `mapstructure:"sum_of_squared_deviation"`
+
+	// GetMetricName is not settable in config files, but can be used by other
+	// exporters which extend the functionality of this exporter. It allows
+	// customizing the naming of metrics. baseName already includes type
+	// suffixes for summary metrics, but does not (yet) include the domain prefix
+	GetMetricName func(baseName string, metric pdata.Metric) (string, error)
+
+	// MapMonitoredResource is not exposed as an option in the configuration, but
+	// can be used by other exporters to extend the functionality of this
+	// exporter. It allows overriding the function used to map otel resource to
+	// monitored resource.
+	MapMonitoredResource func(pdata.Resource) *monitoredrespb.MonitoredResource
 }
 
 type ResourceFilter struct {
@@ -135,6 +149,8 @@ func DefaultConfig() Config {
 			InstrumentationLibraryLabels:     true,
 			ServiceResourceLabels:            true,
 			CumulativeNormalization:          true,
+			GetMetricName:                    defaultGetMetricName,
+			MapMonitoredResource:             defaultResourceToMonitoredResource,
 		},
 	}
 }

exporter/collector/googlemanagedprometheus/README.md

@@ -0,0 +1,52 @@
+# Google Managed Service for Prometheus Collector Exporter
+
+## Building a container image with the googlemanagedprometheus exporter
+
+In your own fork of [open-telemetry/opentelemetry-collector-releases](https://github.com/open-telemetry/opentelemetry-collector-releases), add your own "distribution" directory within the distributions directory, based on either the otelcol or otelcol-contrib distributions. In the `exporters` list in `manifest.yaml`, add:
+```yaml
+exporters:
+  - gomod: "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector/googlemanagedprometheus v0.29.0"
+```
+
+The syntax of `manifest.yaml` is described in the [Collector Builder documentation](https://github.com/open-telemetry/opentelemetry-collector/blob/54f271b7d473f36b4ecbc21994d59359dbd263f6/cmd/builder/README.md#opentelemetry-collector-builder).
+
+In the `configs` directory, add your collector configuration yaml file, which should look something like:
+
+```yaml
+receivers:
+    prometheus:
+        config:
+          scrape_configs:
+            # TODO: Add your prometheus scrape configuration here.
+            # Using kubernetes_sd_configs with namespaced resources
+            # ensures the namespace is set on your metrics.
+processors:
+    batch:
+        # batch metrics before sending to reduce API usage
+        send_batch_max_size: 200
+        send_batch_size: 200
+        timeout: 5s
+    memory_limiter:
+        # drop metrics if memory usage gets too high
+        check_interval: 1s
+        limit_percentage: 65
+        spike_limit_percentage: 20
+    resourcedetection:
+        # detect cluster name and location
+        detectors: [gce, gke]
+        timeout: 10s
+exporters:
+    googlemanagedprometheus:
+```
+
+Change the Dockerfile in your directory within `distributions` to point to your collector config [here](https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/Dockerfile#L17).
+
+Finally, build the image: 
+
+```sh
+DISTRIBUTIONS=my-distribution make build
+```
+
+## Additional Options
+
+The [filterprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor) can filter out metrics. The [metricstransformprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/metricstransformprocessor) can manipulate metrics in a variety of ways, including synthesizing new metrics from other metrics, adding or removing labels, renaming metrics, and scaling metrics. `metric_relabl_configs` within the prometheus receiver configuration can also be used to manipulate metrics.

exporter/collector/googlemanagedprometheus/config.go

@@ -0,0 +1,52 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package googlemanagedprometheus
+
+import (
+	"fmt"
+
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector"
+)
+
+// Config defines configuration for Google Cloud Managed Service for Prometheus exporter.
+type Config struct {
+	config.ExporterSettings `mapstructure:",squash"`
+	GMPConfig               `mapstructure:",squash"`
+
+	// Timeout for all API calls. If not set, defaults to 12 seconds.
+	exporterhelper.TimeoutSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
+	exporterhelper.QueueSettings   `mapstructure:"sending_queue"`
+	exporterhelper.RetrySettings   `mapstructure:"retry_on_failure"`
+}
+
+// GMPConfig is a subset of the collector config.
+type GMPConfig struct {
+	ProjectID    string                 `mapstructure:"project"`
+	UserAgent    string                 `mapstructure:"user_agent"`
+	ClientConfig collector.ClientConfig `mapstructure:",squash"`
+}
+
+func (cfg *Config) Validate() error {
+	if err := cfg.ExporterSettings.Validate(); err != nil {
+		return fmt.Errorf("exporter settings are invalid :%w", err)
+	}
+	if err := collector.ValidateConfig(cfg.toCollectorConfig()); err != nil {
+		return fmt.Errorf("exporter settings are invalid :%w", err)
+	}
+	return nil
+}

exporter/collector/googlemanagedprometheus/factory.go

@@ -0,0 +1,98 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package googlemanagedprometheus
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector"
+)
+
+const (
+	// The value of "type" key in configuration.
+	typeStr        = "googlemanagedprometheus"
+	defaultTimeout = 12 * time.Second // Consistent with Cloud Monitoring's timeout
+)
+
+// NewFactory creates a factory for the googlemanagedprometheus exporter
+func NewFactory() component.ExporterFactory {
+	return component.NewExporterFactory(
+		typeStr,
+		createDefaultConfig,
+		component.WithMetricsExporter(createMetricsExporter),
+	)
+}
+
+// createDefaultConfig creates the default configuration for exporter.
+func createDefaultConfig() config.Exporter {
+	return &Config{
+		ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+		TimeoutSettings:  exporterhelper.TimeoutSettings{Timeout: defaultTimeout},
+		RetrySettings:    exporterhelper.NewDefaultRetrySettings(),
+		QueueSettings:    exporterhelper.NewDefaultQueueSettings(),
+		GMPConfig: GMPConfig{
+			UserAgent: "opentelemetry-collector-contrib/{{version}}",
+		},
+	}
+}
+
+// createMetricsExporter creates a metrics exporter based on this config.
+func createMetricsExporter(
+	ctx context.Context,
+	params component.ExporterCreateSettings,
+	cfg config.Exporter) (component.MetricsExporter, error) {
+	eCfg := cfg.(*Config)
+	mExp, err := collector.NewGoogleCloudMetricsExporter(ctx, eCfg.GMPConfig.toCollectorConfig(), params.TelemetrySettings.Logger, params.BuildInfo.Version, eCfg.Timeout)
+	if err != nil {
+		return nil, err
+	}
+	return exporterhelper.NewMetricsExporter(
+		cfg,
+		params,
+		mExp.PushMetrics,
+		exporterhelper.WithShutdown(mExp.Shutdown),
+		// Disable exporterhelper Timeout, since we are using a custom mechanism
+		// within exporter itself
+		exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}),
+		exporterhelper.WithQueue(eCfg.QueueSettings),
+		exporterhelper.WithRetry(eCfg.RetrySettings))
+}
+
+func (c *GMPConfig) toCollectorConfig() collector.Config {
+	// start with whatever the default collector config is.
+	cfg := collector.DefaultConfig()
+	// hard-code some config options to make it work with GMP
+	cfg.MetricConfig.Prefix = "prometheus.googleapis.com"
+	cfg.MetricConfig.SkipCreateMetricDescriptor = true
+	cfg.MetricConfig.InstrumentationLibraryLabels = false
+	cfg.MetricConfig.ServiceResourceLabels = false
+	// Update metric naming to match GMP conventions
+	cfg.MetricConfig.GetMetricName = GetMetricName
+	// Map to the prometheus_target monitored resource
+	cfg.MetricConfig.MapMonitoredResource = MapToPrometheusTarget
+	cfg.MetricConfig.EnableSumOfSquaredDeviation = true
+	// TODO: Change to GMP's method of reset handling.
+	cfg.MetricConfig.CumulativeNormalization = false
+	// map the GMP config's fields to the collector config
+	cfg.ProjectID = c.ProjectID
+	cfg.UserAgent = c.UserAgent
+	cfg.MetricConfig.ClientConfig = c.ClientConfig
+	return cfg
+}

exporter/collector/googlemanagedprometheus/go.mod

@@ -0,0 +1,60 @@
+module github.com/dashpole/opentelemetry-operations-go/exporter/collector/googlemanagedprometheus
+
+go 1.17
+
+require (
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector v0.28.0
+	github.com/stretchr/testify v1.7.1
+	go.opentelemetry.io/collector v0.49.0
+	go.opentelemetry.io/collector/model v0.49.0
+	go.opentelemetry.io/collector/pdata v0.49.0
+	google.golang.org/genproto v0.0.0-20220405205423-9d709892a2bf
+)
+
+require (
+	cloud.google.com/go v0.100.2 // indirect
+	cloud.google.com/go/compute v1.5.0 // indirect
+	cloud.google.com/go/logging v1.4.2 // indirect
+	cloud.google.com/go/monitoring v1.4.0 // indirect
+	cloud.google.com/go/trace v1.2.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.5.1 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.29.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/googleapis/gax-go/v2 v2.2.0 // indirect
+	github.com/knadh/koanf v1.4.1 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/otel v1.6.3 // indirect
+	go.opentelemetry.io/otel/metric v0.29.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.6.3 // indirect
+	go.opentelemetry.io/otel/trace v1.6.3 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/zap v1.21.0 // indirect
+	golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect
+	golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/api v0.74.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/grpc v1.45.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)
+
+replace (
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector => ../../collector
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace => ../../trace
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping => ../../../internal/resourcemapping
+)