Add an otelcol.processor.resourcedetection component (#5764)

* Add a otelcol.processor.resourcedetection component * PR review suggestions * Make k8s API config authentication options const. * Fail validation if an incorrect detector is supplied. * Remove unnecessary Consul attributes. --------- Co-authored-by: Clayton Cornell <[email protected]>
grafana · Jan 23, 2024 · 74245fd · 74245fd
1 parent 25f8b51
commit 74245fd
Show file tree

Hide file tree

Showing 24 changed files with 3,751 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,9 @@ Main (unreleased)
 
 - A new `pyroscope.java` component for profiling Java processes using async-profiler. (@korniltsev)
 
+- A new `otelcol.processor.resourcedetection` component which inserts resource attributes 
+  to OTLP telemetry based on the host on which Grafana Agent is running. (@ptodev)
+
 ### Enhancements
 
 - Include line numbers in profiles produced by `pyrsocope.java` component. (@korniltsev)

diff --git a/component/all/all.go b/component/all/all.go
@@ -82,6 +82,7 @@ import (
 	_ "github.com/grafana/agent/component/otelcol/processor/k8sattributes"          // Import otelcol.processor.k8sattributes
 	_ "github.com/grafana/agent/component/otelcol/processor/memorylimiter"          // Import otelcol.processor.memory_limiter
 	_ "github.com/grafana/agent/component/otelcol/processor/probabilistic_sampler"  // Import otelcol.processor.probabilistic_sampler
+	_ "github.com/grafana/agent/component/otelcol/processor/resourcedetection"      // Import otelcol.processor.resourcedetection
 	_ "github.com/grafana/agent/component/otelcol/processor/span"                   // Import otelcol.processor.span
 	_ "github.com/grafana/agent/component/otelcol/processor/tail_sampling"          // Import otelcol.processor.tail_sampling
 	_ "github.com/grafana/agent/component/otelcol/processor/transform"              // Import otelcol.processor.transform

diff --git a/component/otelcol/config_k8s.go b/component/otelcol/config_k8s.go
@@ -0,0 +1,35 @@
+package otelcol
+
+import "fmt"
+
+const (
+	KubernetesAPIConfig_AuthType_None           = "none"
+	KubernetesAPIConfig_AuthType_ServiceAccount = "serviceAccount"
+	KubernetesAPIConfig_AuthType_KubeConfig     = "kubeConfig"
+	KubernetesAPIConfig_AuthType_TLS            = "tls"
+)
+
+// KubernetesAPIConfig contains options relevant to connecting to the K8s API
+type KubernetesAPIConfig struct {
+	// How to authenticate to the K8s API server.  This can be one of `none`
+	// (for no auth), `serviceAccount` (to use the standard service account
+	// token provided to the agent pod), or `kubeConfig` to use credentials
+	// from `~/.kube/config`.
+	AuthType string `river:"auth_type,attr,optional"`
+
+	// When using auth_type `kubeConfig`, override the current context.
+	Context string `river:"context,attr,optional"`
+}
+
+// Validate returns an error if the config is invalid.
+func (c *KubernetesAPIConfig) Validate() error {
+	switch c.AuthType {
+	case KubernetesAPIConfig_AuthType_None,
+		KubernetesAPIConfig_AuthType_ServiceAccount,
+		KubernetesAPIConfig_AuthType_KubeConfig,
+		KubernetesAPIConfig_AuthType_TLS:
+		return nil
+	default:
+		return fmt.Errorf("invalid auth_type %q", c.AuthType)
+	}
+}
diff --git a/component/otelcol/processor/resourcedetection/internal/aws/ec2/config.go b/component/otelcol/processor/resourcedetection/internal/aws/ec2/config.go
@@ -0,0 +1,72 @@
+package ec2
+
+import (
+	rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config"
+	"github.com/grafana/river"
+)
+
+const Name = "ec2"
+
+// Config defines user-specified configurations unique to the EC2 detector
+type Config struct {
+	// Tags is a list of regex's to match ec2 instance tag keys that users want
+	// to add as resource attributes to processed data
+	Tags               []string                 `river:"tags,attr,optional"`
+	ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block,optional"`
+}
+
+// DefaultArguments holds default settings for Config.
+var DefaultArguments = Config{
+	ResourceAttributes: ResourceAttributesConfig{
+		CloudAccountID:        rac.ResourceAttributeConfig{Enabled: true},
+		CloudAvailabilityZone: rac.ResourceAttributeConfig{Enabled: true},
+		CloudPlatform:         rac.ResourceAttributeConfig{Enabled: true},
+		CloudProvider:         rac.ResourceAttributeConfig{Enabled: true},
+		CloudRegion:           rac.ResourceAttributeConfig{Enabled: true},
+		HostID:                rac.ResourceAttributeConfig{Enabled: true},
+		HostImageID:           rac.ResourceAttributeConfig{Enabled: true},
+		HostName:              rac.ResourceAttributeConfig{Enabled: true},
+		HostType:              rac.ResourceAttributeConfig{Enabled: true},
+	},
+}
+
+var _ river.Defaulter = (*Config)(nil)
+
+// SetToDefault implements river.Defaulter.
+func (args *Config) SetToDefault() {
+	*args = DefaultArguments
+}
+
+func (args Config) Convert() map[string]interface{} {
+	return map[string]interface{}{
+		"tags":                append([]string{}, args.Tags...),
+		"resource_attributes": args.ResourceAttributes.Convert(),
+	}
+}
+
+// ResourceAttributesConfig provides config to enable and disable resource attributes.
+type ResourceAttributesConfig struct {
+	CloudAccountID        rac.ResourceAttributeConfig `river:"cloud.account.id,block,optional"`
+	CloudAvailabilityZone rac.ResourceAttributeConfig `river:"cloud.availability_zone,block,optional"`
+	CloudPlatform         rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"`
+	CloudProvider         rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"`
+	CloudRegion           rac.ResourceAttributeConfig `river:"cloud.region,block,optional"`
+	HostID                rac.ResourceAttributeConfig `river:"host.id,block,optional"`
+	HostImageID           rac.ResourceAttributeConfig `river:"host.image.id,block,optional"`
+	HostName              rac.ResourceAttributeConfig `river:"host.name,block,optional"`
+	HostType              rac.ResourceAttributeConfig `river:"host.type,block,optional"`
+}
+
+func (r ResourceAttributesConfig) Convert() map[string]interface{} {
+	return map[string]interface{}{
+		"cloud.account.id":        r.CloudAccountID.Convert(),
+		"cloud.availability_zone": r.CloudAvailabilityZone.Convert(),
+		"cloud.platform":          r.CloudPlatform.Convert(),
+		"cloud.provider":          r.CloudProvider.Convert(),
+		"cloud.region":            r.CloudRegion.Convert(),
+		"host.id":                 r.HostID.Convert(),
+		"host.image.id":           r.HostImageID.Convert(),
+		"host.name":               r.HostName.Convert(),
+		"host.type":               r.HostType.Convert(),
+	}
+}
diff --git a/component/otelcol/processor/resourcedetection/internal/aws/ecs/config.go b/component/otelcol/processor/resourcedetection/internal/aws/ecs/config.go
@@ -0,0 +1,86 @@
+package ecs
+
+import (
+	rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config"
+	"github.com/grafana/river"
+)
+
+const Name = "ecs"
+
+type Config struct {
+	ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block,optional"`
+}
+
+// DefaultArguments holds default settings for Config.
+var DefaultArguments = Config{
+	ResourceAttributes: ResourceAttributesConfig{
+		AwsEcsClusterArn:      rac.ResourceAttributeConfig{Enabled: true},
+		AwsEcsLaunchtype:      rac.ResourceAttributeConfig{Enabled: true},
+		AwsEcsTaskArn:         rac.ResourceAttributeConfig{Enabled: true},
+		AwsEcsTaskFamily:      rac.ResourceAttributeConfig{Enabled: true},
+		AwsEcsTaskRevision:    rac.ResourceAttributeConfig{Enabled: true},
+		AwsLogGroupArns:       rac.ResourceAttributeConfig{Enabled: true},
+		AwsLogGroupNames:      rac.ResourceAttributeConfig{Enabled: true},
+		AwsLogStreamArns:      rac.ResourceAttributeConfig{Enabled: true},
+		AwsLogStreamNames:     rac.ResourceAttributeConfig{Enabled: true},
+		CloudAccountID:        rac.ResourceAttributeConfig{Enabled: true},
+		CloudAvailabilityZone: rac.ResourceAttributeConfig{Enabled: true},
+		CloudPlatform:         rac.ResourceAttributeConfig{Enabled: true},
+		CloudProvider:         rac.ResourceAttributeConfig{Enabled: true},
+		CloudRegion:           rac.ResourceAttributeConfig{Enabled: true},
+	},
+}
+
+var _ river.Defaulter = (*Config)(nil)
+
+// SetToDefault implements river.Defaulter.
+func (args *Config) SetToDefault() {
+	*args = DefaultArguments
+}
+
+func (args *Config) Convert() map[string]interface{} {
+	if args == nil {
+		return nil
+	}
+
+	return map[string]interface{}{
+		"resource_attributes": args.ResourceAttributes.Convert(),
+	}
+}
+
+// ResourceAttributesConfig provides config for ecs resource attributes.
+type ResourceAttributesConfig struct {
+	AwsEcsClusterArn      rac.ResourceAttributeConfig `river:"aws.ecs.cluster.arn,block,optional"`
+	AwsEcsLaunchtype      rac.ResourceAttributeConfig `river:"aws.ecs.launchtype,block,optional"`
+	AwsEcsTaskArn         rac.ResourceAttributeConfig `river:"aws.ecs.task.arn,block,optional"`
+	AwsEcsTaskFamily      rac.ResourceAttributeConfig `river:"aws.ecs.task.family,block,optional"`
+	AwsEcsTaskRevision    rac.ResourceAttributeConfig `river:"aws.ecs.task.revision,block,optional"`
+	AwsLogGroupArns       rac.ResourceAttributeConfig `river:"aws.log.group.arns,block,optional"`
+	AwsLogGroupNames      rac.ResourceAttributeConfig `river:"aws.log.group.names,block,optional"`
+	AwsLogStreamArns      rac.ResourceAttributeConfig `river:"aws.log.stream.arns,block,optional"`
+	AwsLogStreamNames     rac.ResourceAttributeConfig `river:"aws.log.stream.names,block,optional"`
+	CloudAccountID        rac.ResourceAttributeConfig `river:"cloud.account.id,block,optional"`
+	CloudAvailabilityZone rac.ResourceAttributeConfig `river:"cloud.availability_zone,block,optional"`
+	CloudPlatform         rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"`
+	CloudProvider         rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"`
+	CloudRegion           rac.ResourceAttributeConfig `river:"cloud.region,block,optional"`
+}
+
+func (r ResourceAttributesConfig) Convert() map[string]interface{} {
+	return map[string]interface{}{
+		"aws.ecs.cluster.arn":     r.AwsEcsClusterArn.Convert(),
+		"aws.ecs.launchtype":      r.AwsEcsLaunchtype.Convert(),
+		"aws.ecs.task.arn":        r.AwsEcsTaskArn.Convert(),
+		"aws.ecs.task.family":     r.AwsEcsTaskFamily.Convert(),
+		"aws.ecs.task.revision":   r.AwsEcsTaskRevision.Convert(),
+		"aws.log.group.arns":      r.AwsLogGroupArns.Convert(),
+		"aws.log.group.names":     r.AwsLogGroupNames.Convert(),
+		"aws.log.stream.arns":     r.AwsLogStreamArns.Convert(),
+		"aws.log.stream.names":    r.AwsLogStreamNames.Convert(),
+		"cloud.account.id":        r.CloudAccountID.Convert(),
+		"cloud.availability_zone": r.CloudAvailabilityZone.Convert(),
+		"cloud.platform":          r.CloudPlatform.Convert(),
+		"cloud.provider":          r.CloudProvider.Convert(),
+		"cloud.region":            r.CloudRegion.Convert(),
+	}
+}
diff --git a/component/otelcol/processor/resourcedetection/internal/aws/eks/config.go b/component/otelcol/processor/resourcedetection/internal/aws/eks/config.go
@@ -0,0 +1,46 @@
+package eks
+
+import (
+	rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config"
+	"github.com/grafana/river"
+)
+
+const Name = "eks"
+
+type Config struct {
+	ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block,optional"`
+}
+
+// DefaultArguments holds default settings for Config.
+var DefaultArguments = Config{
+	ResourceAttributes: ResourceAttributesConfig{
+		CloudPlatform: rac.ResourceAttributeConfig{Enabled: true},
+		CloudProvider: rac.ResourceAttributeConfig{Enabled: true},
+	},
+}
+
+var _ river.Defaulter = (*Config)(nil)
+
+// SetToDefault implements river.Defaulter.
+func (args *Config) SetToDefault() {
+	*args = DefaultArguments
+}
+
+func (args Config) Convert() map[string]interface{} {
+	return map[string]interface{}{
+		"resource_attributes": args.ResourceAttributes.Convert(),
+	}
+}
+
+// ResourceAttributesConfig provides config for eks resource attributes.
+type ResourceAttributesConfig struct {
+	CloudPlatform rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"`
+	CloudProvider rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"`
+}
+
+func (r ResourceAttributesConfig) Convert() map[string]interface{} {
+	return map[string]interface{}{
+		"cloud.platform": r.CloudPlatform.Convert(),
+		"cloud.provider": r.CloudProvider.Convert(),
+	}
+}
diff --git a/component/otelcol/processor/resourcedetection/internal/aws/elasticbeanstalk/config.go b/component/otelcol/processor/resourcedetection/internal/aws/elasticbeanstalk/config.go
@@ -0,0 +1,55 @@
+package elasticbeanstalk
+
+import (
+	rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config"
+	"github.com/grafana/river"
+)
+
+const Name = "elasticbeanstalk"
+
+type Config struct {
+	ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block,optional"`
+}
+
+// DefaultArguments holds default settings for Config.
+var DefaultArguments = Config{
+	ResourceAttributes: ResourceAttributesConfig{
+		CloudPlatform:         rac.ResourceAttributeConfig{Enabled: true},
+		CloudProvider:         rac.ResourceAttributeConfig{Enabled: true},
+		DeploymentEnvironment: rac.ResourceAttributeConfig{Enabled: true},
+		ServiceInstanceID:     rac.ResourceAttributeConfig{Enabled: true},
+		ServiceVersion:        rac.ResourceAttributeConfig{Enabled: true},
+	},
+}
+
+var _ river.Defaulter = (*Config)(nil)
+
+// SetToDefault implements river.Defaulter.
+func (args *Config) SetToDefault() {
+	*args = DefaultArguments
+}
+
+func (args Config) Convert() map[string]interface{} {
+	return map[string]interface{}{
+		"resource_attributes": args.ResourceAttributes.Convert(),
+	}
+}
+
+// ResourceAttributesConfig provides config for elastic_beanstalk resource attributes.
+type ResourceAttributesConfig struct {
+	CloudPlatform         rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"`
+	CloudProvider         rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"`
+	DeploymentEnvironment rac.ResourceAttributeConfig `river:"deployment.environment,block,optional"`
+	ServiceInstanceID     rac.ResourceAttributeConfig `river:"service.instance.id,block,optional"`
+	ServiceVersion        rac.ResourceAttributeConfig `river:"service.version,block,optional"`
+}
+
+func (r ResourceAttributesConfig) Convert() map[string]interface{} {
+	return map[string]interface{}{
+		"cloud.platform":         r.CloudPlatform.Convert(),
+		"cloud.provider":         r.CloudProvider.Convert(),
+		"deployment.environment": r.DeploymentEnvironment.Convert(),
+		"service.instance.id":    r.ServiceInstanceID.Convert(),
+		"service.version":        r.ServiceVersion.Convert(),
+	}
+}
diff --git a/component/otelcol/processor/resourcedetection/internal/aws/lambda/config.go b/component/otelcol/processor/resourcedetection/internal/aws/lambda/config.go
@@ -0,0 +1,67 @@
+package lambda
+
+import (
+	rac "github.com/grafana/agent/component/otelcol/processor/resourcedetection/internal/resource_attribute_config"
+	"github.com/grafana/river"
+)
+
+const Name = "lambda"
+
+type Config struct {
+	ResourceAttributes ResourceAttributesConfig `river:"resource_attributes,block,optional"`
+}
+
+// DefaultArguments holds default settings for Config.
+var DefaultArguments = Config{
+	ResourceAttributes: ResourceAttributesConfig{
+		AwsLogGroupNames:  rac.ResourceAttributeConfig{Enabled: true},
+		AwsLogStreamNames: rac.ResourceAttributeConfig{Enabled: true},
+		CloudPlatform:     rac.ResourceAttributeConfig{Enabled: true},
+		CloudProvider:     rac.ResourceAttributeConfig{Enabled: true},
+		CloudRegion:       rac.ResourceAttributeConfig{Enabled: true},
+		FaasInstance:      rac.ResourceAttributeConfig{Enabled: true},
+		FaasMaxMemory:     rac.ResourceAttributeConfig{Enabled: true},
+		FaasName:          rac.ResourceAttributeConfig{Enabled: true},
+		FaasVersion:       rac.ResourceAttributeConfig{Enabled: true},
+	},
+}
+
+var _ river.Defaulter = (*Config)(nil)
+
+// SetToDefault implements river.Defaulter.
+func (args *Config) SetToDefault() {
+	*args = DefaultArguments
+}
+
+func (args Config) Convert() map[string]interface{} {
+	return map[string]interface{}{
+		"resource_attributes": args.ResourceAttributes.Convert(),
+	}
+}
+
+// ResourceAttributesConfig provides config for lambda resource attributes.
+type ResourceAttributesConfig struct {
+	AwsLogGroupNames  rac.ResourceAttributeConfig `river:"aws.log.group.names,block,optional"`
+	AwsLogStreamNames rac.ResourceAttributeConfig `river:"aws.log.stream.names,block,optional"`
+	CloudPlatform     rac.ResourceAttributeConfig `river:"cloud.platform,block,optional"`
+	CloudProvider     rac.ResourceAttributeConfig `river:"cloud.provider,block,optional"`
+	CloudRegion       rac.ResourceAttributeConfig `river:"cloud.region,block,optional"`
+	FaasInstance      rac.ResourceAttributeConfig `river:"faas.instance,block,optional"`
+	FaasMaxMemory     rac.ResourceAttributeConfig `river:"faas.max_memory,block,optional"`
+	FaasName          rac.ResourceAttributeConfig `river:"faas.name,block,optional"`
+	FaasVersion       rac.ResourceAttributeConfig `river:"faas.version,block,optional"`
+}
+
+func (r ResourceAttributesConfig) Convert() map[string]interface{} {
+	return map[string]interface{}{
+		"aws.log.group.names":  r.AwsLogGroupNames.Convert(),
+		"aws.log.stream.names": r.AwsLogStreamNames.Convert(),
+		"cloud.platform":       r.CloudPlatform.Convert(),
+		"cloud.provider":       r.CloudProvider.Convert(),
+		"cloud.region":         r.CloudRegion.Convert(),
+		"faas.instance":        r.FaasInstance.Convert(),
+		"faas.max_memory":      r.FaasMaxMemory.Convert(),
+		"faas.name":            r.FaasName.Convert(),
+		"faas.version":         r.FaasVersion.Convert(),
+	}
+}