Skip to content

Commit

Permalink
Improved ECS attribute and origin translation in awsxrayexporter (#1428)
Browse files Browse the repository at this point in the history 
* initial commit of ecs translation

* reverted trace ID change

* added unit tests

* addressed comments and unit test issues

* fixed breaking API changes
  • Loading branch information
@willarmiros
willarmiros authored Nov 5, 2020
1 parent 507ad41 commit 86af421
Show file tree
Hide file tree
Showing 12 changed files with 1,776 additions and 36 deletions.
96 changes: 89 additions & 7 deletions exporter/awsxrayexporter/translator/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package translator

import (
"bytes"
"strconv"

"github.com/aws/aws-sdk-go/aws"
Expand All @@ -24,9 +25,21 @@ import (
"github.com/open-telemetry/opentelemetry-collector-contrib/internal/awsxray"
)

const (
attributeInfrastructureService = "cloud.infrastructure_service"
awsEcsClusterArn = "aws.ecs.cluster.arn"
awsEcsContainerArn = "aws.ecs.container.arn"
awsEcsTaskArn = "aws.ecs.task.arn"
awsEcsTaskFamily = "aws.ecs.task.family"
awsEcsLaunchType = "aws.ecs.launchtype"
awsLogGroupNames = "aws.log.group.names"
awsLogGroupArns = "aws.log.group.arns"
)

func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]string, *awsxray.AWSData) {
var (
cloud string
service string
account string
zone string
hostID string
Expand All @@ -49,6 +62,14 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
containerID string
clusterName string
podUID string
clusterArn string
containerArn string
taskArn string
taskFamily string
launchType string
logGroups pdata.AnyValueArray
logGroupArns pdata.AnyValueArray
cwl []awsxray.LogGroupMetadata
ec2 *awsxray.EC2Metadata
ecs *awsxray.ECSMetadata
ebs *awsxray.BeanstalkMetadata
Expand All @@ -61,6 +82,8 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
switch key {
case semconventions.AttributeCloudProvider:
cloud = value.StringVal()
case attributeInfrastructureService:
service = value.StringVal()
case semconventions.AttributeCloudAccount:
account = value.StringVal()
case semconventions.AttributeCloudZone:
Expand Down Expand Up @@ -95,6 +118,20 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
containerID = value.StringVal()
case semconventions.AttributeK8sCluster:
clusterName = value.StringVal()
case awsEcsClusterArn:
clusterArn = value.StringVal()
case awsEcsContainerArn:
containerArn = value.StringVal()
case awsEcsTaskArn:
taskArn = value.StringVal()
case awsEcsTaskFamily:
taskFamily = value.StringVal()
case awsEcsLaunchType:
launchType = value.StringVal()
case awsLogGroupNames:
logGroups = value.ArrayVal()
case awsLogGroupArns:
logGroupArns = value.ArrayVal()
}
})
}
Expand Down Expand Up @@ -125,25 +162,32 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
filtered[key] = value
}
}
if cloud != "aws" && cloud != "" {
if cloud != semconventions.AttributeCloudProviderAWS && cloud != "" {
return filtered, nil // not AWS so return nil
}
// progress from least specific to most specific origin so most specific ends up as origin
// as per X-Ray docs
if hostID != "" {

if service == "EC2" || hostID != "" {
ec2 = &awsxray.EC2Metadata{
InstanceID: awsxray.String(hostID),
AvailabilityZone: awsxray.String(zone),
InstanceSize: awsxray.String(hostType),
AmiID: awsxray.String(amiID),
}
}
if container != "" {
if service == "ECS" || container != "" {
ecs = &awsxray.ECSMetadata{
ContainerName: awsxray.String(container),
ContainerID: awsxray.String(containerID),
ContainerName: awsxray.String(container),
ContainerID: awsxray.String(containerID),
AvailabilityZone: awsxray.String(zone),
ContainerArn: awsxray.String(containerArn),
ClusterArn: awsxray.String(clusterArn),
TaskArn: awsxray.String(taskArn),
TaskFamily: awsxray.String(taskFamily),
LaunchType: awsxray.String(launchType),
}
}

// TODO(willarmiros): Add infrastructure_service checks once their resource detectors are implemented
if deployID != "" {
deployNum, err := strconv.ParseInt(deployID, 10, 64)
if err != nil {
Expand All @@ -163,6 +207,14 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
}
}

// Since we must couple log group ARNs and Log Group Names in the same CWLogs object, we first try to derive the
// names from the ARN, then fall back to just recording the names
if logGroupArns != (pdata.AnyValueArray{}) && logGroupArns.Len() > 0 {
cwl = getLogGroupMetadata(logGroupArns, true)
} else if logGroups != (pdata.AnyValueArray{}) && logGroups.Len() > 0 {
cwl = getLogGroupMetadata(logGroups, false)
}

if sdkName != "" && sdkLanguage != "" {
// Convention for SDK name for xray SDK information is e.g., `X-Ray SDK for Java`, `X-Ray for Go`.
// We fill in with e.g, `opentelemetry for java` by using the conventions
Expand All @@ -180,6 +232,7 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
awsData := &awsxray.AWSData{
AccountID: awsxray.String(account),
Beanstalk: ebs,
CWLogs: cwl,
ECS: ecs,
EC2: ec2,
EKS: eks,
Expand All @@ -192,3 +245,32 @@ func makeAws(attributes map[string]string, resource pdata.Resource) (map[string]
}
return filtered, awsData
}

// Given an array of log group ARNs, create a corresponding amount of LogGroupMetadata objects with log_group and arn
// populated, or given an array of just log group names, create the LogGroupMetadata objects with arn omitted
func getLogGroupMetadata(logGroups pdata.AnyValueArray, isArn bool) []awsxray.LogGroupMetadata {
var lgm []awsxray.LogGroupMetadata
for i := 0; i < logGroups.Len(); i++ {
if isArn {
lgm = append(lgm, awsxray.LogGroupMetadata{
Arn: awsxray.String(logGroups.At(i).StringVal()),
LogGroup: awsxray.String(parseLogGroup(logGroups.At(i).StringVal())),
})
} else {
lgm = append(lgm, awsxray.LogGroupMetadata{
LogGroup: awsxray.String(logGroups.At(i).StringVal()),
})
}
}

return lgm
}

func parseLogGroup(arn string) string {
i := bytes.LastIndexByte([]byte(arn), byte(':'))
if i != -1 {
return arn[i+1:]
}

return arn
}
84 changes: 81 additions & 3 deletions exporter/awsxrayexporter/translator/aws_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func TestAwsFromEc2Resource(t *testing.T) {
resource.InitEmpty()
attrs := pdata.NewAttributeMap()
attrs.InsertString(semconventions.AttributeCloudProvider, semconventions.AttributeCloudProviderAWS)
attrs.InsertString(attributeInfrastructureService, "EC2")
attrs.InsertString(semconventions.AttributeCloudAccount, "123456789")
attrs.InsertString(semconventions.AttributeCloudZone, "us-east-1c")
attrs.InsertString(semconventions.AttributeHostID, instanceID)
Expand Down Expand Up @@ -63,12 +64,19 @@ func TestAwsFromEcsResource(t *testing.T) {
instanceID := "i-00f7c0bcb26da2a99"
containerName := "signup_aggregator-x82ufje83"
containerID := "0123456789A"
az := "us-east-1c"
launchType := "fargate"
family := "family"
taskArn := "arn:aws:ecs:us-west-2:123456789123:task/123"
clusterArn := "arn:aws:ecs:us-west-2:123456789123:cluster/my-cluster"
containerArn := "arn:aws:ecs:us-west-2:123456789123:container-instance/123"
resource := pdata.NewResource()
resource.InitEmpty()
attrs := pdata.NewAttributeMap()
attrs.InsertString(semconventions.AttributeCloudProvider, semconventions.AttributeCloudProviderAWS)
attrs.InsertString(attributeInfrastructureService, "ECS")
attrs.InsertString(semconventions.AttributeCloudAccount, "123456789")
attrs.InsertString(semconventions.AttributeCloudZone, "us-east-1c")
attrs.InsertString(semconventions.AttributeCloudZone, az)
attrs.InsertString(semconventions.AttributeContainerImage, "otel/signupaggregator")
attrs.InsertString(semconventions.AttributeContainerTag, "v1")
attrs.InsertString(semconventions.AttributeK8sCluster, "production")
Expand All @@ -78,7 +86,13 @@ func TestAwsFromEcsResource(t *testing.T) {
attrs.InsertString(semconventions.AttributeContainerName, containerName)
attrs.InsertString(semconventions.AttributeContainerID, containerID)
attrs.InsertString(semconventions.AttributeHostID, instanceID)
attrs.InsertString(awsEcsClusterArn, clusterArn)
attrs.InsertString(awsEcsContainerArn, containerArn)
attrs.InsertString(awsEcsTaskArn, taskArn)
attrs.InsertString(awsEcsTaskFamily, family)
attrs.InsertString(awsEcsLaunchType, launchType)
attrs.InsertString(semconventions.AttributeHostType, "m5.xlarge")

attrs.CopyTo(resource.Attributes())

attributes := make(map[string]string)
Expand All @@ -92,8 +106,14 @@ func TestAwsFromEcsResource(t *testing.T) {
assert.Nil(t, awsData.Beanstalk)
assert.NotNil(t, awsData.EKS)
assert.Equal(t, &awsxray.ECSMetadata{
ContainerName: aws.String(containerName),
ContainerID: aws.String(containerID),
ContainerName: aws.String(containerName),
ContainerID: aws.String(containerID),
AvailabilityZone: aws.String(az),
ClusterArn: aws.String(clusterArn),
ContainerArn: aws.String(containerArn),
TaskArn: aws.String(taskArn),
TaskFamily: aws.String(family),
LaunchType: aws.String(launchType),
}, awsData.ECS)
}

Expand Down Expand Up @@ -343,3 +363,61 @@ func TestCustomSDK(t *testing.T) {
assert.Equal(t, "opentracing for java", *awsData.XRay.SDK)
assert.Equal(t, "2.0.3", *awsData.XRay.SDKVersion)
}

func TestLogGroups(t *testing.T) {
cwl1 := awsxray.LogGroupMetadata{
LogGroup: awsxray.String("group1"),
}
cwl2 := awsxray.LogGroupMetadata{
LogGroup: awsxray.String("group2"),
}

attributes := make(map[string]string)
resource := pdata.NewResource()
resource.InitEmpty()
lg := pdata.NewAttributeValueArray()
ava := lg.ArrayVal()
ava.Append(pdata.NewAttributeValueString("group1"))
ava.Append(pdata.NewAttributeValueString("group2"))

resource.Attributes().Insert(awsLogGroupNames, lg)

filtered, awsData := makeAws(attributes, resource)

assert.NotNil(t, filtered)
assert.NotNil(t, awsData)
assert.Equal(t, 2, len(awsData.CWLogs))
assert.Contains(t, awsData.CWLogs, cwl1)
assert.Contains(t, awsData.CWLogs, cwl2)
}

func TestLogGroupsFromArns(t *testing.T) {
group1 := "arn:aws:logs:us-east-1:123456789123:log-group:group1"
cwl1 := awsxray.LogGroupMetadata{
LogGroup: awsxray.String("group1"),
Arn: awsxray.String(group1),
}
group2 := "arn:aws:logs:us-east-1:123456789123:log-group:group2"
cwl2 := awsxray.LogGroupMetadata{
LogGroup: awsxray.String("group2"),
Arn: awsxray.String(group2),
}

attributes := make(map[string]string)
resource := pdata.NewResource()
resource.InitEmpty()
lga := pdata.NewAttributeValueArray()
ava := lga.ArrayVal()
ava.Append(pdata.NewAttributeValueString(group1))
ava.Append(pdata.NewAttributeValueString(group2))

resource.Attributes().Insert(awsLogGroupArns, lga)

filtered, awsData := makeAws(attributes, resource)

assert.NotNil(t, filtered)
assert.NotNil(t, awsData)
assert.Equal(t, 2, len(awsData.CWLogs))
assert.Contains(t, awsData.CWLogs, cwl1)
assert.Contains(t, awsData.CWLogs, cwl2)
}
47 changes: 42 additions & 5 deletions exporter/awsxrayexporter/translator/segment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,12 @@ import (

// AWS X-Ray acceptable values for origin field.
const (
OriginEC2 = "AWS::EC2::Instance"
OriginECS = "AWS::ECS::Container"
OriginEB = "AWS::ElasticBeanstalk::Environment"
OriginEKS = "AWS::EKS::Container"
OriginEC2 = "AWS::EC2::Instance"
OriginECS = "AWS::ECS::Container"
OriginECSEC2 = "AWS::ECS::EC2"
OriginECSFargate = "AWS::ECS::Fargate"
OriginEB = "AWS::ElasticBeanstalk::Environment"
OriginEKS = "AWS::EKS::Container"
)

var (
Expand Down Expand Up @@ -230,6 +232,37 @@ func determineAwsOrigin(resource pdata.Resource) string {
return ""
}
}

// TODO(willarmiros): Only use infrastructure_service for origin resolution once detectors for all AWS environments are
// implemented for robustness
if is, present := resource.Attributes().Get("cloud.infrastructure_service"); present {
switch is.StringVal() {
case "EKS":
return OriginEKS
case "ElasticBeanstalk":
return OriginEB
case "ECS":
lt, present := resource.Attributes().Get("aws.ecs.launchtype")
if !present {
return OriginECS
}
switch lt.StringVal() {
case "ec2":
return OriginECSEC2
case "fargate":
return OriginECSFargate
default:
return OriginECS
}
case "EC2":
return OriginEC2

// If infrastructure_service is defined with a non-AWS value, we should not assign it an AWS origin
default:
return ""
}
}

// EKS > EB > ECS > EC2
_, eks := resource.Attributes().Get(semconventions.AttributeK8sCluster)
if eks {
Expand All @@ -243,7 +276,11 @@ func determineAwsOrigin(resource pdata.Resource) string {
if ecs {
return OriginECS
}
return OriginEC2
_, ec2 := resource.Attributes().Get(semconventions.AttributeHostID)
if ec2 {
return OriginEC2
}
return ""
}

// convertToAmazonTraceID converts a trace ID to the Amazon format.
Expand Down
Loading

0 comments on commit 86af421

Please sign in to comment.