From e3a3c2694bbbd734aed12a710558a960acc15808 Mon Sep 17 00:00:00 2001 From: Dmitry Kolomiets Date: Fri, 22 Apr 2022 10:51:06 +0300 Subject: [PATCH 1/3] WIP - trying to run performance tests against deployed QS --- README.md | 9 +++ tests/performance.yaml | 132 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 tests/performance.yaml diff --git a/README.md b/README.md index d081ea4..098b0c9 100644 --- a/README.md +++ b/README.md @@ -106,3 +106,12 @@ docker run jaeger-taskcat docker run jaeger-taskcat test -h docker run jaeger-taskcat test run -h ``` + + + + + + + + +docker run -e TEST_CONFIG_FILE=/config/performance.yaml -e ELASTICSEARCH_PROVIDER=none -v $(pwd):/config -it jkandasa/jaeger-performance-test:3.6 \ No newline at end of file diff --git a/tests/performance.yaml b/tests/performance.yaml new file mode 100644 index 0000000..d73a78d --- /dev/null +++ b/tests/performance.yaml @@ -0,0 +1,132 @@ +# +# Copyright 2018 The Jaeger Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# + +# Jaeger tracing performance test suite configurations. +# Some of these values below will be used in final report, +# also used on OpenShift/Kubernetes environment to install +# Jaeger services and database in Jenkins environment + +# what are all the tests need to be executed +testsToRun: 'smoke,performance' + +elasticsearchProvider: none + +# performance test data, two type of performance test can be executed +# data format: [type,data] +# 1. quick run example, 'quick,50', data: delay between spans in milliseconds +# 2. long run example, 'long,600', data: test duration in seconds +# performanceTestData: 'quick,165' + +nodeCountSpansReporter: 1 + +# tracers count, number of tracers shold send spans +numberOfTracers: 100 + +# spans count for per tracer. +numberOfSpans: 4000 + +# Is your test is running on OpenShift environment? +# if you run your test on OpenShift environment, you should use Jenkins +runningOnOpenshift: false + +useInternalReporter: true +reportSpansDuration: 10m + + +# maximum spans/data limit on Jaeger query test +queryLimit: 20000 + +# Number of times the query to be executed repetitively. +# To get average time +querySamples: 5 + +# How long once repeat the query test? '-1' indicates only at final +# values in seconds +queryInterval: -1 + +# where to send spans, options: http, udp +# 'http' will send spans to 'collector' +# 'udp' will send spans to 'agent' +sender: http + +# storage type, options: elasticsearch, cassandra +# storageType: elasticsearch + +# Get spans on count from storage at the end of test +# options: storage, jaeger-query +# Type 'storage' will be used direct storage query api +# Type 'jaeger-query' will be used jaeger query api +spansCountFrom: jaeger-query + +# storage hostname and port number +storageHost: localhost +storagePort: 9200 + +# storage keyspace, used in cassandra storage +# storageKeyspace: keyspace + +# jaeger query hostname and port number +jaegerQueryHost: localhost +jaegerQueryPort: 16686 + +# jaeger collector hostname and port number +jaegerCollectorHost: localhost +jaegerCollectorPort: 14268 + +# jaeger agent hostname and port number +jaegerAgentHost: localhost +jaegerAgentPort: 6831 + +# jaeger client java library configurations +# jaegerFlushInterval: 100 +# jaegerMaxPocketSize: 0 +# jaegerMaxQueueSize: 10000 + +# if the test is running on OpenShift environment, +# where you want to store all the collected logs +logsDirectory: /config/ + +# Number of collector pods to be deployed +# will be used on OpenShift environment +# collectorPods: 1 +collectorQueueSize: 2000 +collectorNumWorkers: 50 + +# jaeger agent configurations +# will be used on OpenShift environment +jaegerAgentQueueSize: 1000 +jaegerAgentWorkers: 10 + +# jaeger query static files location for jaeger query docker image +# will be used on OpenShift environment +jaegerQueryStaticFiles: '' + +# elasticsearch configurations +# will be used on OpenShift environment +esMemory: 1Gi +collectorEsBulkSize: 5000000 +collectorEsBulkWorkers: 1 +collectorEsBulkFlushInterval: 200ms + +# docker image details +# will be used on OpenShift environment +# imageJaegerAgent: +# imageJaegerCollector: +# imageJaegerQuery: +# imageStorage: +# storageImageInSecure: false + +# docker image details for performance test, +# this image includes: smoketest as well +# imagePerformanceTest: From ff19a13e65d733ccc93e8b46242e5c872e1dca8c Mon Sep 17 00:00:00 2001 From: Dmitry Kolomiets Date: Fri, 6 May 2022 09:11:30 +0100 Subject: [PATCH 2/3] Add support for OpenTelemetry Collector in front of Jaeger --- .taskcat.yml | 1 + .../jaeger-entry-existing-vpc.template.yaml | 34 ++- templates/jaeger-entry-new-vpc.template.yaml | 34 ++- templates/workload/jaeger-otel-template.yaml | 229 ++++++++++++++++++ templates/workload/jaeger-template.yaml | 33 ++- 5 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 templates/workload/jaeger-otel-template.yaml diff --git a/.taskcat.yml b/.taskcat.yml index 40402ed..e7f8105 100644 --- a/.taskcat.yml +++ b/.taskcat.yml @@ -26,6 +26,7 @@ tests: PrivateDNS: CreateNew ExportMetrics: 'Yes' EnableContainerInsights: 'Yes' + OpenTelemetryCollector: 'Yes' regions: - us-east-1 diff --git a/templates/jaeger-entry-existing-vpc.template.yaml b/templates/jaeger-entry-existing-vpc.template.yaml index 9a45e2d..ee490a9 100644 --- a/templates/jaeger-entry-existing-vpc.template.yaml +++ b/templates/jaeger-entry-existing-vpc.template.yaml @@ -49,6 +49,12 @@ Metadata: - JaegerVersion - StorageBackend - InternalLoadBalancer + - Label: + default: OpenTelemetry Collector configuration + Parameters: + - OpenTelemetryCollector + - OpenTelemetryCollectorImage + - OpenTelemetryCollectorVersion - Label: default: "[Only for Elasticsearch backend] Elasticsearch configuration" Parameters: @@ -112,7 +118,14 @@ Metadata: default: Jaeger storage backend InternalLoadBalancer: default: Use internal load balancer - # Elasticsearch parameters + # OpenTelemetry Collector configuration + OpenTelemetryCollector: + default: OpenTelemetry Collector + OpenTelemetryCollectorImage: + default: OpenTelemetry Collector image + OpenTelemetryCollectorVersion: + default: OpenTelemetry Collector version + # Elasticsearch configuration ESEngineVersion: default: Elasticsearch version ESInstanceCount: @@ -220,6 +233,21 @@ Parameters: Default: 'No' Description: Choose "Yes" to deploy Jaeger load balancer in private VPC subnets. +# OpenTelemetry Collector parameters + OpenTelemetryCollector: + Type: String + AllowedValues: [ 'Yes', 'No' ] + Default: 'No' + Description: Choose "Yes" to deploy OpenTelemetry Collector in fron of Jaeger. + OpenTelemetryCollectorImage: + Type: String + Default: otel/opentelemetry-collector + Description: Docker image for the OpenTelemetry Collector + OpenTelemetryCollectorVersion: + Type: String + Default: '0.50.0' + Description: Version of the OpenTelemetry Collector. + # Elasticsearch parameters ESEngineVersion: Type: String @@ -378,6 +406,10 @@ Resources: JaegerVersion: !Ref JaegerVersion StorageBackend: !Ref StorageBackend InternalLoadBalancer: !Ref InternalLoadBalancer + # OpenTelemetry Collector parameters + OpenTelemetryCollector: !Ref OpenTelemetryCollector + OpenTelemetryCollectorImage: !Ref OpenTelemetryCollectorImage + OpenTelemetryCollectorVersion: !Ref OpenTelemetryCollectorVersion # Elasticsearch parameters ESEngineVersion: !Ref ESEngineVersion ESInstanceCount: !Ref ESInstanceCount diff --git a/templates/jaeger-entry-new-vpc.template.yaml b/templates/jaeger-entry-new-vpc.template.yaml index 0d791dc..8c908e1 100644 --- a/templates/jaeger-entry-new-vpc.template.yaml +++ b/templates/jaeger-entry-new-vpc.template.yaml @@ -50,6 +50,12 @@ Metadata: - JaegerVersion - StorageBackend - InternalLoadBalancer + - Label: + default: OpenTelemetry Collector configuration + Parameters: + - OpenTelemetryCollector + - OpenTelemetryCollectorImage + - OpenTelemetryCollectorVersion - Label: default: "[Only for Elasticsearch backend] Elasticsearch configuration" Parameters: @@ -115,7 +121,14 @@ Metadata: default: Jaeger storage backend InternalLoadBalancer: default: Use internal load balancer - # Elasticsearch parameters + # OpenTelemetry Collector configuration + OpenTelemetryCollector: + default: OpenTelemetry Collector + OpenTelemetryCollectorImage: + default: OpenTelemetry Collector image + OpenTelemetryCollectorVersion: + default: OpenTelemetry Collector version + # Elasticsearch configuration ESEngineVersion: default: Elasticsearch version ESInstanceCount: @@ -250,6 +263,21 @@ Parameters: Default: 'No' Description: Choose "Yes" to deploy Jaeger load balancer in private VPC subnets. +# OpenTelemetry Collector parameters + OpenTelemetryCollector: + Type: String + AllowedValues: [ 'Yes', 'No' ] + Default: 'No' + Description: Choose "Yes" to deploy OpenTelemetry Collector in fron of Jaeger. + OpenTelemetryCollectorImage: + Type: String + Default: otel/opentelemetry-collector + Description: Docker image for the OpenTelemetry Collector + OpenTelemetryCollectorVersion: + Type: String + Default: '0.50.0' + Description: Version of the OpenTelemetry Collector. + # Elasticsearch parameters ESEngineVersion: Type: String @@ -437,6 +465,10 @@ Resources: JaegerVersion: !Ref JaegerVersion StorageBackend: !Ref StorageBackend InternalLoadBalancer: !Ref InternalLoadBalancer + # OpenTelemetry Collector parameters + OpenTelemetryCollector: !Ref OpenTelemetryCollector + OpenTelemetryCollectorImage: !Ref OpenTelemetryCollectorImage + OpenTelemetryCollectorVersion: !Ref OpenTelemetryCollectorVersion # Elasticsearch parameters ESEngineVersion: !Ref ESEngineVersion ESInstanceCount: !Ref ESInstanceCount diff --git a/templates/workload/jaeger-otel-template.yaml b/templates/workload/jaeger-otel-template.yaml new file mode 100644 index 0000000..49e9ff2 --- /dev/null +++ b/templates/workload/jaeger-otel-template.yaml @@ -0,0 +1,229 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: Deploys OpenTelemetry Collector (TODO) + +Parameters: +# Basic Configuration + EnvironmentName: + Type: String +# VPC parameters + VPCID: + Type: AWS::EC2::VPC::Id + Description: ID of your existing VPC (e.g., vpc-0343606e). + VPCCIDR: + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-8]))$ + Type: String + PrivateSubnet1ID: + Type: AWS::EC2::Subnet::Id + PrivateSubnet2ID: + Type: AWS::EC2::Subnet::Id + PrivateSubnet3ID: + Type: String + Default: '' +# Jaeger parameters + ClusterArn: + Type: String + LoadBalancerArn: + Type: String + JaegerEndpoint: + Type: String + OpenTelemetryCollectorImage: + Type: String + OpenTelemetryCollectorVersion: + Type: String + +Conditions: + Using3AvailabilityZones: !Not [!Equals [!Ref PrivateSubnet3ID, '']] + +Resources: + +# Configuration + ConfigurationParameter: + Type: AWS::SSM::Parameter + Properties: + Description: Jaeger OpenTelemetry Collector configuration + Name: !Sub "/quickstart/jaeger/${EnvironmentName}/otel-collector-config" + Type: String + Value: + Fn::Base64: + Fn::Sub: | + receivers: + otlp: + protocols: + grpc: + http: + exporters: + logging: + loglevel: debug + jaeger: + endpoint: ${JaegerEndpoint}:14250 + tls: + insecure: true + service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging, jaeger] + +# Target Groups + TargetGroup4317: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 4317 + Protocol: TCP + TargetType: ip + VpcId: !Ref VPCID + Tags: + - Key: Name + Value: !Sub jaeger-4317-${EnvironmentName} + +# Listeners + Listener4317: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - TargetGroupArn: + Ref: TargetGroup4317 + Type: forward + LoadBalancerArn: !Ref LoadBalancerArn + Port: 4317 + Protocol: TCP + +# Roles + TaskRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Version: "2012-10-17" + + ExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Version: "2012-10-17" + Policies: + - PolicyName: jaeger-otel-execution-policy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Effect: Allow + Resource: !GetAtt LogGroup.Arn + - Action: + - ssm:GetParameters + - ssm:GetParameter + Effect: Allow + Resource: !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter${ConfigurationParameter}" + +# Task Definition + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + RetentionInDays: 30 + + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub quickstart-jaeger-otel-${EnvironmentName} + Volumes: + - Name: otel-conf-volume + Host: {} + ContainerDefinitions: + - Name: jaeger-otel + Essential: true + Image: !Sub ${OpenTelemetryCollectorImage}:${OpenTelemetryCollectorVersion} + Command: + - --config=/otel/conf/jaeger-export.conf + DependsOn: + - Condition: COMPLETE + ContainerName: jaeger-otel-config + PortMappings: + - ContainerPort: 4317 # Default endpoint for OpenTelemetry receiver + Protocol: tcp + MountPoints: + - ContainerPath: /otel/conf + SourceVolume: otel-conf-volume + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: jaeger-otel + awslogs-region: !Ref AWS::Region + - Name: jaeger-otel-config + Image: bash + Essential: false + Command: + - -c + - echo $OTEL_CONFIG | base64 -d - | tee /otel/conf/jaeger-export.conf + Secrets: + - Name: OTEL_CONFIG + ValueFrom: !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter${ConfigurationParameter}" + MountPoints: + - ContainerPath: /otel/conf + SourceVolume: otel-conf-volume + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: jaeger-otel-config + awslogs-region: !Ref AWS::Region + Cpu: "1024" + ExecutionRoleArn: !GetAtt ExecutionRole.Arn + Memory: "2048" + NetworkMode: awsvpc + RequiresCompatibilities: + - FARGATE + TaskRoleArn: !GetAtt TaskRole.Arn + +# Service + Service: + Type: AWS::ECS::Service + Properties: + ServiceName: jaeger-otel + Cluster: !Ref ClusterArn + DesiredCount: 1 + HealthCheckGracePeriodSeconds: 30 + LaunchType: FARGATE + LoadBalancers: + - ContainerName: jaeger-otel + ContainerPort: 4317 + TargetGroupArn: !Ref TargetGroup4317 + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: DISABLED + SecurityGroups: + - !GetAtt ServiceSecurityGroup.GroupId + Subnets: + - !Ref PrivateSubnet1ID + - !Ref PrivateSubnet2ID + - !If [Using3AvailabilityZones, !Ref PrivateSubnet3ID, !Ref 'AWS::NoValue'] + TaskDefinition: !Ref TaskDefinition + + ServiceSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: quickstart-jaeger/jaeger/jaeger-otel/SecurityGroup + VpcId: !Ref VPCID + SecurityGroupEgress: + - CidrIp: 0.0.0.0/0 + Description: Allow all outbound traffic by default + IpProtocol: tcp + FromPort: 0 + ToPort: 65535 + SecurityGroupIngress: + - CidrIp: !Ref VPCCIDR + Description: Allow ECS Service access (port 4317) from within VPC + IpProtocol: tcp + FromPort: 4317 + ToPort: 4317 \ No newline at end of file diff --git a/templates/workload/jaeger-template.yaml b/templates/workload/jaeger-template.yaml index b98503b..d1c6b02 100644 --- a/templates/workload/jaeger-template.yaml +++ b/templates/workload/jaeger-template.yaml @@ -46,7 +46,13 @@ Parameters: InternalLoadBalancer: Type: String AllowedValues: [ 'Yes', 'No' ] - + OpenTelemetryCollector: + Type: String + AllowedValues: [ 'Yes', 'No' ] + OpenTelemetryCollectorImage: + Type: String + OpenTelemetryCollectorVersion: + Type: String # Elasticsearch parameters ESEngineVersion: Type: String @@ -114,6 +120,7 @@ Conditions: InternalLoadBalancer: !Equals [!Ref InternalLoadBalancer, 'Yes'] ExportMetrics: !Equals [!Ref ExportMetrics, 'Yes'] EnableContainerInsights: !Equals [!Ref EnableContainerInsights, 'Yes'] + OpenTelemetryCollector: !Equals [!Ref OpenTelemetryCollector, 'Yes'] InMemoryStorage: !Equals [!Ref StorageBackend, 'InMemory'] ElasticseachStorage: !Equals [!Ref StorageBackend, 'Elasticseach'] @@ -144,6 +151,30 @@ Resources: - Key: deletion_protection.enabled Value: "false" + OtelStack: + Type: AWS::CloudFormation::Stack + Condition: OpenTelemetryCollector + Properties: + TemplateURL: !Sub + - 'https://${S3Bucket}.s3.${S3Region}.${AWS::URLSuffix}/${QSS3KeyPrefix}templates/workload/jaeger-otel-template.yaml' + - S3Region: !If [UsingDefaultBucket, !Ref 'AWS::Region', !Ref QSS3BucketRegion] + S3Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] + Parameters: + # Basic Configuration + EnvironmentName: !Ref EnvironmentName + # VPC parameters + VPCID: !Ref VPCID + VPCCIDR: !Ref VPCCIDR + PrivateSubnet1ID: !Ref PrivateSubnet1ID + PrivateSubnet2ID: !Ref PrivateSubnet2ID + PrivateSubnet3ID: !Ref PrivateSubnet3ID + # Jaeger parameters + ClusterArn: !GetAtt Cluster.Arn + LoadBalancerArn: !Ref LoadBalancer + JaegerEndpoint: !If [CustomDomainName, !Sub '${SubdomainName}.${DomainName}', !GetAtt LoadBalancer.DNSName] + OpenTelemetryCollectorImage: !Ref OpenTelemetryCollectorImage + OpenTelemetryCollectorVersion: !Ref OpenTelemetryCollectorVersion + DNSStack: Type: AWS::CloudFormation::Stack Properties: From 8758d028c444eb758e7416d42a7d11e87bb834e4 Mon Sep 17 00:00:00 2001 From: Dmitry Kolomiets Date: Fri, 6 May 2022 09:15:23 +0100 Subject: [PATCH 3/3] Cleanup the readme --- README.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/README.md b/README.md index 5b3e824..9301bca 100644 --- a/README.md +++ b/README.md @@ -120,13 +120,4 @@ Run docker image without parameters to get the help output. Some examples: docker run jaeger-taskcat docker run jaeger-taskcat test -h docker run jaeger-taskcat test run -h -``` - - - - - - - - -docker run -e TEST_CONFIG_FILE=/config/performance.yaml -e ELASTICSEARCH_PROVIDER=none -v $(pwd):/config -it jkandasa/jaeger-performance-test:3.6 \ No newline at end of file +``` \ No newline at end of file