Added concurrent testing and release updates

.github/workflows/integrationTest.yml

@@ -21,6 +21,7 @@ on:
     branches:
       - master
 
+
   workflow_dispatch:
 
 concurrency:
@@ -135,12 +136,12 @@ jobs:
       - name: Upload to s3
         if: steps.cached_binaries.outputs.cache-hit != 'true'
         run: aws s3 cp build/bin s3://${S3_INTEGRATION_BUCKET}/integration-test/binary/${{ github.sha }} --recursive
-
   GenerateTestMatrix:
     name: 'GenerateTestMatrix'
     runs-on: ubuntu-latest
     outputs:
       ec2_linux_matrix: ${{ steps.set-matrix.outputs.ec2_linux_matrix }}
+      ec2_performance_matrix: ${{steps.set-matrix.outputs.ec2_performance_matrix}}
       ec2_windows_matrix: ${{ steps.set-matrix.outputs.ec2_windows_matrix }}
       ecs_fargate_matrix: ${{ steps.set-matrix.outputs.ecs_fargate_matrix }}
     steps:
@@ -156,12 +157,14 @@ jobs:
         run: |
           go run --tags=generator integration/generator/test_case_generator.go
           echo "::set-output name=ec2_linux_matrix::$(echo $(cat integration/generator/resources/ec2_linux_complete_test_matrix.json))"
+          echo "::set-output name=ec2_performance_matrix::$(echo $(cat integration/generator/resources/ec2_performance_complete_test_matrix.json))"
           echo "::set-output name=ec2_windows_matrix::$(echo $(cat integration/generator/resources/ec2_windows_complete_test_matrix.json))"
           echo "::set-output name=ecs_fargate_matrix::$(echo $(cat integration/generator/resources/ecs_fargate_complete_test_matrix.json))"
 
       - name: Echo test plan matrix
         run: |
           echo ${{ steps.set-matrix.outputs.ec2_linux_matrix }}
+          echo ${{ steps.set-matrix.outputs.ec2_performance_matrix}}
           echo ${{ steps.set-matrix.outputs.ec2_windows_matrix }}
           echo ${{ steps.set-matrix.outputs.ecs_fargate_matrix }}
 
@@ -614,6 +617,10 @@ jobs:
     name: "PerformanceTrackingTest"
     needs: [MakeBinary, StartLocalStack, GenerateTestMatrix]
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_performance_matrix) }}
     steps:
       - uses: actions/checkout@v2
 
@@ -633,52 +640,51 @@ jobs:
 
       - name: Echo Test Info
         run: echo run performance-tracking
-
-      - name: Verify Terraform version
-        run: terraform --version
-
+      - name: Get SHA
+        id: sha
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
       - name: Get git date
         id: sha_date
         run: echo "::set-output name=sha_date::$(git show -s --format=%ct ${{ steps.sha.outputs.sha_short }} )"
-
       - name: Check env
-        run: echo "SHA ${GITHUB_SHA} | Date ${{ steps.sha_date.outputs.sha_date }}"
-
+        run: echo "SHA ${{ steps.sha.outputs.sha_short }} | Date ${{ steps.sha_date.outputs.sha_date }} "
+      - name: Verify Terraform version
+        run: terraform --version
       - name: Terraform apply
         if: steps.performance-tracking.outputs.cache-hit != 'true'
         uses: nick-invision/retry@v2
         with:
           max_attempts: 1
-          timeout_minutes: 50
+          timeout_minutes: 30
           retry_wait_seconds: 5
           command: |
             cd integration/terraform/ec2/linux
             terraform init
             if terraform apply --auto-approve \
               -var="ssh_key=${PRIVATE_KEY}" -var="github_repo=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" \
-              -var="github_sha=${GITHUB_SHA}" -var="install_agent=rpm -U ./amazon-cloudwatch-agent.rpm" \
-              -var="user=ec2-user" \
-              -var="ami=cloudwatch-agent-integration-test-al2*" \
-              -var="ca_cert_path=/etc/ssl/certs/ca-bundle.crt" \
-              -var="arc=amd64" \
-              -var="binary_name=amazon-cloudwatch-agent.rpm" \
+              -var="github_sha=${GITHUB_SHA}" -var="install_agent=${{ matrix.arrays.installAgentCommand }}" \
+              -var="user=${{ matrix.arrays.username }}" \
+              -var="ami=${{ matrix.arrays.ami }}" \
+              -var="ca_cert_path=${{ matrix.arrays.caCertPath }}" \
+              -var="arc=${{ matrix.arrays.arc }}" \
+              -var="binary_name=${{ matrix.arrays.binaryName }}" \
               -var="local_stack_host_name=${{ needs.StartLocalStack.outputs.local_stack_host_name }}" \
               -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \
               -var="key_name=${KEY_NAME}" \
-              -var="test_name=cw-integ-test-al2" \
               -var="github_sha_date=${{ steps.sha_date.outputs.sha_date }}" \
-              -var="test_dir=./integration/test/performancetest" ; then terraform destroy -auto-approve
-
+              -var="test_name=cw-integ-test-${{ matrix.arrays.os }}" \
+              -var="performance_number_of_logs=${{ matrix.arrays.performance_number_of_logs}}"\
+              -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve
             else
               terraform destroy -auto-approve && exit 1
             fi
 
       #This is here just in case workflow cancel
       - name: Terraform destroy
-        if: ${{ cancelled() && steps.performance-tracking.outputs.cache-hit != 'true' }}
+        if: ${{ cancelled() && steps.ec2-linux-integration-test.outputs.cache-hit != 'true' }}
         uses: nick-invision/retry@v2
         with:
           max_attempts: 3
           timeout_minutes: 8
           retry_wait_seconds: 5
-          command: cd integration/terraform/ec2/linux && terraform destroy --auto-approve
+          command: cd integration/terraform/ec2/linux && terraform destroy --auto-approve

.github/workflows/releaseTest.yml

@@ -0,0 +1,52 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT
+
+name: Release Update 
+env:
+  PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY  }}
+  TERRAFORM_AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }}
+  TERRAFORM_AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }}
+  S3_INTEGRATION_BUCKET: ${{ secrets.S3_INTEGRATION_BUCKET }}
+  KEY_NAME: ${{ secrets.KEY_NAME }}
+  VPC_SECURITY_GROUPS_IDS: ${{ secrets.VPC_SECURITY_GROUPS_IDS }}
+  IAM_ROLE: ${{ secrets.IAM_ROLE }}
+  GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
+  PASSPHRASE: ${{ secrets.PASSPHRASE }}
+  GPG_KEY_NAME: ${{ secrets.GPG_KEY_NAME }}
+  GPG_TTY: $(tty)
+
+on:
+  release:
+      types: [created]
+
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  UpdatePerformanceMetrics:
+    name: "UpdatePerformanceMetrics"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ~1.18.3
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Update isRelease for this release
+        run: |
+            cd integration/test/performancetest
+            export IS_RELEASE=true
+            export SHA=$GITHUB_SHA
+            go test -run TestUpdateCommit -p 1 -v --tags=integration
+

go.mod

@@ -68,6 +68,7 @@ require (
 	github.com/gobwas/glob v0.2.3
 	github.com/google/cadvisor v0.44.0
 	github.com/google/go-cmp v0.5.8
+	github.com/google/uuid v1.3.0
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/influxdata/telegraf v0.0.0-00010101000000-000000000000
 	github.com/influxdata/toml v0.0.0-20190415235208-270119a8ce65
@@ -177,7 +178,6 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.3.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gophercloud/gophercloud v0.24.0 // indirect

go.sum

@@ -2723,4 +2723,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLz
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

integration/generator/resources/ec2_performance_test_matrix.json

@@ -0,0 +1,32 @@
+[
+    {
+        "os": "al2",
+        "username": "ec2-user",
+        "installAgentCommand": "rpm -U ./amazon-cloudwatch-agent.rpm",
+        "ami": "cloudwatch-agent-integration-test-al2*",
+        "caCertPath": "/etc/ssl/certs/ca-bundle.crt",
+        "arc": "amd64",
+        "binaryName": "amazon-cloudwatch-agent.rpm",
+        "performance_number_of_logs": "10"
+      },
+      {
+        "os": "al2",
+        "username": "ec2-user",
+        "installAgentCommand": "rpm -U ./amazon-cloudwatch-agent.rpm",
+        "ami": "cloudwatch-agent-integration-test-al2*",
+        "caCertPath": "/etc/ssl/certs/ca-bundle.crt",
+        "arc": "amd64",
+        "binaryName": "amazon-cloudwatch-agent.rpm",
+        "performance_number_of_logs": "100"
+      },
+      {
+        "os": "al2",
+        "username": "ec2-user",
+        "installAgentCommand": "rpm -U ./amazon-cloudwatch-agent.rpm",
+        "ami": "cloudwatch-agent-integration-test-al2*",
+        "caCertPath": "/etc/ssl/certs/ca-bundle.crt",
+        "arc": "amd64",
+        "binaryName": "amazon-cloudwatch-agent.rpm",
+        "performance_number_of_logs": "1000"
+      }
+]

integration/generator/test_case_generator.go

@@ -25,6 +25,9 @@ var osToTestDirMap = map[string][]string{
 		"./integration/test/cloudwatchlogs",
 		"./integration/test/metrics_number_dimension",
 	},
+	"ec2_performance":{
+		"./integration/test/performancetest",
+	},
 	// @TODO add real tests
 	"ec2_windows": {""},
 	"ec2_mac":     {},

integration/terraform/ec2/linux/iam.tf

@@ -47,7 +47,9 @@ data "aws_iam_policy_document" "user-managed-policy-document" {
       "s3:ListBucket",
       "dynamodb:DescribeTable",
       "dynamodb:PutItem",
-      "dynamodb:CreateTable"
+      "dynamodb:CreateTable",
+      "dynamodb:Query",
+      "dynamodb:UpdateItem"
     ]
     resources = ["*"]
   }
@@ -61,4 +63,4 @@ resource "aws_iam_policy" "cwagent_server_policy" {
 resource "aws_iam_role_policy_attachment" "cwagent_server_policy_attachment" {
   role       = aws_iam_role.cwagent_role.name
   policy_arn = aws_iam_policy.cwagent_server_policy.arn
-}
+}

integration/terraform/ec2/linux/main.tf

@@ -46,6 +46,7 @@ resource "null_resource" "integration_test" {
   # Prepare Integration Test
   provisioner "remote-exec" {
     inline = [
+      "echo sha ${var.github_sha}",
       "cloud-init status --wait",
       "echo clone and install agent",
       "git clone ${var.github_repo}",
@@ -80,10 +81,11 @@ resource "null_resource" "integration_test" {
       "export PATH=$PATH:/snap/bin:/usr/local/go/bin",
       "echo run integration test",
       "cd ~/amazon-cloudwatch-agent",
-      "go test ./integration/test/sanity -p 1 -v --tags=integration",
+      "echo run sanity test && go test ./integration/test/sanity -p 1 -v --tags=integration",
       "export SHA=${var.github_sha}",
       "export SHA_DATE=${var.github_sha_date}",
-      "go test ${var.test_dir} -p 1 -v --tags=integration"
+      "export PERFORMANCE_NUMBER_OF_LOGS=${var.performance_number_of_logs}",
+      "go test ${var.test_dir} -p 1 -timeout 30m -v --tags=integration "
     ]
     connection {
       type        = "ssh"

integration/terraform/ec2/linux/variables.tf

@@ -82,4 +82,8 @@ variable "github_repo" {
 variable "github_sha_date"{
   type = string
   default = ""
-}
+}
+variable "performance_number_of_logs"{
+  type = string
+  default = ""
+}

integration/test/performancetest/performance_query_utils.go

@@ -16,20 +16,32 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
+
+	"github.com/google/uuid"
 )
 
 const (
 	Namespace = "CWAgent"
 	DimensionName = "InstanceId"
 	Stat = "Average"
 	Period = 10
-
 	METRIC_PERIOD = 5 * 60 // this const is in seconds , 5 mins
 	PARTITION_KEY ="Year"
 	HASH = "Hash"
 	COMMIT_DATE= "CommitDate"
 	SHA_ENV  = "SHA"
 	SHA_DATE_ENV = "SHA_DATE"
+	IS_RELEASE = "isRelease"
+	TEST_ID ="TestID"
+	TPS = "TPS"
+	PERFORMANCE_NUMBER_OF_LOGS = "PERFORMANCE_NUMBER_OF_LOGS"
+	RESULTS = "Results"
+	/*
+	TEST_ID is used for version control, in order to make sure the
+	item has not changed between item being editted and updated.
+	TEST_ID is checked atomicaly.
+	 TEST_ID uses UIUD to give unique id to each packet.
+	*/
 )
 
 //struct that holds statistics on the returned data
@@ -177,16 +189,19 @@ func GetPerformanceMetrics(instanceId string, agentRuntime, logNum, tps int, age
 	packet[PARTITION_KEY] = time.Now().Year()
 	packet[HASH] = os.Getenv(SHA_ENV) //fmt.Sprintf("%d", time.Now().UnixNano())
 	packet[COMMIT_DATE],_ = strconv.Atoi(os.Getenv(SHA_DATE_ENV))
-
+	packet[IS_RELEASE] = false
 	//add test metadata
-	packet["NumberOfLogsMonitored"] = logNum
-	packet["TPS"] = tps
+	packet[TEST_ID] = uuid.New().String()
+	testSettings := fmt.Sprintf("%d-%d",logNum,tps)
+	testMetricResults := make(map[string]Stats)
+
 
 	//add actual test data with statistics
 	for _, result := range metrics.MetricDataResults {
-		packet[*result.Label] = CalcStats(result.Values)
+		stats:= CalcStats(result.Values)
+		testMetricResults[*result.Label] = stats
 	}
-
+	packet[RESULTS] = map[string]map[string]Stats{ testSettings: testMetricResults}
 	return packet, nil
 }