diff --git a/integration/generator/test_case_generator.go b/integration/generator/test_case_generator.go index 74aafde052..9c76da74b6 100644 --- a/integration/generator/test_case_generator.go +++ b/integration/generator/test_case_generator.go @@ -27,8 +27,9 @@ var osToTestDirMap = map[string][]string{ "./integration/test/ca_bundle", "./integration/test/cloudwatchlogs", "./integration/test/metrics_number_dimension", + "./integration/test/metric_value_benchmark", }, - "ec2_performance":{ + "ec2_performance": { "./integration/test/performancetest", }, // @TODO add real tests @@ -48,14 +49,14 @@ func main() { func genMatrix(targetOS string, testDirList []string) []map[string]string { openTestMatrix, err := os.Open(fmt.Sprintf("integration/generator/resources/%v_test_matrix.json", targetOS)) - + if err != nil { log.Panicf("can't read file %v_test_matrix.json err %v", targetOS, err) } - + byteValueTestMatrix, _ := ioutil.ReadAll(openTestMatrix) _ = openTestMatrix.Close() - + var testMatrix []map[string]string err = json.Unmarshal(byteValueTestMatrix, &testMatrix) if err != nil { diff --git a/integration/test/metric/cpu.go b/integration/test/metric/cpu.go new file mode 100644 index 0000000000..c953cf250d --- /dev/null +++ b/integration/test/metric/cpu.go @@ -0,0 +1,68 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build linux && integration +// +build linux,integration + +package metric + +import ( + "log" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/aws/aws-sdk-go/aws" +) + +type CPUMetricValueFetcher struct { + baseMetricValueFetcher +} + +func (f *CPUMetricValueFetcher) Fetch(namespace string, metricName string, stat Statistics) ([]float64, error) { + dimensions := f.getMetricSpecificDimensions() + values, err := f.fetch(namespace, dimensions, metricName, stat) + if err != nil { + log.Printf("Error while fetching metric value for %v: %v", metricName, err.Error()) + } + return values, err +} + +var cpuSupportedMetricValues = map[string]struct{}{ + "cpu_time_active": {}, + "cpu_time_guest": {}, + "cpu_time_guest_nice": {}, + "cpu_time_idle": {}, + "cpu_time_iowait": {}, + "cpu_time_irq": {}, + "cpu_time_nice": {}, + "cpu_time_softirq": {}, + "cpu_time_steal": {}, + "cpu_time_system": {}, + "cpu_time_user": {}, + "cpu_usage_active": {}, + "cpu_usage_quest": {}, + "cpu_usage_quest_nice": {}, + "cpu_usage_idle": {}, + "cpu_usage_iowait": {}, + "cpu_usage_irq": {}, + "cpu_usage_nice": {}, + "cpu_usage_softirq": {}, + "cpu_usage_steal": {}, + "cpu_usage_system": {}, + "cpu_usage_user": {}, +} + +func (f *CPUMetricValueFetcher) isApplicable(metricName string) bool { + _, exists := cpuSupportedMetricValues[metricName] + return exists +} + +var cpuMetricsSpecificDimension = []types.Dimension{ + { + Name: aws.String("cpu"), + Value: aws.String("cpu-total"), + }, +} + +func (f *CPUMetricValueFetcher) getMetricSpecificDimensions() []types.Dimension { + return cpuMetricsSpecificDimension +} diff --git a/integration/test/metric/metric_value_query.go b/integration/test/metric/metric_value_query.go new file mode 100644 index 0000000000..de44c19a8a --- /dev/null +++ b/integration/test/metric/metric_value_query.go @@ -0,0 +1,97 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build linux && integration +// +build linux,integration + +package metric + +import ( + "fmt" + "github.com/aws/amazon-cloudwatch-agent/integration/test" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/aws/aws-sdk-go/aws" + "log" + "time" +) + +var metricValueFetchers = []MetricValueFetcher{ + &CPUMetricValueFetcher{}, +} + +func GetMetricFetcher(metricName string) (MetricValueFetcher, error) { + for _, fetcher := range metricValueFetchers { + if fetcher.isApplicable(metricName) { + return fetcher, nil + } + } + err := fmt.Errorf("No metric fetcher for metricName %v", metricName) + log.Printf("%s", err) + return nil, err +} + +type MetricValueFetcher interface { + Fetch(namespace string, metricName string, stat Statistics) ([]float64, error) + fetch(namespace string, metricSpecificDimensions []types.Dimension, metricName string, stat Statistics) ([]float64, error) + isApplicable(metricName string) bool + getMetricSpecificDimensions() []types.Dimension +} + +type baseMetricValueFetcher struct{} + +func (f *baseMetricValueFetcher) fetch(namespace string, metricSpecificDimensions []types.Dimension, metricName string, stat Statistics) ([]float64, error) { + ec2InstanceId := test.GetInstanceId() + instanceIdDimension := types.Dimension{ + Name: aws.String("InstanceId"), + Value: aws.String(ec2InstanceId), + } + dimensions := append(metricSpecificDimensions, instanceIdDimension) + metricToFetch := types.Metric{ + Namespace: aws.String(namespace), + MetricName: aws.String(metricName), + Dimensions: dimensions, + } + + metricQueryPeriod := int32(60) + metricDataQueries := []types.MetricDataQuery{ + { + MetricStat: &types.MetricStat{ + Metric: &metricToFetch, + Period: &metricQueryPeriod, + Stat: aws.String(string(stat)), + }, + Id: aws.String(metricName), + }, + } + + endTime := time.Now() + startTime := subtractMinutes(endTime, 10) + getMetricDataInput := cloudwatch.GetMetricDataInput{ + StartTime: &startTime, + EndTime: &endTime, + MetricDataQueries: metricDataQueries, + } + + log.Printf("Metric data input is : %s", fmt.Sprint(getMetricDataInput)) + + cwmClient, clientContext, err := test.GetCloudWatchMetricsClient() + if err != nil { + return nil, fmt.Errorf("Error occurred while creating CloudWatch client: %v", err.Error()) + } + + output, err := cwmClient.GetMetricData(*clientContext, &getMetricDataInput) + if err != nil { + return nil, fmt.Errorf("Error getting metric data %v", err) + } + + result := output.MetricDataResults[0].Values + log.Printf("Metric Value is : %s", fmt.Sprint(result)) + + return result, nil +} + +func subtractMinutes(fromTime time.Time, minutes int) time.Time { + tenMinutes := time.Duration(-1*minutes) * time.Minute + return fromTime.Add(tenMinutes) +} diff --git a/integration/test/metric/query-json.json b/integration/test/metric/query-json.json new file mode 100644 index 0000000000..ef2f00b655 --- /dev/null +++ b/integration/test/metric/query-json.json @@ -0,0 +1,23 @@ +[ + { + "Id": "m1", + "MetricStat": { + "Metric": { + "Namespace": "MetricValueBenchmarkTest", + "MetricName": "cpu_usage_active", + "Dimensions": [ + { + "Name": "InstanceId", + "Value": "i-095d623fa10a192e3" + }, + { + "Name": "cpu", + "Value": "cpu-total" + } + ] + }, + "Period": 60, + "Stat": "Average" + } + } +] \ No newline at end of file diff --git a/integration/test/metric/stat.go b/integration/test/metric/stat.go new file mode 100644 index 0000000000..63779f1936 --- /dev/null +++ b/integration/test/metric/stat.go @@ -0,0 +1,7 @@ +package metric + +type Statistics string + +const ( + AVERAGE Statistics = "Average" +) diff --git a/integration/test/metric_value_benchmark/agent_configs/base_config.json b/integration/test/metric_value_benchmark/agent_configs/base_config.json new file mode 100644 index 0000000000..f24092a493 --- /dev/null +++ b/integration/test/metric_value_benchmark/agent_configs/base_config.json @@ -0,0 +1,24 @@ +{ + "agent": { + "metrics_collection_interval": 60, + "run_as_user": "root", + "debug": true, + "logfile": "" + }, + "metrics": { + "namespace": "MetricValueBenchmarkTest", + "append_dimensions": { + "InstanceId": "${aws:InstanceId}" + }, + "metrics_collected": { + "cpu": { + "measurement": [ + "time_active", "time_guest", "time_guest_nice", "time_idle", "time_iowait", "time_irq", + "time_nice", "time_softirq", "time_steal", "time_system", "time_user", + "usage_active", "usage_guest", "usage_guest_nice", "usage_idle", "usage_iowait", "usage_irq", + "usage_nice", "usage_softirq", "usage_steal", "usage_system", "usage_user" + ] + } + } + } +} \ No newline at end of file diff --git a/integration/test/metric_value_benchmark/metrics_value_benchmark_test.go b/integration/test/metric_value_benchmark/metrics_value_benchmark_test.go new file mode 100644 index 0000000000..fe9ccc3c6e --- /dev/null +++ b/integration/test/metric_value_benchmark/metrics_value_benchmark_test.go @@ -0,0 +1,128 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build linux && integration +// +build linux,integration + +package metric_value_benchmark + +import ( + "fmt" + "log" + "testing" + "text/tabwriter" + "time" + + "github.com/aws/amazon-cloudwatch-agent/integration/test" + "github.com/aws/amazon-cloudwatch-agent/integration/test/metric" + "github.com/aws/amazon-cloudwatch-agent/integration/test/status" +) + +const configOutputPath = "/opt/aws/amazon-cloudwatch-agent/bin/config.json" +const configJSON = "/base_config.json" + +const namespace = "MetricValueBenchmarkTest" +const instanceId = "InstanceId" + +const minimumAgentRuntime = 3 * time.Minute + +func TestCPUValue(t *testing.T) { + log.Printf("testing cpu value...") + + resourcePath := "agent_configs" + + log.Printf("resource file location %s", resourcePath) + + t.Run(fmt.Sprintf("resource file location %s ", resourcePath), func(t *testing.T) { + test.CopyFile(resourcePath+configJSON, configOutputPath) + err := test.StartAgent(configOutputPath, false) + + if err != nil { + t.Fatalf("Agent could not start") + } + + time.Sleep(minimumAgentRuntime) + log.Printf("Agent has been running for : %s", minimumAgentRuntime.String()) + test.StopAgent() + + testResult := validateCpuMetrics() + testSuiteStatus := getTestSuiteStatus(testResult) + printTestResult(testSuiteStatus, testResult) + + if testSuiteStatus == status.FAILED { + t.Fatalf("Cpu test failed to validate that every metric value is greater than zero") + } + }) + + // TODO: Get CPU value > 0 + // TODO: Range test with >0 and <100 + // TODO: Range test: which metric to get? api reference check. should I get average or test every single datapoint for 10 minutes? (and if 90%> of them are in range, we are good) +} + +var metricsToFetch = []string{ + "cpu_time_active", "cpu_time_guest", "cpu_time_guest_nice", "cpu_time_idle", "cpu_time_iowait", "cpu_time_irq", + "cpu_time_nice", "cpu_time_softirq", "cpu_time_steal", "cpu_time_system", "cpu_time_user", + "cpu_usage_active", "cpu_usage_quest", "cpu_usage_quest_nice", "cpu_usage_idle", "cpu_usage_iowait", + "cpu_usage_irq", "cpu_usage_nice", "cpu_usage_softirq", "cpu_usage_steal", "cpu_usage_system", "cpu_usage_user"} + +func validateCpuMetrics() map[string]status.TestStatus { + validationResult := map[string]status.TestStatus{} + for _, metricName := range metricsToFetch { + validationResult[metricName] = status.FAILED + + fetcher, err := metric.GetMetricFetcher(metricName) + if err != nil { + continue + } + + values, err := fetcher.Fetch(namespace, metricName, metric.AVERAGE) + if err != nil { + continue + } + + if !isAllValuesGreaterThanZero(metricName, values) { + continue + } + + validationResult[metricName] = status.SUCCESSFUL + } + return validationResult +} + +func isAllValuesGreaterThanZero(metricName string, values []float64) bool { + if len(values) == 0 { + log.Printf("No values found %v", metricName) + return false + } + for _, value := range values { + if value <= 0 { + log.Printf("Values are not all greater than zero for %v", metricName) + return false + } + } + log.Printf("Values are all greater than zero for %v", metricName) + return true +} + +func printTestResult(testSuiteStatus status.TestStatus, testSummary map[string]status.TestStatus) { + testSuite := "CPU Test" + + log.Printf("Finished %v", testSuite) + log.Printf("==============%v==============", testSuite) + log.Printf("==============%v==============", string(testSuiteStatus)) + w := tabwriter.NewWriter(log.Writer(), 1, 1, 1, ' ', 0) + for metricName, status := range testSummary { + fmt.Fprintln(w, metricName, "\t", status, "\t") + } + w.Flush() + log.Printf("==============================") +} + +func getTestSuiteStatus(testSummary map[string]status.TestStatus) status.TestStatus { + for _, value := range testSummary { + if value == status.FAILED { + return status.FAILED + } + } + return status.SUCCESSFUL +} diff --git a/integration/test/status/test_status.go b/integration/test/status/test_status.go new file mode 100644 index 0000000000..9bb61d7418 --- /dev/null +++ b/integration/test/status/test_status.go @@ -0,0 +1,14 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build linux && integration +// +build linux,integration + +package status + +type TestStatus string + +const ( + SUCCESSFUL TestStatus = "Successful" + FAILED TestStatus = "Failed" +)