diff --git a/tests/canary/canary.buildspec.yaml b/tests/canary/canary.buildspec.yaml index fd119f8fc8..ebbb1a0473 100644 --- a/tests/canary/canary.buildspec.yaml +++ b/tests/canary/canary.buildspec.yaml @@ -6,7 +6,6 @@ phases: # Get cached test image - aws ecr get-login-password --region $CLUSTER_REGION | docker login --username AWS --password-stdin $ECR_CACHE_URI || true - docker pull ${ECR_CACHE_URI}:latest --quiet || true - # Build test image - > docker build -f ./tests/canary/Dockerfile.canary . -t ${ECR_CACHE_URI}:latest --quiet @@ -15,8 +14,15 @@ phases: commands: # Run tests - docker run --name kf-distro-canary $(env | cut -f1 -d= | sed 's/^/-e /') --mount type=bind,source="$(pwd)/",target="/kubeflow-manifests/" ${ECR_CACHE_URI}:latest - - # Push test image to cache ECR repo + post_build: + commands: + - docker cp kf-distro-canary:/kubeflow-manifests/tests/canary/integration_tests.xml /tmp/results.xml || true + # Push test image to cache ECR repo - docker push ${ECR_CACHE_URI}:latest || true - + +reports: + IntegrationTestReport: + files: + - "results.xml" + base-directory: "/tmp" diff --git a/tests/canary/scripts/push_stats_to_cloudwatch.py b/tests/canary/scripts/push_stats_to_cloudwatch.py new file mode 100644 index 0000000000..8130cbfe2c --- /dev/null +++ b/tests/canary/scripts/push_stats_to_cloudwatch.py @@ -0,0 +1,90 @@ +import boto3 +from datetime import datetime +import xml.etree.ElementTree as ET +import os + + +xml_path = "../canary/integration_tests.xml" + + +def readXML_and_publish_metrics_to_cw(): + if os.path.isfile(xml_path): + tree = ET.parse(xml_path) + testsuite = tree.find("testsuite") + failures = testsuite.attrib["failures"] + tests = testsuite.attrib["tests"] + successes = int(tests) - int(failures) + else: + failures = 0 + successes = 0 + tests = 1 + + timestamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + + print(f"Failures: {failures}") + print(f"Total tests: {tests}") + print(f"Success: {successes}") + + # push to cloudwatch + cw_client = boto3.client("cloudwatch") + project_name = "CodeBuild-Run-All-Tests" + + # Define the metric data + metric_data = [ + { + "MetricName": "failures", + "Timestamp": timestamp, + "Dimensions": [ + {"Name": "CodeBuild Project Name", "Value": project_name}, + ], + "Value": int(failures), + "Unit": "Count", + }, + { + "MetricName": "total_tests", + "Timestamp": timestamp, + "Dimensions": [ + {"Name": "CodeBuild Project Name", "Value": project_name}, + ], + "Value": int(tests), + "Unit": "Count", + }, + { + "MetricName": "successes", + "Timestamp": timestamp, + "Dimensions": [ + {"Name": "CodeBuild Project Name", "Value": project_name}, + ], + "Value": int(successes), + "Unit": "Count", + }, + ] + + # Use the put_metric_data method to push the metric data to CloudWatch + try: + response = cw_client.put_metric_data( + Namespace="Canary_Metrics", MetricData=metric_data + ) + if response["ResponseMetadata"]["HTTPStatusCode"] == 200: + print("Successfully pushed data to CloudWatch") + # return 200 status code if successful + return 200 + else: + # raise exception if the status code is not 200 + raise Exception( + "Unexpected response status code: {}".format( + response["ResponseMetadata"]["HTTPStatusCode"] + ) + ) + except Exception as e: + print("Error pushing data to CloudWatch: {}".format(e)) + # raise exception if there was an error pushing data to CloudWatch + raise + + +def main(): + readXML_and_publish_metrics_to_cw() + + +if __name__ == "__main__": + main() diff --git a/tests/canary/scripts/run_test.sh b/tests/canary/scripts/run_test.sh index 574d6774ff..b21daa43c5 100755 --- a/tests/canary/scripts/run_test.sh +++ b/tests/canary/scripts/run_test.sh @@ -9,10 +9,11 @@ # Script configuration set -euo pipefail -function onError { - echo "Run test FAILED. Exiting." +function push_to_cloudwatch { + echo "Pushing Codebuild stats to Cloudwatch." + python ../canary/scripts/push_stats_to_cloudwatch.py } -trap onError ERR +trap push_to_cloudwatch EXIT export CANARY_TEST_DIR=${REPO_PATH}/tests/canary export E2E_TEST_DIR=${REPO_PATH}/tests/e2e @@ -31,6 +32,5 @@ mkdir -p $E2E_TEST_DIR/.metadata/ cp metadata-canary $E2E_TEST_DIR/.metadata/ cd $E2E_TEST_DIR -pytest tests/test_sanity_portforward.py -s -q --metadata .metadata/metadata-canary --region $CLUSTER_REGION --installation_option $INSTALLATION_OPTION - +pytest tests/test_sanity_portforward.py -s -q --metadata .metadata/metadata-canary --region $CLUSTER_REGION --installation_option $INSTALLATION_OPTION --junitxml ../canary/integration_tests.xml