diff --git a/buildspec.yml b/buildspec.yml index 0ea147669..a2c772c21 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -14,9 +14,24 @@ phases: # List the docker images - docker images - # Push the image to ECR in the same account and same region the pipeline is hosted. - - docker tag amazon/aws-for-fluent-bit:latest amazon/aws-for-fluent-bit-test:latest - - ecs-cli push amazon/aws-for-fluent-bit-test:latest + # Push the image to ECR with corresponding architecture as the tag. + - aws ecr get-login-password --region ${AWS_REGION}| docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com + - aws ecr create-repository --repository-name amazon/aws-for-fluent-bit-test --image-scanning-configuration scanOnPush=true --region ${AWS_REGION} || true + - architecture=$(docker inspect --format='{{.Architecture}}' amazon/aws-for-fluent-bit) + - docker tag amazon/aws-for-fluent-bit:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture + - docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture + + # Create manifest list + - export DOCKER_CLI_EXPERIMENTAL=enabled + - docker manifest create ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 || true + - docker manifest annotate --arch arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 || true + - docker manifest annotate --arch amd64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 || true + + # Sanity check for the debug log + - docker manifest inspect ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest || true + + # Push manifest list + - docker manifest push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest || true artifacts: files: - - '**/*' + - '**/*' \ No newline at end of file diff --git a/buildspec_integ.yml b/buildspec_integ.yml index 67b0c3f66..bcc5cd2c7 100644 --- a/buildspec_integ.yml +++ b/buildspec_integ.yml @@ -15,14 +15,15 @@ phases: - 'export AWS_SESSION_TOKEN=`echo $CREDS | jq -r .Token`' # Pull the image that we built and pushed in the `Build` stage - - 'ecs-cli pull amazon/aws-for-fluent-bit-test:latest' - - 'docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest amazon/aws-for-fluent-bit:latest' - - # List the images to do a double check - - 'docker images' + - aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com + - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest + - docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test amazon/aws-for-fluent-bit:latest + + # List the images to do a double check + - docker images # Command to run the integration test - - 'make integ' + - make integ artifacts: files: - '**/*' diff --git a/buildspec_publish_dockerhub.yml b/buildspec_publish_dockerhub.yml index d9e9124cc..c2dee8193 100644 --- a/buildspec_publish_dockerhub.yml +++ b/buildspec_publish_dockerhub.yml @@ -9,8 +9,11 @@ phases: build: commands: # Pull the image that we built and pushed in the `Build` stage - - 'ecs-cli pull amazon/aws-for-fluent-bit-test:latest' - - 'docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest amazon/aws-for-fluent-bit:latest' + - aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com + - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"amd64" + - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"arm64" + - docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 amazon/aws-for-fluent-bit:amd64 + - docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 amazon/aws-for-fluent-bit:arm64 # List the docker images - docker images diff --git a/buildspec_publish_ecr.yml b/buildspec_publish_ecr.yml index 68e46cc6d..0592b2dca 100644 --- a/buildspec_publish_ecr.yml +++ b/buildspec_publish_ecr.yml @@ -9,9 +9,10 @@ phases: build: commands: # Pull the image that we built and pushed in the `Build` stage - - 'ecs-cli pull amazon/aws-for-fluent-bit-test:latest' - - 'docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest amazon/aws-for-fluent-bit:latest' - + - aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com + - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"amd64" + - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"arm64" + # List the docker images - docker images diff --git a/integ/integ.sh b/integ/integ.sh index 4a69c1d78..829cf8f0e 100755 --- a/integ/integ.sh +++ b/integ/integ.sh @@ -3,12 +3,13 @@ export AWS_REGION="us-west-2" export PROJECT_ROOT="$(pwd)" test_cloudwatch() { - export LOG_GROUP_NAME="fluent-bit-integ-test" + export ARCHITECTURE=$(uname -m) + export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}" # Tag is used to name the log stream; each test run has a unique (random) log stream name export TAG=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 10) docker-compose --file ./integ/test_cloudwatch/docker-compose.test.yml build docker-compose --file ./integ/test_cloudwatch/docker-compose.test.yml up --abort-on-container-exit - sleep 10 + sleep 120 # Creates a file as a flag for the validation failure mkdir -p ./integ/out @@ -20,7 +21,7 @@ test_cloudwatch() { } clean_cloudwatch() { - export LOG_GROUP_NAME="fluent-bit-integ-test" + export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}" # Clean up resources that were created in the test docker-compose --file ./integ/test_cloudwatch/docker-compose.clean.yml build docker-compose --file ./integ/test_cloudwatch/docker-compose.clean.yml up --abort-on-container-exit @@ -81,7 +82,6 @@ test_firehose() { clean_s3() { validate_or_clean_s3 clean } - if [ "${1}" = "cloudwatch" ]; then export PLUGIN_UNDER_TEST="cloudwatch" test_cloudwatch @@ -163,4 +163,4 @@ fi if [ "${1}" = "delete" ]; then source ./integ/resources/delete_test_resources.sh -fi +fi \ No newline at end of file diff --git a/integ/resources/create_test_resources.sh b/integ/resources/create_test_resources.sh index 691d265a3..960040006 100755 --- a/integ/resources/create_test_resources.sh +++ b/integ/resources/create_test_resources.sh @@ -1,5 +1,6 @@ #!/bin/bash # Deploys the CloudFormation template to create the stack and necessary resources- kinesis data stream, s3 bucket, and kinesis firehose delivery stream -# Resource (stream, s3, delivery stream) names will start with the stack name "integ-test-fluent-bit" -aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit --region us-west-2 --capabilities CAPABILITY_NAMED_IAM +# Resource (stream, s3, delivery stream) names will start with the stack name followed by the corresponding architecture. "integ-test-fluent-bit-architecture" +ARCHITECTURE=$(uname -m | tr '_' '-') +aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit-${ARCHITECTURE} --region us-west-2 --capabilities CAPABILITY_NAMED_IAM \ No newline at end of file diff --git a/integ/resources/delete_test_resources.sh b/integ/resources/delete_test_resources.sh index c3348edaf..a5c4f74d8 100644 --- a/integ/resources/delete_test_resources.sh +++ b/integ/resources/delete_test_resources.sh @@ -1,2 +1,3 @@ # Delete the CloudFormation stack which created all the resources for running the integration test -aws cloudformation delete-stack --stack-name integ-test-fluent-bit \ No newline at end of file +ARCHITECTURE=$(uname -m | tr '_' '-') +aws cloudformation delete-stack --stack-name integ-test-fluent-bit-${ARCHITECTURE} \ No newline at end of file diff --git a/integ/resources/setup_test_environment.sh b/integ/resources/setup_test_environment.sh index 9a9a47a1b..525d531a9 100644 --- a/integ/resources/setup_test_environment.sh +++ b/integ/resources/setup_test_environment.sh @@ -1,7 +1,8 @@ #!/bin/bash # Using CloudFormation describe-stacks extracts the output values for kinesis stream and s3 bucket name, and sets them as environment variables -stackOutputs=$(aws cloudformation describe-stacks --stack-name integ-test-fluent-bit --output text --query 'Stacks[0].Outputs[*].OutputValue') +ARCHITECTURE=$(uname -m | tr '_' '-') +stackOutputs=$(aws cloudformation describe-stacks --stack-name integ-test-fluent-bit-${ARCHITECTURE} --output text --query 'Stacks[0].Outputs[*].OutputValue') read -r -a outputArray <<< "$stackOutputs" export FIREHOSE_STREAM="${outputArray[0]}" export KINESIS_STREAM="${outputArray[1]}" diff --git a/integ/validate_cloudwatch/validator.py b/integ/validate_cloudwatch/validator.py index 147dd9ce5..5a201793f 100644 --- a/integ/validate_cloudwatch/validator.py +++ b/integ/validate_cloudwatch/validator.py @@ -8,7 +8,7 @@ client = boto3.client('logs', region_name=os.environ.get('AWS_REGION')) metrics_client = boto3.client("cloudwatch", region_name=os.environ["AWS_REGION"]) -start_time = datetime.utcnow() - timedelta(seconds=60) +start_time = datetime.utcnow() - timedelta(seconds=600) end_time = datetime.utcnow() LOG_GROUP_NAME = os.environ.get('LOG_GROUP_NAME') @@ -55,7 +55,7 @@ def validate_metric(test_name, metric_namespace, dim_key, dim_value, expected_sa return True attempts += 1 print(f"No metrics yet. Sleeping before trying again. Attempt # {attempts}") - time.sleep(2) + time.sleep(10) sys.exit('TEST_FAILURE: failed to validate metric existence in CloudWatch') @@ -102,4 +102,4 @@ def get_expected_metric_name(): if success_case_1 and success_case_2 and success_case_emf: # if this file is still present, integ script will mark the test as a failure - os.remove("/out/cloudwatch-test") + os.remove("/out/cloudwatch-test") \ No newline at end of file diff --git a/scripts/publish.sh b/scripts/publish.sh index 17d3f7cda..fcb8a76e6 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -73,8 +73,11 @@ gamma_account_id="626332813196" DOCKER_HUB_SECRET="com.amazonaws.dockerhub.aws-for-fluent-bit.credentials" +ARCHITECTURES=("amd64" "arm64") + publish_to_docker_hub() { DRY_RUN="${DRY_RUN:-true}" + export DOCKER_CLI_EXPERIMENTAL=enabled username="$(aws secretsmanager get-secret-value --secret-id $DOCKER_HUB_SECRET --region us-west-2 | jq -r '.SecretString | fromjson.username')" password="$(aws secretsmanager get-secret-value --secret-id $DOCKER_HUB_SECRET --region us-west-2 | jq -r '.SecretString | fromjson.password')" @@ -90,13 +93,22 @@ publish_to_docker_hub() { # Publish to DockerHub only if $DRY_RUN is set to false if [[ "${DRY_RUN}" == "false" ]]; then - docker tag ${1} ${2} - docker push ${1} - docker push ${2} + for arch in "${ARCHITECTURES[@]}" + do + docker tag ${1}:"$arch" ${1}:"${arch}"-${AWS_FOR_FLUENT_BIT_VERSION} + docker push ${1}:"$arch"-${AWS_FOR_FLUENT_BIT_VERSION} + done + create_manifest_list ${1} "latest" + create_manifest_list ${1} ${AWS_FOR_FLUENT_BIT_VERSION} + else - echo "DRY_RUN: docker tag ${1} ${2}" - echo "DRY_RUN: docker push ${1}" - echo "DRY_RUN: docker push ${2}" + for arch in "${ARCHITECTURES[@]}" + do + echo "DRY_RUN: docker tag ${1}:${arch} ${1}:${arch}-${AWS_FOR_FLUENT_BIT_VERSION}" + echo "DRY_RUN: docker push ${1}:${arch}-${AWS_FOR_FLUENT_BIT_VERSION}" + done + echo "DRY_RUN: create manifest list ${1}:latest" + echo "DRY_RUN: create manifest list ${1}:${AWS_FOR_FLUENT_BIT_VERSION}" echo "DRY_RUN is NOT set to 'false', skipping DockerHub update. Exiting..." fi @@ -173,11 +185,37 @@ verify_ssm() { fi } -push_to_ecr() { - docker tag ${1} ${2} - ecs-cli push ${2} --region ${3} --registry-id ${4} +create_manifest_list() { + + export DOCKER_CLI_EXPERIMENTAL=enabled + tag=${2} + + # TODO: Add a way to automatically generate arch images in manifest + docker manifest create ${1}:${tag} ${1}:arm64-${AWS_FOR_FLUENT_BIT_VERSION} ${1}:amd64-${AWS_FOR_FLUENT_BIT_VERSION} + + for arch in "${ARCHITECTURES[@]}" + do + docker manifest annotate --arch "$arch" ${1}:${tag} ${1}:"$arch"-${AWS_FOR_FLUENT_BIT_VERSION} + done + + # sanity check on the debug log. + docker manifest inspect ${1}:${tag} + docker manifest push ${1}:${tag} +} + +push_image_ecr() { + account_id=${1} + region=${2} + + for arch in "${ARCHITECTURES[@]}" + do + docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"$arch" \ + ${account_id}.dkr.ecr.${region}.amazonaws.com/aws-for-fluent-bit:"$arch"-${AWS_FOR_FLUENT_BIT_VERSION} + docker push ${account_id}.dkr.ecr.${region}.amazonaws.com/aws-for-fluent-bit:"$arch"-${AWS_FOR_FLUENT_BIT_VERSION} + done } +# TODO: remove dependency on ecs-cli pull_ecr() { ecs-cli pull ${1} --region ${2} } @@ -189,8 +227,17 @@ make_repo_public() { publish_ecr() { region=${1} account_id=${2} - push_to_ecr amazon/aws-for-fluent-bit:latest aws-for-fluent-bit:latest ${region} ${account_id} - push_to_ecr amazon/aws-for-fluent-bit:latest "aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION}" ${region} ${account_id} + echo $region + echo $account_id + + aws ecr get-login-password --region ${region}| docker login --username AWS --password-stdin ${account_id}.dkr.ecr.${region}.amazonaws.com + aws ecr create-repository --repository-name aws-for-fluent-bit --image-scanning-configuration scanOnPush=true --region ${region} || true + + push_image_ecr ${account_id} ${region} + + create_manifest_list ${account_id}.dkr.ecr.${region}.amazonaws.com/aws-for-fluent-bit ${AWS_FOR_FLUENT_BIT_VERSION} + create_manifest_list ${account_id}.dkr.ecr.${region}.amazonaws.com/aws-for-fluent-bit "latest" + make_repo_public ${region} } @@ -204,14 +251,15 @@ verify_ecr() { endpoint=${endpoint}.cn fi - pull_ecr ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:latest ${region} + aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${account_id}.dkr.ecr.${region}.amazonaws.com + docker pull ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:latest sha1=$(docker inspect --format='{{index .RepoDigests 0}}' ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:latest) if [ "${is_sync_task}" = "true" ]; then pull_ecr ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION_DOCKERHUB} ${region} sha2=$(docker inspect --format='{{index .RepoDigests 0}}' ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION_DOCKERHUB}) else - pull_ecr ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION} ${region} + docker pull ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION} sha2=$(docker inspect --format='{{index .RepoDigests 0}}' ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION}) fi @@ -259,7 +307,7 @@ match_two_sha() { if [ "${1}" = "publish" ]; then if [ "${2}" = "dockerhub" ]; then - publish_to_docker_hub amazon/aws-for-fluent-bit:latest amazon/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION} + publish_to_docker_hub amazon/aws-for-fluent-bit fi if [ "${2}" = "aws" ]; then @@ -417,7 +465,7 @@ fi # Following scripts will be called only from the CI/CD pipeline if [ "${1}" = "cicd-publish" ]; then if [ "${2}" = "dockerhub" ]; then - publish_to_docker_hub amazon/aws-for-fluent-bit:latest amazon/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION} + publish_to_docker_hub amazon/aws-for-fluent-bit elif [ "${2}" = "us-gov-east-1" ] || [ "${2}" = "us-gov-west-1" ]; then for region in ${gov_regions}; do sync_latest_image ${region} ${gov_regions_account_id}