Support for ARM64 build

aws · Sep 15, 2020 · 5da9c8f · 5da9c8f
1 parent 6ca1b6e
commit 5da9c8f
Show file tree

Hide file tree

Showing 10 changed files with 132 additions and 44 deletions.
diff --git a/buildspec.yml b/buildspec.yml
@@ -14,9 +14,24 @@ phases:
       # List the docker images 
       - docker images
 
-      # Push the image to ECR in the same account and same region the pipeline is hosted.
-      - docker tag amazon/aws-for-fluent-bit:latest amazon/aws-for-fluent-bit-test:latest
-      - ecs-cli push amazon/aws-for-fluent-bit-test:latest
+      # Push the image to ECR with corresponding architecture as the tag.
+      - aws ecr get-login-password --region ${AWS_REGION}| docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
+      - aws ecr create-repository --repository-name amazon/aws-for-fluent-bit-test --image-scanning-configuration scanOnPush=true --region ${AWS_REGION}  || true
+      - architecture=$(docker inspect --format='{{.Architecture}}'  amazon/aws-for-fluent-bit)
+      - docker tag amazon/aws-for-fluent-bit:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture
+      - docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture
+
+       # Create manifest list
+      - export DOCKER_CLI_EXPERIMENTAL=enabled
+      - docker manifest create ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 || true
+      - docker manifest annotate --arch arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 || true
+      - docker manifest annotate --arch amd64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 || true
+
+      # Sanity check for the debug log
+      - docker manifest inspect ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest || true
+
+      # Push manifest list
+      - docker manifest push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest || true 
 artifacts:
   files:
-    - '**/*'
+    - '**/*'
diff --git a/buildspec_integ.yml b/buildspec_integ.yml
@@ -15,14 +15,15 @@ phases:
       - 'export AWS_SESSION_TOKEN=`echo $CREDS | jq -r .Token`'
 
       # Pull the image that we built and pushed in the `Build` stage
-      - 'ecs-cli pull amazon/aws-for-fluent-bit-test:latest'                                                            
-      - 'docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest amazon/aws-for-fluent-bit:latest'
+      - aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
+      - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest
+      - docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test amazon/aws-for-fluent-bit:latest
+
+      # List the images to do a double check          
+      - docker images
 
-      # List the images to do a double check
-      - 'docker images'
-
-      # Command to run the integration test
-      - 'make integ'
+    # Command to run the integration test
+      - make integ
 artifacts:
   files:
     - '**/*'
diff --git a/buildspec_publish_dockerhub.yml b/buildspec_publish_dockerhub.yml
@@ -9,8 +9,11 @@ phases:
   build:
     commands:
       # Pull the image that we built and pushed in the `Build` stage
-      - 'ecs-cli pull amazon/aws-for-fluent-bit-test:latest'
-      - 'docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest amazon/aws-for-fluent-bit:latest'
+      - aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
+      - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"amd64"
+      - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"arm64"
+      - docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 amazon/aws-for-fluent-bit:amd64
+      - docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 amazon/aws-for-fluent-bit:arm64
 
       # List the docker images 
       - docker images

diff --git a/buildspec_publish_ecr.yml b/buildspec_publish_ecr.yml
@@ -9,9 +9,10 @@ phases:
   build:
     commands:
       # Pull the image that we built and pushed in the `Build` stage
-      - 'ecs-cli pull amazon/aws-for-fluent-bit-test:latest'
-      - 'docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest amazon/aws-for-fluent-bit:latest'
-
+      - aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
+      - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"amd64"
+      - docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"arm64"
+
       # List the docker images 
       - docker images
 

diff --git a/integ/integ.sh b/integ/integ.sh
@@ -3,12 +3,13 @@ export AWS_REGION="us-west-2"
 export PROJECT_ROOT="$(pwd)"
 
 test_cloudwatch() {
-	export LOG_GROUP_NAME="fluent-bit-integ-test"
+	export ARCHITECTURE=$(uname -m)
+	export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}"
 	# Tag is used to name the log stream; each test run has a unique (random) log stream name
 	export TAG=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 10)
 	docker-compose --file ./integ/test_cloudwatch/docker-compose.test.yml build
 	docker-compose --file ./integ/test_cloudwatch/docker-compose.test.yml up --abort-on-container-exit
-	sleep 10
+	sleep 120
 
 	# Creates a file as a flag for the validation failure
 	mkdir -p ./integ/out
@@ -20,7 +21,7 @@ test_cloudwatch() {
 }
 
 clean_cloudwatch() {
-	export LOG_GROUP_NAME="fluent-bit-integ-test"
+	export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}"
 	# Clean up resources that were created in the test
 	docker-compose --file ./integ/test_cloudwatch/docker-compose.clean.yml build
 	docker-compose --file ./integ/test_cloudwatch/docker-compose.clean.yml up --abort-on-container-exit
@@ -81,7 +82,6 @@ test_firehose() {
 clean_s3() {
 	validate_or_clean_s3 clean
 }
-
 if [ "${1}" = "cloudwatch" ]; then
 	export PLUGIN_UNDER_TEST="cloudwatch"
 	test_cloudwatch
@@ -163,4 +163,4 @@ fi
 
 if [ "${1}" = "delete" ]; then
 	source ./integ/resources/delete_test_resources.sh
-fi
+fi
diff --git a/integ/resources/create_test_resources.sh b/integ/resources/create_test_resources.sh
@@ -1,5 +1,10 @@
 #!/bin/bash
 
 # Deploys the CloudFormation template to create the stack and necessary resources- kinesis data stream, s3 bucket, and kinesis firehose delivery stream
-# Resource (stream, s3, delivery stream) names will start with the stack name "integ-test-fluent-bit"
-aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit --region us-west-2 --capabilities CAPABILITY_NAMED_IAM
+# Resource (stream, s3, delivery stream) names will start with the stack name followed by the corresponding architecture. "integ-test-fluent-bit-architecture"
+ARCHITECTURE=$(uname -m)
+if [ ARCHITECTURE=="x86_64" ] 
+then
+    ARCHITECTURE="x86-64"
+fi
+aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit-${ARCHITECTURE} --region us-west-2 --capabilities CAPABILITY_NAMED_IAM
diff --git a/integ/resources/delete_test_resources.sh b/integ/resources/delete_test_resources.sh
@@ -1,2 +1,7 @@
 # Delete the CloudFormation stack which created all the resources for running the integration test
-aws cloudformation delete-stack --stack-name integ-test-fluent-bit
+ARCHITECTURE=$(uname -m)
+if [ ARCHITECTURE=="x86_64" ] 
+then
+    ARCHITECTURE="x86-64"
+fi
+aws cloudformation delete-stack --stack-name integ-test-fluent-bit-${ARCHITECTURE}
diff --git a/integ/resources/setup_test_environment.sh b/integ/resources/setup_test_environment.sh
@@ -1,7 +1,12 @@
 #!/bin/bash
 
 # Using CloudFormation describe-stacks extracts the output values for kinesis stream and s3 bucket name, and sets them as environment variables
-stackOutputs=$(aws cloudformation describe-stacks --stack-name integ-test-fluent-bit --output text --query 'Stacks[0].Outputs[*].OutputValue')
+ARCHITECTURE=$(uname -m)
+if [ ARCHITECTURE=="x86_64" ] 
+then
+    ARCHITECTURE="x86-64"
+fi
+stackOutputs=$(aws cloudformation describe-stacks --stack-name integ-test-fluent-bit-${ARCHITECTURE} --output text --query 'Stacks[0].Outputs[*].OutputValue')
 read -r -a outputArray <<< "$stackOutputs"
 export FIREHOSE_STREAM="${outputArray[0]}"
 export KINESIS_STREAM="${outputArray[1]}"

diff --git a/integ/validate_cloudwatch/validator.py b/integ/validate_cloudwatch/validator.py
@@ -8,7 +8,7 @@
 
 client = boto3.client('logs', region_name=os.environ.get('AWS_REGION'))
 metrics_client = boto3.client("cloudwatch", region_name=os.environ["AWS_REGION"])
-start_time = datetime.utcnow() - timedelta(seconds=60)
+start_time = datetime.utcnow() - timedelta(seconds=600)
 end_time = datetime.utcnow()
 
 LOG_GROUP_NAME = os.environ.get('LOG_GROUP_NAME')
@@ -55,7 +55,7 @@ def validate_metric(test_name, metric_namespace, dim_key, dim_value, expected_sa
             return True
         attempts += 1
         print(f"No metrics yet. Sleeping before trying again. Attempt # {attempts}")
-        time.sleep(2)
+        time.sleep(10)
 
     sys.exit('TEST_FAILURE: failed to validate metric existence in CloudWatch')
 
@@ -102,4 +102,4 @@ def get_expected_metric_name():
 
 if success_case_1 and success_case_2 and success_case_emf:
     # if this file is still present, integ script will mark the test as a failure
-    os.remove("/out/cloudwatch-test")
+    os.remove("/out/cloudwatch-test")
diff --git a/scripts/publish.sh b/scripts/publish.sh
@@ -73,8 +73,11 @@ gamma_account_id="626332813196"
 
 DOCKER_HUB_SECRET="com.amazonaws.dockerhub.aws-for-fluent-bit.credentials"
 
+ARCHITECTURES=("amd64" "arm64")
+
 publish_to_docker_hub() {
 	DRY_RUN="${DRY_RUN:-true}"
+	export DOCKER_CLI_EXPERIMENTAL=enabled
 
 	username="$(aws secretsmanager get-secret-value --secret-id $DOCKER_HUB_SECRET --region us-west-2 | jq -r '.SecretString | fromjson.username')"
 	password="$(aws secretsmanager get-secret-value --secret-id $DOCKER_HUB_SECRET --region us-west-2 | jq -r '.SecretString | fromjson.password')"
@@ -90,13 +93,23 @@ publish_to_docker_hub() {
 
 	# Publish to DockerHub only if $DRY_RUN is set to false
 	if [[ "${DRY_RUN}" == "false" ]]; then
-		docker tag ${1} ${2}
-		docker push ${1}
-		docker push ${2}
+		for arch in "${ARCHITECTURES[@]}"
+		do	
+			docker tag ${1}:"$arch" ${1}:"${arch}"-${AWS_FOR_FLUENT_BIT_VERSION} 
+			docker push ${1}:"$arch"-${AWS_FOR_FLUENT_BIT_VERSION}  
+		done
+
+		create_manifest_list ${1} "latest"
+        create_manifest_list ${1} ${AWS_FOR_FLUENT_BIT_VERSION} 
+
 	else
-		echo "DRY_RUN: docker tag ${1} ${2}"
-		echo "DRY_RUN: docker push ${1}"
-		echo "DRY_RUN: docker push ${2}"
+		for arch in "${ARCHITECTURES[@]}"
+		do
+			echo "DRY_RUN: docker tag ${1}:${arch} ${1}:${arch}-${AWS_FOR_FLUENT_BIT_VERSION}"
+			echo "DRY_RUN: docker push ${1}:${arch}-${AWS_FOR_FLUENT_BIT_VERSION}"
+		done
+		echo "DRY_RUN: create manifest list ${1}:latest"
+		echo "DRY_RUN: create manifest list ${1}:${AWS_FOR_FLUENT_BIT_VERSION}"
 		echo "DRY_RUN is NOT set to 'false', skipping DockerHub update. Exiting..."
 	fi
 
@@ -173,11 +186,41 @@ verify_ssm() {
 	fi
 }
 
-push_to_ecr() {
-	docker tag ${1} ${2}
-	ecs-cli push ${2} --region ${3} --registry-id ${4}
+create_manifest_list() {
+
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+	tag=${2}
+
+	# TODO: Add a way to automatically generate arch images in manifest 
+	docker manifest create ${1}:${tag} \
+		${1}:arm64-${AWS_FOR_FLUENT_BIT_VERSION} \
+		${1}:amd64-${AWS_FOR_FLUENT_BIT_VERSION} 
+
+	for arch in "${ARCHITECTURES[@]}"
+	do
+    docker manifest annotate --arch "$arch" \
+		${1}:${tag} \
+		${1}:"$arch"-${AWS_FOR_FLUENT_BIT_VERSION} 
+	done 
+
+	# sanity check on the debug log.
+ 	docker manifest inspect ${1}:${tag}
+	docker manifest push ${1}:${tag}
+}
+
+push_image_ecr() {
+	account_id=${1}
+	region=${2}
+
+	for arch in "${ARCHITECTURES[@]}"
+	do
+		docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:"$arch" \
+			${account_id}.dkr.ecr.${region}.amazonaws.com/aws-for-fluent-bit:"$arch"-${AWS_FOR_FLUENT_BIT_VERSION}
+    	docker push ${account_id}.dkr.ecr.${region}.amazonaws.com/aws-for-fluent-bit:"$arch"-${AWS_FOR_FLUENT_BIT_VERSION}
+    done
 }
 
+# TODO: remove dependency on ecs-cli
 pull_ecr() {
 	ecs-cli pull ${1} --region ${2}
 }
@@ -189,8 +232,17 @@ make_repo_public() {
 publish_ecr() {
 	region=${1}
 	account_id=${2}
-	push_to_ecr amazon/aws-for-fluent-bit:latest aws-for-fluent-bit:latest ${region} ${account_id}
-	push_to_ecr amazon/aws-for-fluent-bit:latest "aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION}" ${region} ${account_id}
+	echo $region
+	echo $account_id
+
+	aws ecr get-login-password --region ${region}| docker login --username AWS --password-stdin ${account_id}.dkr.ecr.${region}.amazonaws.com
+	aws ecr create-repository --repository-name aws-for-fluent-bit --image-scanning-configuration scanOnPush=true --region ${region}  || true
+
+	push_image_ecr ${account_id} ${region}	
+
+    create_manifest_list ${account_id}.dkr.ecr.${region}.amazonaws.com/aws-for-fluent-bit ${AWS_FOR_FLUENT_BIT_VERSION}
+	create_manifest_list ${account_id}.dkr.ecr.${region}.amazonaws.com/aws-for-fluent-bit "latest"
+
 	make_repo_public ${region}
 }
 
@@ -204,14 +256,15 @@ verify_ecr() {
 		endpoint=${endpoint}.cn
 	fi
 
-	pull_ecr ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:latest ${region}
+	aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${account_id}.dkr.ecr.${region}.amazonaws.com
+	docker pull ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:latest
 	sha1=$(docker inspect --format='{{index .RepoDigests 0}}' ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:latest)
 
 	if [ "${is_sync_task}" = "true" ]; then
 		pull_ecr ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION_DOCKERHUB} ${region}
 		sha2=$(docker inspect --format='{{index .RepoDigests 0}}' ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION_DOCKERHUB})
 	else
-		pull_ecr ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION} ${region}
+		docker pull ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION}
 		sha2=$(docker inspect --format='{{index .RepoDigests 0}}' ${account_id}.dkr.ecr.${region}.${endpoint}/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION})
 	fi
 
@@ -259,7 +312,7 @@ match_two_sha() {
 
 if [ "${1}" = "publish" ]; then
 	if [ "${2}" = "dockerhub" ]; then
-		publish_to_docker_hub amazon/aws-for-fluent-bit:latest amazon/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION}
+		publish_to_docker_hub amazon/aws-for-fluent-bit 
 	fi
 
 	if [ "${2}" = "aws" ]; then
@@ -417,7 +470,7 @@ fi
 # Following scripts will be called only from the CI/CD pipeline
 if [ "${1}" = "cicd-publish" ]; then
 	if [ "${2}" = "dockerhub" ]; then
-		publish_to_docker_hub amazon/aws-for-fluent-bit:latest amazon/aws-for-fluent-bit:${AWS_FOR_FLUENT_BIT_VERSION}
+		publish_to_docker_hub amazon/aws-for-fluent-bit  
 	elif [ "${2}" = "us-gov-east-1" ] || [ "${2}" = "us-gov-west-1" ]; then
 		for region in ${gov_regions}; do
 			sync_latest_image ${region} ${gov_regions_account_id}
@@ -496,4 +549,4 @@ if [ "${1}" = "cicd-verify-ssm" ]; then
 			verify_ssm ${region}
 		done
 	fi
-fi
+fi