aws · bryce-shang · Mar 8, 2021 · Mar 1, 2021 · Mar 1, 2021 · Mar 3, 2021
diff --git a/tests/ci/docker_images/linux-x86/README.md b/tests/ci/docker_images/linux-x86/README.md
@@ -1,18 +1,15 @@
 # Prerequistes
-EC2 x86-64 Ubuntu 19.10 host with Docker server installed (Docker instructions
-taken from
-[Docker docs](https://docs.docker.com/install/linux/docker-ce/ubuntu/)):
-```
-$ sudo apt-get update
-$ sudo apt-get install -y awscli apt-transport-https ca-certificates curl gnupg-agent software-properties-common
-$ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-$ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-$ sudo apt-get update
-$ sudo apt-get install -y docker-ce
-$ sudo usermod -aG docker ${USER}
-# Log in and out
-```
 
-Build images locally with `build_images.sh`, to push to the main repository run
-`push_images.sh`. To push to your own repository pass in a complete ECS url
-`push_images.sh ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${REPOSITORY}`. 
+* [Create GitHub Personal Access Token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token)
+  * Note: This token ONLY needs ['read:packages' permission](https://docs.github.com/en/packages/learn-github-packages/about-github-packages#authenticating-to-github-packages), and should be deleted from GitHub account after docker image build.
+  * This token is needed when pulling images from 'docker.pkg.github.com'.
+
+# Usage
+
+## Docker image build and pull.
+Build images locally with `build_images.sh`, which pulls Docker images from Docker hub.  
+Pull images from 'docker.pkg.github.com' with `pull_github_pkg.sh`.
+
+## Docker image push.
+To push to the ECR repository of default AWS account, run `push_images.sh`.  
+To push to the ECR repository pass in a complete ECS url `push_images.sh ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${REPOSITORY}`. 
diff --git a/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh b/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh
@@ -0,0 +1,74 @@
+#!/bin/bash -ex
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+parent_dir="$(dirname -- "$(realpath -- "$file_name")")"
+# shellcheck source=../common.sh
+source "${parent_dir}/common.sh"
+
+#################################
+# Set up environment variables #
+#################################
+
+if [[ -z "${ECS_REPO+x}" || -z "${ECS_REPO}" ]]; then
+  export ECS_REPO="620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86"
+fi
+
+if [[ -z "${GITHUB_REPO_OWNER+x}" || -z "${GITHUB_REPO_OWNER}" ]]; then
+  # The owner of the GitHub personal access token. Required when login 'docker.pkg.github.com'.
+  export GITHUB_REPO_OWNER='awslabs'
+fi
+
+# Environment variables used by AWS-C-CAL
+# CI of AWS-C-CAL https://github.com/awslabs/aws-c-cal/blob/905b0fd0ff2ca0d86c7e49b8ef4b636bac4600ea/.github/workflows/ci.yml
+# The CI uses Docker images provided by https://github.com/awslabs/aws-crt-builder
+BUILDER_VERSION='v0.7.2'
+BASE_AWS_CRT_DOCKER_IMG_PREFIX='docker.pkg.github.com/awslabs/aws-crt-builder'
+AWS_CRT_MANY_LINUX_SUFFIXES=('aws-crt-manylinux1-x64' 'aws-crt-manylinux1-x86' 'aws-crt-manylinux2014-x64' 'aws-crt-manylinux2014-x86')
+AWS_CRT_AL2_X64_SUFFIX='aws-crt-al2-x64'
+AWS_CRT_AL2_X64="${BASE_AWS_CRT_DOCKER_IMG_PREFIX}/${AWS_CRT_AL2_X64_SUFFIX}:${BUILDER_VERSION}"
+AWS_CRT_UBUNTU_16_X64_SUFFIX='aws-crt-ubuntu-16-x64'
+AWS_CRT_UBUNTU_16_X64="${BASE_AWS_CRT_DOCKER_IMG_PREFIX}/${AWS_CRT_UBUNTU_16_X64_SUFFIX}:${BUILDER_VERSION}"
+
+####################################
+# Migrate Docker images to AWS ECR #
+####################################
+
+function login_github_docker_pkg() {
+  if [[ -z "${GITHUB_READ_PKG_ACCESS_TOKEN+x}" || -z "${GITHUB_READ_PKG_ACCESS_TOKEN}" ]]; then
+    # GITHUB_READ_PKG_ACCESS_TOKEN is required when pulling images from 'docker.pkg.github.com'.
+    # When creating GitHub personal access token, make sure ONLY "read:packages" is granted.
+    # https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token
+    # https://docs.github.com/en/packages/learn-github-packages/about-github-packages#authenticating-to-github-packages
+    echo "GITHUB_READ_PKG_ACCESS_TOKEN is not defined or empty."
+    exit 1
+  fi
+  echo "${GITHUB_READ_PKG_ACCESS_TOKEN}" | docker login docker.pkg.github.com -u "${GITHUB_REPO_OWNER}" --password-stdin
+}
+
+function pull_aws_crt_docker_img() {
+  # Pull aws-crt-manylinux*:v0.7.2
+  for i in "${AWS_CRT_MANY_LINUX_SUFFIXES[@]}"; do
+    docker_img="${BASE_AWS_CRT_DOCKER_IMG_PREFIX}/${i}:${BUILDER_VERSION}"
+    docker pull "${docker_img}"
+  done
+  # Pull aws-crt-al2-x64:v0.7.2
+  docker pull "${AWS_CRT_AL2_X64}"
+  # Pull aws-crt-ubuntu-16-x64:v0.7.2
+  docker pull "${AWS_CRT_UBUNTU_16_X64}"
+}
+
+function push_aws_crt_docker_img() {
+  # Push aws-crt-manylinux*_v0.7.2
+  for i in "${AWS_CRT_MANY_LINUX_SUFFIXES[@]}"; do
+    source_docker_img="${BASE_AWS_CRT_DOCKER_IMG_PREFIX}/${i}:${BUILDER_VERSION}"
+    target_docker_img="${ECS_REPO}:${i}_${BUILDER_VERSION}"
+    tag_and_push_img "${source_docker_img}" "${target_docker_img}"
+  done
+  # Push aws-crt-al2-x64_v0.7.2
+  target_aws_art_al2_x64="${ECS_REPO}:${AWS_CRT_AL2_X64_SUFFIX}_${BUILDER_VERSION}"
+  tag_and_push_img "${AWS_CRT_AL2_X64}" "${target_aws_art_al2_x64}"
+  # Push aws-crt-ubuntu-16-x64_v0.7.2
+  target_aws_art_ubuntu_16_x64="${ECS_REPO}:${AWS_CRT_UBUNTU_16_X64_SUFFIX}_${BUILDER_VERSION}"
+  tag_and_push_img "${AWS_CRT_UBUNTU_16_X64}" "${target_aws_art_ubuntu_16_x64}"
+}
diff --git a/tests/ci/docker_images/linux-x86/common.sh b/tests/ci/docker_images/linux-x86/common.sh
@@ -0,0 +1,27 @@
+#!/bin/bash -ex
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+function validate_input() {
+  key="${1}"
+  value="${2}"
+  if [[ -z "${value}" ]]; then
+    echo "The value of ${key} is empty."
+    exit 1
+  fi
+}
+
+# Tag images with date to help find old images, CodeBuild uses the latest tag and gets updated automatically.
+function tag_and_push_img() {
+  source="${1}"
+  validate_input 'source' "${source}"
+  target="${2}"
+  validate_input 'target' "${target}"
+  img_push_date=$(date +%Y-%m-%d)
+  docker_img_with_latest="${target}_latest"
+  docker_img_with_date="${target}_${img_push_date}"
+  docker tag "${source}" "${docker_img_with_latest}"
+  docker tag "${source}" "${docker_img_with_date}"
+  docker push "${docker_img_with_latest}"
+  docker push "${docker_img_with_date}"
+}
diff --git a/tests/ci/docker_images/linux-x86/pull_github_pkg.sh b/tests/ci/docker_images/linux-x86/pull_github_pkg.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -ex
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Note: this file is reserved for pulling Docker images from 'docker.pkg.github.com', which needs external credential.
+# Pull Docker images used by https://github.com/awslabs/aws-c-cal
+# These Docker images are hosted in 'docker.pkg.github.com', which needs GitHub personal access token.
+source ./aws-crt/aws-crt-common.sh
+login_github_docker_pkg
+pull_aws_crt_docker_img
diff --git a/tests/ci/docker_images/linux-x86/push_images.sh b/tests/ci/docker_images/linux-x86/push_images.sh
@@ -2,6 +2,8 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+source ./common.sh
+
 if [ -z ${1+x} ]; then
   ECS_REPO="620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86"
 else
@@ -12,73 +14,24 @@ echo "Uploading docker images to ${ECS_REPO}."
 
 $(aws ecr get-login --no-include-email)
 
-# # Tag images with date to help find old images, CodeBuild uses the latest tag and gets updated automatically
-docker tag ubuntu-16.04:gcc-5x ${ECS_REPO}:ubuntu-16.04_gcc-5x_`date +%Y-%m-%d`
-docker tag ubuntu-16.04:gcc-5x ${ECS_REPO}:ubuntu-16.04_gcc-5x_latest
-docker push ${ECS_REPO}:ubuntu-16.04_gcc-5x_latest
-docker push ${ECS_REPO}:ubuntu-16.04_gcc-5x_`date +%Y-%m-%d`
-
-docker tag ubuntu-18.04:gcc-7x ${ECS_REPO}:ubuntu-18.04_gcc-7x_`date +%Y-%m-%d`
-docker tag ubuntu-18.04:gcc-7x ${ECS_REPO}:ubuntu-18.04_gcc-7x_latest
-docker push ${ECS_REPO}:ubuntu-18.04_gcc-7x_latest
-docker push ${ECS_REPO}:ubuntu-18.04_gcc-7x_`date +%Y-%m-%d`
-
-docker tag ubuntu-18.04:clang-6x ${ECS_REPO}:ubuntu-18.04_clang-6x_`date +%Y-%m-%d`
-docker tag ubuntu-18.04:clang-6x ${ECS_REPO}:ubuntu-18.04_clang-6x_latest
-docker push ${ECS_REPO}:ubuntu-18.04_clang-6x_latest
-docker push ${ECS_REPO}:ubuntu-18.04_clang-6x_`date +%Y-%m-%d`
-
-docker tag ubuntu-19.04:gcc-8x ${ECS_REPO}:ubuntu-19.04_gcc-8x_`date +%Y-%m-%d`
-docker tag ubuntu-19.04:gcc-8x ${ECS_REPO}:ubuntu-19.04_gcc-8x_latest
-docker push ${ECS_REPO}:ubuntu-19.04_gcc-8x_latest
-docker push ${ECS_REPO}:ubuntu-19.04_gcc-8x_`date +%Y-%m-%d`
-
-docker tag ubuntu-19.04:clang-8x ${ECS_REPO}:ubuntu-19.04_clang-8x_`date +%Y-%m-%d`
-docker tag ubuntu-19.04:clang-8x ${ECS_REPO}:ubuntu-19.04_clang-8x_latest
-docker push ${ECS_REPO}:ubuntu-19.04_clang-8x_latest
-docker push ${ECS_REPO}:ubuntu-19.04_clang-8x_`date +%Y-%m-%d`
-
-docker tag ubuntu-19.10:clang-9x ${ECS_REPO}:ubuntu-19.10_clang-9x_`date +%Y-%m-%d`
-docker tag ubuntu-19.10:clang-9x ${ECS_REPO}:ubuntu-19.10_clang-9x_latest
-docker push ${ECS_REPO}:ubuntu-19.10_clang-9x_latest
-docker push ${ECS_REPO}:ubuntu-19.10_clang-9x_`date +%Y-%m-%d`
-
-docker tag ubuntu-19.10:sanitizer ${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer_`date +%Y-%m-%d`
-docker tag ubuntu-19.10:sanitizer ${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer_latest
-docker push ${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer_latest
-docker push ${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer_`date +%Y-%m-%d`
-
-docker tag ubuntu-20.04:clang-10x ${ECS_REPO}:ubuntu-20.04_clang-10x_`date +%Y-%m-%d`
-docker tag ubuntu-20.04:clang-10x ${ECS_REPO}:ubuntu-20.04_clang-10x_latest
-docker push ${ECS_REPO}:ubuntu-20.04_clang-10x_latest
-docker push ${ECS_REPO}:ubuntu-20.04_clang-10x_`date +%Y-%m-%d`
-
-docker tag centos-7:gcc-4x ${ECS_REPO}:centos-7_gcc-4x_`date +%Y-%m-%d`
-docker tag centos-7:gcc-4x ${ECS_REPO}:centos-7_gcc-4x_latest
-docker push ${ECS_REPO}:centos-7_gcc-4x_latest
-docker push ${ECS_REPO}:centos-7_gcc-4x_`date +%Y-%m-%d`
-
-docker tag amazonlinux-2:gcc-7x ${ECS_REPO}:amazonlinux-2_gcc-7x_`date +%Y-%m-%d`
-docker tag amazonlinux-2:gcc-7x ${ECS_REPO}:amazonlinux-2_gcc-7x_latest
-docker push ${ECS_REPO}:amazonlinux-2_gcc-7x_latest
-docker push ${ECS_REPO}:amazonlinux-2_gcc-7x_`date +%Y-%m-%d`
-
-docker tag amazonlinux-2:gcc-7x-intel-sde ${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde_`date +%Y-%m-%d`
-docker tag amazonlinux-2:gcc-7x-intel-sde ${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde_latest
-docker push ${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde_latest
-docker push ${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde_`date +%Y-%m-%d`
-
-docker tag fedora-31:clang-9x ${ECS_REPO}:fedora-31_clang-9x_`date +%Y-%m-%d`
-docker tag fedora-31:clang-9x ${ECS_REPO}:fedora-31_clang-9x_latest
-docker push ${ECS_REPO}:fedora-31_clang-9x_latest
-docker push ${ECS_REPO}:fedora-31_clang-9x_`date +%Y-%m-%d`
-
-docker tag integration:s2n ${ECS_REPO}:s2n_integration_clang-9x_`date +%Y-%m-%d`
-docker tag integration:s2n ${ECS_REPO}:s2n_integration_clang-9x_latest
-docker push ${ECS_REPO}:s2n_integration_clang-9x_latest
-docker push ${ECS_REPO}:s2n_integration_clang-9x_`date +%Y-%m-%d`
-
-docker tag ubuntu-20.04:clang-10x_formal-verification ${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification_`date +%Y-%m-%d`
-docker tag ubuntu-20.04:clang-10x_formal-verification ${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification_latest
-docker push ${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification_latest
-docker push ${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification_`date +%Y-%m-%d`
+# Tag images with date to help find old images, CodeBuild uses the latest tag and gets updated automatically
+tag_and_push_img 'ubuntu-16.04:gcc-5x' "${ECS_REPO}:ubuntu-16.04_gcc-5x"
+tag_and_push_img 'ubuntu-18.04:gcc-7x' "${ECS_REPO}:ubuntu-18.04_gcc-7x"
+tag_and_push_img 'ubuntu-18.04:clang-6x' "${ECS_REPO}:ubuntu-18.04_clang-6x"
+tag_and_push_img 'ubuntu-19.04:gcc-8x' "${ECS_REPO}:ubuntu-19.04_gcc-8x"
+tag_and_push_img 'ubuntu-19.04:clang-8x' "${ECS_REPO}:ubuntu-19.04_clang-8x"
+tag_and_push_img 'ubuntu-19.10:clang-9x' "${ECS_REPO}:ubuntu-19.10_clang-9x"
+tag_and_push_img 'ubuntu-19.10:sanitizer' "${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer"
+tag_and_push_img 'ubuntu-20.04:clang-10x' "${ECS_REPO}:ubuntu-20.04_clang-10x"
+tag_and_push_img 'centos-7:gcc-4x' "${ECS_REPO}:centos-7_gcc-4x"
+tag_and_push_img 'amazonlinux-2:gcc-7x' "${ECS_REPO}:amazonlinux-2_gcc-7x"
+tag_and_push_img 'amazonlinux-2:gcc-7x-intel-sde' "${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde"
+tag_and_push_img 'fedora-31:clang-9x' "${ECS_REPO}:fedora-31_clang-9x"
+tag_and_push_img 'integration:s2n' "${ECS_REPO}:s2n_integration_clang-9x"
+tag_and_push_img 'ubuntu-20.04:clang-10x_formal-verification' "${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification"
+
+#################################
+# Push images used by aws-c-cal #
+#################################
+source ./aws-crt/aws-crt-common.sh
+push_aws_crt_docker_img