From 26ae6f8aca24c21c199b0a1d7513ac9eafb9f4ce Mon Sep 17 00:00:00 2001 From: Bryce Shang Date: Mon, 1 Mar 2021 14:18:14 -0800 Subject: [PATCH 1/7] Pre-pull aws crt Docker images. --- tests/ci/docker_images/linux-x86/README.md | 29 +++--- .../linux-x86/aws-crt/aws-crt-common.sh | 78 ++++++++++++++++ tests/ci/docker_images/linux-x86/common.sh | 27 ++++++ .../linux-x86/pull_github_pkg.sh | 10 ++ .../ci/docker_images/linux-x86/push_images.sh | 93 +++++-------------- 5 files changed, 151 insertions(+), 86 deletions(-) create mode 100755 tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh create mode 100755 tests/ci/docker_images/linux-x86/common.sh create mode 100755 tests/ci/docker_images/linux-x86/pull_github_pkg.sh diff --git a/tests/ci/docker_images/linux-x86/README.md b/tests/ci/docker_images/linux-x86/README.md index dc6bd663fd..d6fbdc4576 100644 --- a/tests/ci/docker_images/linux-x86/README.md +++ b/tests/ci/docker_images/linux-x86/README.md @@ -1,18 +1,15 @@ # Prerequistes -EC2 x86-64 Ubuntu 19.10 host with Docker server installed (Docker instructions -taken from -[Docker docs](https://docs.docker.com/install/linux/docker-ce/ubuntu/)): -``` -$ sudo apt-get update -$ sudo apt-get install -y awscli apt-transport-https ca-certificates curl gnupg-agent software-properties-common -$ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -$ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -$ sudo apt-get update -$ sudo apt-get install -y docker-ce -$ sudo usermod -aG docker ${USER} -# Log in and out -``` -Build images locally with `build_images.sh`, to push to the main repository run -`push_images.sh`. To push to your own repository pass in a complete ECS url -`push_images.sh ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${REPOSITORY}`. +* [Create GitHub Personal Access Token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) + * Note: This token ONLY needs ['read:packages' permission](https://docs.github.com/en/packages/learn-github-packages/about-github-packages#authenticating-to-github-packages), and should be deleted from GitHub account after docker image build. + * This token is needed when pulling images from 'docker.pkg.github.com'. + +# Usage + +## Docker image build and pull. +Build images locally with `build_images.sh`, which pulls Docker images from Docker hub. +Pull images from 'docker.pkg.github.com' with `pull_github_pkg.sh`. + +## Docker image push. +To push to the ECR repository of default AWS account, run `push_images.sh`. +To push to the ECR repository pass in a complete ECS url `push_images.sh ${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${REPOSITORY}`. diff --git a/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh b/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh new file mode 100755 index 0000000000..e725610198 --- /dev/null +++ b/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh @@ -0,0 +1,78 @@ +#!/bin/bash -ex +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +parent_dir="$(dirname -- "$(realpath -- "$file_name")")" +# shellcheck source=../common.sh +source "${parent_dir}/common.sh" + +################################# +# Sett up environment variables # +################################# + +if [[ -z "${ECS_REPO+x}" || -z "${ECS_REPO}" ]]; then + export ECS_REPO="620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86" +fi + +if [[ -z "${GITHUB_REPO_OWNER+x}" || -z "${GITHUB_REPO_OWNER}" ]]; then + # The owner of the GitHub personal access token. Required when login 'docker.pkg.github.com'. + export GITHUB_REPO_OWNER='awslabs' +fi + +# Environment variables used by AWS-C-CAL +# https://github.com/awslabs/aws-c-cal/blob/905b0fd0ff2ca0d86c7e49b8ef4b636bac4600ea/.github/workflows/ci.yml +BUILDER_VERSION='v0.7.2' +BASE_AWS_CRT_DOCKER_IMG_PREFIX='docker.pkg.github.com/awslabs/aws-crt-builder' +AWS_CRT_MANY_LINUX_SUFFIXES=('aws-crt-manylinux1-x64' 'aws-crt-manylinux1-x86' 'aws-crt-manylinux2014-x64' 'aws-crt-manylinux2014-x86') +AWS_CRT_AL2_X64_SUFFIX='aws-crt-al2-x64' +AWS_CRT_AL2_X64="${BASE_AWS_CRT_DOCKER_IMG_PREFIX}/${AWS_CRT_AL2_X64_SUFFIX}:${BUILDER_VERSION}" +AWS_CRT_UBUNTU_16_X64_SUFFIX='aws-crt-ubuntu-16-x64' +AWS_CRT_UBUNTU_16_X64="${BASE_AWS_CRT_DOCKER_IMG_PREFIX}/${AWS_CRT_UBUNTU_16_X64_SUFFIX}:${BUILDER_VERSION}" + +#################################### +# Migrate Docker images to AWS ECR # +#################################### + +function login_github_docker_pkg() { + jq_installed=$(command -v jq) + if [[ -z "${jq_installed}" ]]; then + # Install 'jq' for json response parser. + apt-get update -y && apt-get install -y jq + fi + if [[ -z "${GITHUB_READ_PKG_ACCESS_TOKEN+x}" || -z "${GITHUB_READ_PKG_ACCESS_TOKEN}" ]]; then + # GITHUB_READ_PKG_ACCESS_TOKEN is required when pulling images from 'docker.pkg.github.com'. + # When creating GitHub personal access token, make sure ONLY "read:packages" is granted. + # https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token + # https://docs.github.com/en/packages/learn-github-packages/about-github-packages#authenticating-to-github-packages + echo "GITHUB_READ_PKG_ACCESS_TOKEN is not defined or empty." + exit 1 + fi + echo "${GITHUB_READ_PKG_ACCESS_TOKEN}" | docker login docker.pkg.github.com -u "${GITHUB_REPO_OWNER}" --password-stdin +} + +function pull_aws_crt_docker_img() { + # Pull aws-crt-manylinux*:v0.7.2 + for i in "${AWS_CRT_MANY_LINUX_SUFFIXES[@]}"; do + docker_img="${BASE_AWS_CRT_DOCKER_IMG_PREFIX}/${i}:${BUILDER_VERSION}" + docker pull "${docker_img}" + done + # Pull aws-crt-al2-x64:v0.7.2 + docker pull "${AWS_CRT_AL2_X64}" + # Pull aws-crt-ubuntu-16-x64:v0.7.2 + docker pull "${AWS_CRT_UBUNTU_16_X64}" +} + +function push_aws_crt_docker_img() { + # Push aws-crt-manylinux*_v0.7.2 + for i in "${AWS_CRT_MANY_LINUX_SUFFIXES[@]}"; do + source_docker_img="${BASE_AWS_CRT_DOCKER_IMG_PREFIX}/${i}:${BUILDER_VERSION}" + target_docker_img="${ECS_REPO}:${i}_${BUILDER_VERSION}" + tag_and_push_img "${source_docker_img}" "${target_docker_img}" + done + # Push aws-crt-al2-x64_v0.7.2 + target_aws_art_al2_x64="${ECS_REPO}:${AWS_CRT_AL2_X64_SUFFIX}_${BUILDER_VERSION}" + tag_and_push_img "${AWS_CRT_AL2_X64}" "${target_aws_art_al2_x64}" + # Push aws-crt-ubuntu-16-x64_v0.7.2 + target_aws_art_ubuntu_16_x64="${ECS_REPO}:${AWS_CRT_UBUNTU_16_X64_SUFFIX}_${BUILDER_VERSION}" + tag_and_push_img "${AWS_CRT_UBUNTU_16_X64}" "${target_aws_art_ubuntu_16_x64}" +} diff --git a/tests/ci/docker_images/linux-x86/common.sh b/tests/ci/docker_images/linux-x86/common.sh new file mode 100755 index 0000000000..ee69396897 --- /dev/null +++ b/tests/ci/docker_images/linux-x86/common.sh @@ -0,0 +1,27 @@ +#!/bin/bash -ex +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +function validate_input() { + key="${1}" + value="${2}" + if [[ -z "${value}" ]]; then + echo "The value of ${key} is empty." + exit 1 + fi +} + +# Tag images with date to help find old images, CodeBuild uses the latest tag and gets updated automatically. +function tag_and_push_img() { + source="${1}" + validate_input 'source' "${source}" + target="${2}" + validate_input 'target' "${target}" + img_push_date=$(date +%Y-%m-%d) + docker_img_with_latest="${target}_latest" + docker_img_with_date="${target}_${img_push_date}" + docker tag "${source}" "${docker_img_with_latest}" + docker tag "${source}" "${docker_img_with_date}" + docker push "${docker_img_with_latest}" + docker push "${docker_img_with_date}" +} diff --git a/tests/ci/docker_images/linux-x86/pull_github_pkg.sh b/tests/ci/docker_images/linux-x86/pull_github_pkg.sh new file mode 100755 index 0000000000..b7ccb2d5f5 --- /dev/null +++ b/tests/ci/docker_images/linux-x86/pull_github_pkg.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ex +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Note: this file is reserved for pulling Docker images from 'docker.pkg.github.com', which needs external credential. +# Pull Docker images used by https://github.com/awslabs/aws-c-cal +# These Docker images are hosted in 'docker.pkg.github.com', which needs GitHub personal access token. +source ./aws-crt/aws-crt-common.sh +login_github_docker_pkg +pull_aws_crt_docker_img diff --git a/tests/ci/docker_images/linux-x86/push_images.sh b/tests/ci/docker_images/linux-x86/push_images.sh index 28bdc317d7..76a194aa00 100755 --- a/tests/ci/docker_images/linux-x86/push_images.sh +++ b/tests/ci/docker_images/linux-x86/push_images.sh @@ -2,6 +2,8 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +source ./common.sh + if [ -z ${1+x} ]; then ECS_REPO="620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86" else @@ -12,73 +14,24 @@ echo "Uploading docker images to ${ECS_REPO}." $(aws ecr get-login --no-include-email) -# # Tag images with date to help find old images, CodeBuild uses the latest tag and gets updated automatically -docker tag ubuntu-16.04:gcc-5x ${ECS_REPO}:ubuntu-16.04_gcc-5x_`date +%Y-%m-%d` -docker tag ubuntu-16.04:gcc-5x ${ECS_REPO}:ubuntu-16.04_gcc-5x_latest -docker push ${ECS_REPO}:ubuntu-16.04_gcc-5x_latest -docker push ${ECS_REPO}:ubuntu-16.04_gcc-5x_`date +%Y-%m-%d` - -docker tag ubuntu-18.04:gcc-7x ${ECS_REPO}:ubuntu-18.04_gcc-7x_`date +%Y-%m-%d` -docker tag ubuntu-18.04:gcc-7x ${ECS_REPO}:ubuntu-18.04_gcc-7x_latest -docker push ${ECS_REPO}:ubuntu-18.04_gcc-7x_latest -docker push ${ECS_REPO}:ubuntu-18.04_gcc-7x_`date +%Y-%m-%d` - -docker tag ubuntu-18.04:clang-6x ${ECS_REPO}:ubuntu-18.04_clang-6x_`date +%Y-%m-%d` -docker tag ubuntu-18.04:clang-6x ${ECS_REPO}:ubuntu-18.04_clang-6x_latest -docker push ${ECS_REPO}:ubuntu-18.04_clang-6x_latest -docker push ${ECS_REPO}:ubuntu-18.04_clang-6x_`date +%Y-%m-%d` - -docker tag ubuntu-19.04:gcc-8x ${ECS_REPO}:ubuntu-19.04_gcc-8x_`date +%Y-%m-%d` -docker tag ubuntu-19.04:gcc-8x ${ECS_REPO}:ubuntu-19.04_gcc-8x_latest -docker push ${ECS_REPO}:ubuntu-19.04_gcc-8x_latest -docker push ${ECS_REPO}:ubuntu-19.04_gcc-8x_`date +%Y-%m-%d` - -docker tag ubuntu-19.04:clang-8x ${ECS_REPO}:ubuntu-19.04_clang-8x_`date +%Y-%m-%d` -docker tag ubuntu-19.04:clang-8x ${ECS_REPO}:ubuntu-19.04_clang-8x_latest -docker push ${ECS_REPO}:ubuntu-19.04_clang-8x_latest -docker push ${ECS_REPO}:ubuntu-19.04_clang-8x_`date +%Y-%m-%d` - -docker tag ubuntu-19.10:clang-9x ${ECS_REPO}:ubuntu-19.10_clang-9x_`date +%Y-%m-%d` -docker tag ubuntu-19.10:clang-9x ${ECS_REPO}:ubuntu-19.10_clang-9x_latest -docker push ${ECS_REPO}:ubuntu-19.10_clang-9x_latest -docker push ${ECS_REPO}:ubuntu-19.10_clang-9x_`date +%Y-%m-%d` - -docker tag ubuntu-19.10:sanitizer ${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer_`date +%Y-%m-%d` -docker tag ubuntu-19.10:sanitizer ${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer_latest -docker push ${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer_latest -docker push ${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer_`date +%Y-%m-%d` - -docker tag ubuntu-20.04:clang-10x ${ECS_REPO}:ubuntu-20.04_clang-10x_`date +%Y-%m-%d` -docker tag ubuntu-20.04:clang-10x ${ECS_REPO}:ubuntu-20.04_clang-10x_latest -docker push ${ECS_REPO}:ubuntu-20.04_clang-10x_latest -docker push ${ECS_REPO}:ubuntu-20.04_clang-10x_`date +%Y-%m-%d` - -docker tag centos-7:gcc-4x ${ECS_REPO}:centos-7_gcc-4x_`date +%Y-%m-%d` -docker tag centos-7:gcc-4x ${ECS_REPO}:centos-7_gcc-4x_latest -docker push ${ECS_REPO}:centos-7_gcc-4x_latest -docker push ${ECS_REPO}:centos-7_gcc-4x_`date +%Y-%m-%d` - -docker tag amazonlinux-2:gcc-7x ${ECS_REPO}:amazonlinux-2_gcc-7x_`date +%Y-%m-%d` -docker tag amazonlinux-2:gcc-7x ${ECS_REPO}:amazonlinux-2_gcc-7x_latest -docker push ${ECS_REPO}:amazonlinux-2_gcc-7x_latest -docker push ${ECS_REPO}:amazonlinux-2_gcc-7x_`date +%Y-%m-%d` - -docker tag amazonlinux-2:gcc-7x-intel-sde ${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde_`date +%Y-%m-%d` -docker tag amazonlinux-2:gcc-7x-intel-sde ${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde_latest -docker push ${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde_latest -docker push ${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde_`date +%Y-%m-%d` - -docker tag fedora-31:clang-9x ${ECS_REPO}:fedora-31_clang-9x_`date +%Y-%m-%d` -docker tag fedora-31:clang-9x ${ECS_REPO}:fedora-31_clang-9x_latest -docker push ${ECS_REPO}:fedora-31_clang-9x_latest -docker push ${ECS_REPO}:fedora-31_clang-9x_`date +%Y-%m-%d` - -docker tag integration:s2n ${ECS_REPO}:s2n_integration_clang-9x_`date +%Y-%m-%d` -docker tag integration:s2n ${ECS_REPO}:s2n_integration_clang-9x_latest -docker push ${ECS_REPO}:s2n_integration_clang-9x_latest -docker push ${ECS_REPO}:s2n_integration_clang-9x_`date +%Y-%m-%d` - -docker tag ubuntu-20.04:clang-10x_formal-verification ${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification_`date +%Y-%m-%d` -docker tag ubuntu-20.04:clang-10x_formal-verification ${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification_latest -docker push ${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification_latest -docker push ${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification_`date +%Y-%m-%d` +# Tag images with date to help find old images, CodeBuild uses the latest tag and gets updated automatically +tag_and_push_img 'ubuntu-16.04:gcc-5x' "${ECS_REPO}:ubuntu-16.04_gcc-5x" +tag_and_push_img 'ubuntu-18.04:gcc-7x' "${ECS_REPO}:ubuntu-18.04_gcc-7x" +tag_and_push_img 'ubuntu-18.04:clang-6x' "${ECS_REPO}:ubuntu-18.04_clang-6x" +tag_and_push_img 'ubuntu-19.04:gcc-8x' "${ECS_REPO}:ubuntu-19.04_gcc-8x" +tag_and_push_img 'ubuntu-19.04:clang-8x' "${ECS_REPO}:ubuntu-19.04_clang-8x" +tag_and_push_img 'ubuntu-19.10:clang-9x' "${ECS_REPO}:ubuntu-19.10_clang-9x" +tag_and_push_img 'ubuntu-19.10:sanitizer' "${ECS_REPO}:ubuntu-19.10_clang-9x_sanitizer" +tag_and_push_img 'ubuntu-20.04:clang-10x' "${ECS_REPO}:ubuntu-20.04_clang-10x" +tag_and_push_img 'centos-7:gcc-4x' "${ECS_REPO}:centos-7_gcc-4x" +tag_and_push_img 'amazonlinux-2:gcc-7x' "${ECS_REPO}:amazonlinux-2_gcc-7x" +tag_and_push_img 'amazonlinux-2:gcc-7x-intel-sde' "${ECS_REPO}:amazonlinux-2_gcc-7x_intel-sde" +tag_and_push_img 'fedora-31:clang-9x' "${ECS_REPO}:fedora-31_clang-9x" +tag_and_push_img 'integration:s2n' "${ECS_REPO}:s2n_integration_clang-9x" +tag_and_push_img 'ubuntu-20.04:clang-10x_formal-verification' "${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification" + +################################# +# Push images used by aws-c-cal # +################################# +source ./aws-crt/aws-crt-common.sh +push_aws_crt_docker_img From 87ffc718a4c8cec0033e25e24c3cea57de79fc52 Mon Sep 17 00:00:00 2001 From: Bryce Shang Date: Mon, 1 Mar 2021 15:37:26 -0800 Subject: [PATCH 2/7] Remove unused code. --- tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh b/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh index e725610198..beaf19d3e1 100755 --- a/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh +++ b/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh @@ -34,11 +34,6 @@ AWS_CRT_UBUNTU_16_X64="${BASE_AWS_CRT_DOCKER_IMG_PREFIX}/${AWS_CRT_UBUNTU_16_X64 #################################### function login_github_docker_pkg() { - jq_installed=$(command -v jq) - if [[ -z "${jq_installed}" ]]; then - # Install 'jq' for json response parser. - apt-get update -y && apt-get install -y jq - fi if [[ -z "${GITHUB_READ_PKG_ACCESS_TOKEN+x}" || -z "${GITHUB_READ_PKG_ACCESS_TOKEN}" ]]; then # GITHUB_READ_PKG_ACCESS_TOKEN is required when pulling images from 'docker.pkg.github.com'. # When creating GitHub personal access token, make sure ONLY "read:packages" is granted. From c044042cb59885ebceaab7953caa8626eb241f4c Mon Sep 17 00:00:00 2001 From: Bryce Shang Date: Mon, 1 Mar 2021 17:13:04 -0800 Subject: [PATCH 3/7] Run pull_github_pkg.sh in CodeBuild. --- tests/ci/cdk/README.md | 47 ++- .../linux_docker_image_batch_build_stack.py | 32 +- tests/ci/cdk/run-cdk.sh | 380 ++++++++++++------ tests/ci/cdk/util/iam_policies.py | 20 + tests/ci/cdk/util/metadata.py | 3 +- .../codebuild/linux-x86/docker-img-build.yml | 1 + 6 files changed, 338 insertions(+), 145 deletions(-) diff --git a/tests/ci/cdk/README.md b/tests/ci/cdk/README.md index 43639302d3..29c9899e38 100644 --- a/tests/ci/cdk/README.md +++ b/tests/ci/cdk/README.md @@ -8,8 +8,11 @@ AWS-LC CI uses AWS CDK to define and deploy AWS resources (e.g. AWS CodeBuild, E * Install [AWS CDK](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html#getting_started_install) * Install [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) -* [Connect GitHub and AWS account using access token](https://docs.aws.amazon.com/codebuild/latest/userguide/sample-access-tokens.html) +* [Connect AWS CodeBuild with GitHub](https://docs.aws.amazon.com/codebuild/latest/userguide/sample-access-tokens.html) * Note: This step should grant AWS CodeBuild with access to create WebHook. +* [Create GitHub Personal Access Token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) + * Note: This token ONLY needs ['read:packages' permission](https://docs.github.com/en/packages/learn-github-packages/about-github-packages#authenticating-to-github-packages), and should be deleted from GitHub account after docker image build. + * This token is needed when pulling images from 'docker.pkg.github.com'. ### Minimal permissions: @@ -46,34 +49,32 @@ To setup or update the CI in your account you will need the following IAM permis * ssm:Put* * ssm:Update* * ssm:List* +* SecretsManager + * secretsmanager:CreateSecret + * secretsmanager:PutSecretValue + * secretsmanager:DeleteSecret + * secretsmanager:GetSecretValue ### Command +To set up AWS-LC CI, run command: ``` -$ ./run-cdk.sh {aws-account-id} {region} {github-repo-owner} {github-repo-name} {ACTION} +./run-cdk.sh --action deploy-ci --github-access-token 123456 ``` -#### Examples - -To see the synthesized CloudFormation template, run command: +To create/udpate Docker images, run command: ``` -./run-cdk.sh 123456789 us-west-2 GitHubUserName aws-lc SYNTH +./run-cdk.sh --action build-img --github-access-token 123456 ``` -To set up AWS-LC CI in AWS account `123456789` for the forked GitHub repository `https://github.com/GitHubUserName/aws-lc`, run command: +To destroy AWS-LC CI resources created above, run command: ``` -./run-cdk.sh 123456789 us-west-2 GitHubUserName aws-lc DEPLOY +./run-cdk.sh --action destroy-ci ``` -To destroy all AWS resources created above, run command: +For help, run command: ``` -# This command does not delete S3 and ECR, which require manually deletion. -./run-cdk.sh 123456789 us-west-2 GitHubUserName aws-lc DESTROY -``` - -To compare deployed stack with current state, run command: -``` -./run-cdk.sh 123456789 us-west-2 GitHubUserName aws-lc DIFF +./run-cdk.sh --help ``` ## Files @@ -171,6 +172,12 @@ command. #### Linux Docker image build ```bash +# Create GitHub personal access token, which is needed when pulling Docker images from 'docker.pkg.github.com'. +export AWS_LC_CI_SECRET_NAME='aws-lc-ci-external-credential' +secret_arn=$(aws secretsmanager create-secret --name "${AWS_LC_CI_SECRET_NAME}" --secret-string "${GITHUB_ACCESS_TOKEN}" | jq -r '.ARN') +# Export this variable so CDK can create related IAM policy on this ARN. +export EXTERNAL_CREDENTIAL_SECRET_ARN="${secret_arn}" + # Launch Linux Docker image CodeBuild resources. cdk deploy aws-lc-docker-image-build-linux --require-approval never @@ -186,12 +193,12 @@ Below are some commands specific to windows docker image build. ```bash # Define environment variables needed by Windows docker image build. -export AWS_LC_S3_BUCKET_PREFIX="aws-lc-windows-docker-image-build" -export DATE_NOW="$(date +%Y-%m-%d-%H-%M)" +DATE_NOW="$(date +%Y-%m-%d-%H-%M)" +export AWS_LC_S3_BUCKET_PREFIX='aws-lc-windows-docker-image-build-s3' export S3_FOR_WIN_DOCKER_IMG_BUILD="${AWS_LC_S3_BUCKET_PREFIX}-${DATE_NOW}" -export WIN_EC2_TAG_KEY="aws-lc" +export WIN_EC2_TAG_KEY='aws-lc' export WIN_EC2_TAG_VALUE="aws-lc-windows-docker-image-build-${DATE_NOW}" -export WIN_DOCKER_BUILD_SSM_DOCUMENT="windows-ssm-document-${DATE_NOW}" +export WIN_DOCKER_BUILD_SSM_DOCUMENT="windows-ssm-document" # Clean up all Windows docker image build resources. cdk destroy aws-lc-docker-image-build-windows --force diff --git a/tests/ci/cdk/cdk/linux_docker_image_batch_build_stack.py b/tests/ci/cdk/cdk/linux_docker_image_batch_build_stack.py index 1105331c73..966fdc9cdd 100644 --- a/tests/ci/cdk/cdk/linux_docker_image_batch_build_stack.py +++ b/tests/ci/cdk/cdk/linux_docker_image_batch_build_stack.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 from aws_cdk import core, aws_codebuild as codebuild, aws_iam as iam + from util.metadata import AWS_ACCOUNT, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, GITHUB_SOURCE_VERSION, LINUX_AARCH_ECR_REPO, \ - LINUX_X86_ECR_REPO -from util.iam_policies import codebuild_batch_policy_in_json, ecr_power_user_policy_in_json + LINUX_X86_ECR_REPO, EXTERNAL_CREDENTIAL_SECRET_ARN +from util.iam_policies import codebuild_batch_policy_in_json, ecr_power_user_policy_in_json, \ + aws_secrets_manager_get_secret_policy_in_json from util.yml_loader import YmlLoader @@ -26,6 +28,12 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: ecr_power_user_policy = iam.PolicyDocument.from_json(ecr_power_user_policy_in_json()) inline_policies = {"codebuild_batch_policy": codebuild_batch_policy, "ecr_power_user_policy": ecr_power_user_policy} + # GitHub access token is only needed during CI setup. + # The token is used to pull Docker images hosted in 'docker.pkg.github.com'. + if EXTERNAL_CREDENTIAL_SECRET_ARN: + secrets_manager_get_secret_policy = iam.PolicyDocument.from_json( + aws_secrets_manager_get_secret_policy_in_json(EXTERNAL_CREDENTIAL_SECRET_ARN)) + inline_policies["secrets_manager_get_secret_policy"] = secrets_manager_get_secret_policy role = iam.Role(scope=self, id="{}-role".format(id), assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"), @@ -34,6 +42,20 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: # Create build spec. build_spec_content = YmlLoader.load("./cdk/codebuild/linux_img_build_omnibus.yaml") + # Define environment variables. + environment_variables = { + "AWS_ACCOUNT_ID": codebuild.BuildEnvironmentVariable(value=AWS_ACCOUNT), + "AWS_ECR_REPO_X86": codebuild.BuildEnvironmentVariable(value=LINUX_X86_ECR_REPO), + "AWS_ECR_REPO_AARCH": codebuild.BuildEnvironmentVariable(value=LINUX_AARCH_ECR_REPO), + "GITHUB_REPO_OWNER": codebuild.BuildEnvironmentVariable(value=GITHUB_REPO_OWNER), + } + if EXTERNAL_CREDENTIAL_SECRET_ARN: + # See secrets-manager https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html + github_access_token = "{}:GITHUB_PERSONAL_ACCESS_TOKEN".format(EXTERNAL_CREDENTIAL_SECRET_ARN) + environment_variables["GITHUB_READ_PKG_ACCESS_TOKEN"] = codebuild.BuildEnvironmentVariable( + value=github_access_token, + type=codebuild.BuildEnvironmentVariableType.SECRETS_MANAGER) + # Define CodeBuild project. project = codebuild.Project( scope=self, @@ -44,11 +66,7 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: compute_type=codebuild.ComputeType.SMALL, privileged=False, build_image=codebuild.LinuxBuildImage.STANDARD_4_0), - environment_variables={ - "AWS_ACCOUNT_ID": codebuild.BuildEnvironmentVariable(value=AWS_ACCOUNT), - "AWS_ECR_REPO_X86": codebuild.BuildEnvironmentVariable(value=LINUX_X86_ECR_REPO), - "AWS_ECR_REPO_AARCH": codebuild.BuildEnvironmentVariable(value=LINUX_AARCH_ECR_REPO) - }, + environment_variables=environment_variables, role=role, timeout=core.Duration.minutes(120), build_spec=codebuild.BuildSpec.from_object(build_spec_content)) diff --git a/tests/ci/cdk/run-cdk.sh b/tests/ci/cdk/run-cdk.sh index ea8772f6a0..0dfadbd5be 100755 --- a/tests/ci/cdk/run-cdk.sh +++ b/tests/ci/cdk/run-cdk.sh @@ -11,58 +11,85 @@ set -exuo pipefail # -o pipefail: Makes sure to exit a pipeline with a non-zero error code if any command in the pipeline exists with a # non-zero error code. -# Export customized environment variables. -export CDK_DEPLOY_ACCOUNT=$1 -shift -export CDK_DEPLOY_REGION=$1 -shift -export GITHUB_REPO_OWNER=$1 -shift -export GITHUB_REPO=$1 -shift -export ACTION=$1 -shift - -# Export other environment variables. -export DATE_NOW="$(date +%Y-%m-%d-%H-%M)" -export ECR_LINUX_AARCH_REPO_NAME="aws-lc-docker-images-linux-aarch" -export ECR_LINUX_X86_REPO_NAME="aws-lc-docker-images-linux-x86" -export ECR_WINDOWS_X86_REPO_NAME="aws-lc-docker-images-windows-x86" -export AWS_LC_S3_BUCKET_PREFIX="aws-lc-windows-docker-image-build" -export S3_FOR_WIN_DOCKER_IMG_BUILD="${AWS_LC_S3_BUCKET_PREFIX}-${DATE_NOW}" -export WIN_EC2_TAG_KEY="aws-lc" -export WIN_EC2_TAG_VALUE="aws-lc-windows-docker-image-build-${DATE_NOW}" -export WIN_DOCKER_BUILD_SSM_DOCUMENT="windows-ssm-document-${DATE_NOW}" - -# Functions function delete_s3_buckets() { aws s3api list-buckets --query "Buckets[].Name" | jq '.[]' | while read -r i; do + bucket_name=$(echo "${i}" | tr -d '"') # Delete the bucket if its name uses AWS_LC_S3_BUCKET_PREFIX. - if [[ ${i} == "${AWS_LC_S3_BUCKET_PREFIX}"* ]]; then - aws s3 rm "s3://${i}" --recursive - aws s3api delete-bucket --bucket "${i}" + if [[ "${bucket_name}" == *"${AWS_LC_S3_BUCKET_PREFIX}"* ]]; then + aws s3 rm "s3://${bucket_name}" --recursive + aws s3api delete-bucket --bucket "${bucket_name}" fi done } function delete_container_repositories() { - aws ecr delete-repository --repository-name ${ECR_LINUX_AARCH_REPO_NAME} --force - aws ecr delete-repository --repository-name ${ECR_LINUX_X86_REPO_NAME} --force - aws ecr delete-repository --repository-name ${ECR_WINDOWS_X86_REPO_NAME} --force + ecr_repos=$(aws ecr describe-repositories) + if [[ "${ecr_repos}" == *"${ECR_LINUX_AARCH_REPO_NAME}"* ]]; then + aws ecr delete-repository --repository-name "${ECR_LINUX_AARCH_REPO_NAME}" --force + fi + if [[ "${ecr_repos}" == *"${ECR_LINUX_X86_REPO_NAME}"* ]]; then + aws ecr delete-repository --repository-name "${ECR_LINUX_X86_REPO_NAME}" --force + fi + if [[ "${ecr_repos}" == *"${ECR_WINDOWS_X86_REPO_NAME}"* ]]; then + aws ecr delete-repository --repository-name "${ECR_WINDOWS_X86_REPO_NAME}" --force + fi } -function destroy_all() { +function delete_external_credential() { + # This deletion may take a few minutes to take effects. + # https://docs.aws.amazon.com/cli/latest/reference/secretsmanager/delete-secret.html + # https://aws.amazon.com/premiumsupport/knowledge-center/delete-secrets-manager-secret/ + aws secretsmanager delete-secret --secret-id "${AWS_LC_CI_SECRET_NAME}" --force-delete-without-recovery +} + +function store_external_credential() { + # Always delete the secret to avoid resource name conflicts. + delete_external_credential + # When creating secret with the same name repeatedly, + # below error may occur because the deletion may take a few minutes to take effects. + # "You can't create this secret because a secret with this name is already scheduled for deletion." + # To avoid that, sleep 1 min + sleep 60 + # https://docs.aws.amazon.com/cli/latest/reference/secretsmanager/create-secret.html + key_value="{\"GITHUB_PERSONAL_ACCESS_TOKEN\":\"${GITHUB_ACCESS_TOKEN}\"}" + secret_arn=$(aws secretsmanager create-secret --name "${AWS_LC_CI_SECRET_NAME}" --secret-string "${key_value}" | jq -r '.ARN') + # Export this variable so CDK can create related IAM policy on this ARN. + export EXTERNAL_CREDENTIAL_SECRET_ARN="${secret_arn}" +} + +function destroy_ci() { cdk destroy aws-lc-* --force # CDK stack destroy does not delete s3 bucket automatically. delete_s3_buckets # CDK stack destroy does not delete ecr automatically. delete_container_repositories + # Delete the secret from Secrets Manager. + delete_external_credential +} + +function destroy_docker_img_build_stack() { + # Destroy all temporary resources created for all docker image build. + cdk destroy aws-lc-docker-image-build-* --force + # CDK stack destroy does not delete s3 bucket automatically. + delete_s3_buckets + # Delete the secret from Secrets Manager. + delete_external_credential +} + +function create_docker_img_build_stack() { + # Clean up build stacks if exists. + destroy_docker_img_build_stack + # Use SecretsManager to store GitHub personal access token. + store_external_credential + # Deploy aws-lc ci stacks. + # When repeatedly deploy, error 'EIP failed Reason: Maximum number of addresses has been reached' can happen. + # https://forums.aws.amazon.com/thread.jspa?messageID=952368 + # Workaround: go to AWS EIP console, release unused IP. + cdk deploy aws-lc-docker-image-build-* --require-approval never } -function create_aws_resources() { - # Clean up resources before deployment. - destroy_all - cdk deploy aws-lc-* --require-approval never +function create_github_ci_stack() { + cdk deploy aws-lc-ci-* --require-approval never # Need to use aws cli to change webhook build type because CFN is not ready yet. aws codebuild update-webhook --project-name aws-lc-ci-linux-x86 --build-type BUILD_BATCH aws codebuild update-webhook --project-name aws-lc-ci-linux-arm --build-type BUILD_BATCH @@ -70,28 +97,35 @@ function create_aws_resources() { } function build_linux_img() { - aws codebuild start-build-batch --project-name aws-lc-docker-image-build-linux + # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/codebuild/start-build-batch.html + build_id=$(aws codebuild start-build-batch --project-name aws-lc-docker-image-build-linux | jq -r '.buildBatch.id') + export AWS_LC_LINUX_BUILD_BATCH_ID="${build_id}" } function build_windows_img() { - # EC2 takes 10 min to be ready for running command. - echo "Wait 10 min for EC2 ready for SSM command execution." - sleep 600 - - instance_id=$(aws ec2 describe-instances \ - --filters "Name=tag:${WIN_EC2_TAG_KEY},Values=${WIN_EC2_TAG_VALUE}" | jq -r '.Reservations[0].Instances[0].InstanceId') + # EC2 takes several minutes to be ready for running command. + echo "Wait 3 min for EC2 ready for SSM command execution." + sleep 180 # Run commands on windows EC2 instance to build windows docker images. for i in {1..60}; do + instance_id=$(aws ec2 describe-instances \ + --filters "Name=tag:${WIN_EC2_TAG_KEY},Values=${WIN_EC2_TAG_VALUE}" | jq -r '.Reservations[0].Instances[0].InstanceId') + if [[ "${instance_id}" == "null" ]]; then + sleep 60 + continue + fi instance_ping_status=$(aws ssm describe-instance-information \ --filters "Key=InstanceIds,Values=${instance_id}" | jq -r '.InstanceInformationList[0].PingStatus') if [[ "${instance_ping_status}" == "Online" ]]; then # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ssm/send-command.html - aws ssm send-command \ + command_id=$(aws ssm send-command \ --instance-ids "${instance_id}" \ --document-name "${WIN_DOCKER_BUILD_SSM_DOCUMENT}" \ --output-s3-bucket-name "${S3_FOR_WIN_DOCKER_IMG_BUILD}" \ - --output-s3-key-prefix 'runcommand' + --output-s3-key-prefix 'runcommand' | jq -r '.Command.CommandId') + # Export for checking command run status. + export WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID="${command_id}" echo "Windows ec2 is executing SSM command." return else @@ -103,48 +137,57 @@ function build_windows_img() { exit 1 } -function destroy_docker_img_build_stack() { - # Destroy all temporary resources created for all docker image build. - cdk destroy aws-lc-docker-image-build-* --force +function linux_docker_img_build_status_check() { + # Every 5 min, this function checks if the linux docker image batch code build is finished successfully. + # Normally, docker img build can take up to 1 hour. Here, we wait up to 30 * 5 min. + for i in {1..30}; do + # https://docs.aws.amazon.com/cli/latest/reference/codebuild/batch-get-build-batches.html + build_batch_status=$(aws codebuild batch-get-build-batches --ids "${AWS_LC_LINUX_BUILD_BATCH_ID}" | jq -r '.buildBatches[0].buildBatchStatus') + if [[ ${build_batch_status} == 'SUCCEEDED' ]]; then + echo "Code build ${AWS_LC_LINUX_BUILD_BATCH_ID} finished successfully." + return + elif [[ ${build_batch_status} == 'FAILED' ]]; then + echo "Code build ${AWS_LC_LINUX_BUILD_BATCH_ID} failed." + exit 1 + else + echo "${i}: Wait 5 min for docker image build job finish." + sleep 300 + fi + done + echo "Code build ${AWS_LC_LINUX_BUILD_BATCH_ID} takes more time than expected." + exit 1 } -function images_pushed_to_ecr() { - repo_name=$1 - shift - target_images=("$@") - ecr_repo_name="${CDK_DEPLOY_ACCOUNT}.dkr.ecr.${CDK_DEPLOY_REGION}.amazonaws.com/${repo_name}" - docker_images=$* - echo "Checking if docker images [${docker_images}] are pushed to ${ecr_repo_name}." - - # Every 5 min, this function checks if the target docker img is created. +function win_docker_img_build_status_check() { + # Every 5 min, this function checks if the windows docker image build is finished successfully. # Normally, docker img build can take up to 1 hour. Here, we wait up to 30 * 5 min. for i in {1..30}; do - images_in_ecr=$(aws ecr describe-images --repository-name ${repo_name}) - images_pushed=0 - for target_image in "${target_images[@]}"; do - if [[ ${images_in_ecr} != *"${target_image}"* ]]; then - images_pushed=1 - break - fi - done - if [[ ${images_pushed} == 0 ]]; then - echo "All required images are pushed to ${ecr_repo_name}." + # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ssm/list-commands.html + command_run_status=$(aws ssm list-commands --command-id "${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID}" | jq -r '.Commands[0].Status') + if [[ ${command_run_status} == 'Success' ]]; then + echo "SSM command ${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID} finished successfully." return + elif [[ ${command_run_status} == 'Failed' ]]; then + echo "SSM command ${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID} failed." + exit 1 else - echo "${i}: Wait 5 min for docker image creation." + echo "${i}: Wait 5 min for windows docker image build job finish." sleep 300 fi done - echo "Error: Images are not pushed to ${ecr_repo_name}. Exit." + echo "SSM command ${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID} takes more time than expected." exit 1 } -function deploy() { +function build_docker_images() { # Always destroy docker build stacks (which include EC2 instance) on EXIT. trap destroy_docker_img_build_stack EXIT - echo "Creating AWS resources through CDK." - create_aws_resources + # Create/update aws-ecr repo. + cdk deploy aws-lc-ecr-* --require-approval never + + # Create docker image build stack. + create_docker_img_build_stack echo "Activating AWS CodeBuild to build Linux aarch & x86 docker images." build_linux_img @@ -153,49 +196,152 @@ function deploy() { build_windows_img echo "Waiting for docker images creation. Building the docker images need to take 1 hour." - linux_aarch_img_tags=("amazonlinux-2_gcc-7x_latest" - "ubuntu-20.04_clang-10x_latest" - "ubuntu-19.10_clang-9x_latest" - "ubuntu-19.10_clang-9x_sanitizer_latest") - images_pushed_to_ecr "${ECR_LINUX_AARCH_REPO_NAME}" "${linux_aarch_img_tags[@]}" - linux_x86_img_tags=("ubuntu-16.04_gcc-5x_latest" - "ubuntu-18.04_gcc-7x_latest" - "ubuntu-18.04_clang-6x_latest" - "ubuntu-19.04_gcc-8x_latest" - "ubuntu-19.04_clang-8x_latest" - "ubuntu-19.10_clang-9x_sanitizer_latest" - "ubuntu-19.10_clang-9x_latest" - "ubuntu-19.04_gcc-8x_latest" - "ubuntu-19.04_clang-8x_latest" - "ubuntu-20.04_clang-10x_latest" - "ubuntu-20.04_clang-10x_formal-verification_latest" - "centos-7_gcc-4x_latest" - "amazonlinux-2_gcc-7x_latest" - "amazonlinux-2_gcc-7x_intel-sde_latest" - "s2n_integration_clang-9x_latest") - images_pushed_to_ecr "${ECR_LINUX_X86_REPO_NAME}" "${linux_x86_img_tags[@]}" - windows_img_tags=("vs2015_latest" "vs2017_latest") - images_pushed_to_ecr "${ECR_WINDOWS_X86_REPO_NAME}" "${windows_img_tags[@]}" -} - -# Main logics - -case ${ACTION} in -DEPLOY) - deploy - ;; -DIFF) - cdk diff aws-lc* - ;; -SYNTH) - cdk synth aws-lc* - ;; -DESTROY) - destroy_all - ;; -*) - echo "Action: ${ACTION} is not supported." - ;; -esac - -exit 0 + # TODO(CryptoAlg-624): These image build may fail due to the Docker Hub pull limits made on 2020-11-01. + linux_docker_img_build_status_check + win_docker_img_build_status_check +} + +function setup_ci() { + build_docker_images + + create_github_ci_stack +} + +########################### +# Main and related helper # +########################### + +function script_helper() { + cat < Date: Wed, 3 Mar 2021 09:15:04 -0800 Subject: [PATCH 4/7] Address comments. --- tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh b/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh index beaf19d3e1..871a3dfd5f 100755 --- a/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh +++ b/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh @@ -7,7 +7,7 @@ parent_dir="$(dirname -- "$(realpath -- "$file_name")")" source "${parent_dir}/common.sh" ################################# -# Sett up environment variables # +# Set up environment variables # ################################# if [[ -z "${ECS_REPO+x}" || -z "${ECS_REPO}" ]]; then @@ -20,7 +20,8 @@ if [[ -z "${GITHUB_REPO_OWNER+x}" || -z "${GITHUB_REPO_OWNER}" ]]; then fi # Environment variables used by AWS-C-CAL -# https://github.com/awslabs/aws-c-cal/blob/905b0fd0ff2ca0d86c7e49b8ef4b636bac4600ea/.github/workflows/ci.yml +# CI of AWS-C-CAL https://github.com/awslabs/aws-c-cal/blob/905b0fd0ff2ca0d86c7e49b8ef4b636bac4600ea/.github/workflows/ci.yml +# The CI uses Docker images provided by https://github.com/awslabs/aws-crt-builder BUILDER_VERSION='v0.7.2' BASE_AWS_CRT_DOCKER_IMG_PREFIX='docker.pkg.github.com/awslabs/aws-crt-builder' AWS_CRT_MANY_LINUX_SUFFIXES=('aws-crt-manylinux1-x64' 'aws-crt-manylinux1-x86' 'aws-crt-manylinux2014-x64' 'aws-crt-manylinux2014-x86') From d0fbfce9aa08002d68e5c888f9faf8f0f22e659c Mon Sep 17 00:00:00 2001 From: Bryce Shang Date: Fri, 5 Mar 2021 09:12:05 -0800 Subject: [PATCH 5/7] Move common source. --- tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh | 4 ---- tests/ci/docker_images/linux-x86/pull_github_pkg.sh | 6 +++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh b/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh index 871a3dfd5f..247251998a 100755 --- a/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh +++ b/tests/ci/docker_images/linux-x86/aws-crt/aws-crt-common.sh @@ -2,10 +2,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -parent_dir="$(dirname -- "$(realpath -- "$file_name")")" -# shellcheck source=../common.sh -source "${parent_dir}/common.sh" - ################################# # Set up environment variables # ################################# diff --git a/tests/ci/docker_images/linux-x86/pull_github_pkg.sh b/tests/ci/docker_images/linux-x86/pull_github_pkg.sh index b7ccb2d5f5..2bc2a0b8c7 100755 --- a/tests/ci/docker_images/linux-x86/pull_github_pkg.sh +++ b/tests/ci/docker_images/linux-x86/pull_github_pkg.sh @@ -5,6 +5,10 @@ # Note: this file is reserved for pulling Docker images from 'docker.pkg.github.com', which needs external credential. # Pull Docker images used by https://github.com/awslabs/aws-c-cal # These Docker images are hosted in 'docker.pkg.github.com', which needs GitHub personal access token. -source ./aws-crt/aws-crt-common.sh +cur_dir="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=./common.sh +source "${cur_dir}/common.sh" +# shellcheck source=./aws-crt/aws-crt-common.sh +source "${cur_dir}/aws-crt/aws-crt-common.sh" login_github_docker_pkg pull_aws_crt_docker_img From e62d23259437bed87f0734ba8bb6726294edb51e Mon Sep 17 00:00:00 2001 From: Bryce Shang Date: Tue, 16 Mar 2021 10:53:49 -0700 Subject: [PATCH 6/7] Change README.md --- tests/ci/cdk/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/ci/cdk/README.md b/tests/ci/cdk/README.md index 29c9899e38..8ba6d8417e 100644 --- a/tests/ci/cdk/README.md +++ b/tests/ci/cdk/README.md @@ -59,12 +59,14 @@ To setup or update the CI in your account you will need the following IAM permis To set up AWS-LC CI, run command: ``` -./run-cdk.sh --action deploy-ci --github-access-token 123456 +export GITHUB_ACCESS_TOKEN='xxxxx' +./run-cdk.sh --action deploy-ci --github-access-token ${GITHUB_ACCESS_TOKEN} ``` To create/udpate Docker images, run command: ``` -./run-cdk.sh --action build-img --github-access-token 123456 +export GITHUB_ACCESS_TOKEN='xxxxx' +./run-cdk.sh --action build-img --github-access-token ${GITHUB_ACCESS_TOKEN} ``` To destroy AWS-LC CI resources created above, run command: From 2662e7d284dca37f3193a23f3ecd311255feb33a Mon Sep 17 00:00:00 2001 From: Bryce Shang Date: Fri, 19 Mar 2021 15:27:33 -0700 Subject: [PATCH 7/7] Address comments. --- tests/ci/cdk/run-cdk.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/cdk/run-cdk.sh b/tests/ci/cdk/run-cdk.sh index 0dfadbd5be..667ce38fad 100755 --- a/tests/ci/cdk/run-cdk.sh +++ b/tests/ci/cdk/run-cdk.sh @@ -138,7 +138,7 @@ function build_windows_img() { } function linux_docker_img_build_status_check() { - # Every 5 min, this function checks if the linux docker image batch code build is finished successfully. + # Every 5 min, this function checks if the linux docker image batch code build finished successfully. # Normally, docker img build can take up to 1 hour. Here, we wait up to 30 * 5 min. for i in {1..30}; do # https://docs.aws.amazon.com/cli/latest/reference/codebuild/batch-get-build-batches.html @@ -337,7 +337,7 @@ function main() { cdk diff aws-lc-ci-* ;; *) - echo "--action is required. Use '--help' to see allowed enums." + echo "--action is required. Use '--help' to see allowed actions." exit 1 ;; esac