diff --git a/tests/ci/cdk/README.md b/tests/ci/cdk/README.md index c8b2c3107d..e4505e2d71 100644 --- a/tests/ci/cdk/README.md +++ b/tests/ci/cdk/README.md @@ -18,6 +18,9 @@ To setup or update the CI in your account you will need the following IAM permis * codebuild:Create* * codebuild:Update* * codebuild:Batch* + * codebuild:StartBuild + * codebuild:StopBuild + * codebuild:RetryBuild * EC2 * ec2:Describe* * ec2:Create* @@ -86,17 +89,15 @@ Below is CI file structure. ├── cdk │   ├── __init__.py │   ├── ecr_stack.py -│   ├── github_codebuild_stack.py -│   ├── linux_docker_images_build_stack.py -│   ├── windows_docker_build_ssm_document.yaml -│   └── windows_docker_image_build_stack.py +│   ├── ... ├── cdk.json ├── requirements.txt ├── run-cdk.sh ├── setup.py └── util ├── __init__.py - └── util.py + └── env_util.py + └── ... ``` * `README.md` — The introductory README for this project. * `app.py` — The “main” for this sample application. @@ -158,3 +159,57 @@ command. * `cdk deploy` deploy this stack to your default AWS account/region * `cdk diff` compare deployed stack with current state * `cdk docs` open CDK documentation + +### Useful Docker image build commands + +**Notes**: +* below commands replicate steps that are performed in `run-cdk.sh` but use default values set in `cdk/util/metadata.py`. +* Always clean up resources set up for Docker image build. + * `cdk destroy aws-lc-docker-image-build-* --force` + +#### Linux Docker image build + +```bash +# Launch Linux Docker image CodeBuild resources. +cdk deploy aws-lc-docker-image-build-linux --require-approval never + +# Trigger CodeBuild to build Linux Docker Images +aws codebuild start-build-batch --project-name aws-lc-docker-image-build-linux + +# Go to AWS console, you can check CodeBuild by clicking "Developer Tools > CodeBuild > Build projects". +``` + +#### Windows Docker image build +Windows docker image build requires more resources (like EC2 host, S3, SSM and so on) set up because DIND (Docker in Docker) is not supported by Windows. +Below are some commands specific to windows docker image build. + +```bash +# Define environment variables needed by Windows docker image build. +export DATE_NOW="$(date +%Y-%m-%d-%H-%M)" +export S3_FOR_WIN_DOCKER_IMG_BUILD="${AWS_LC_S3_BUCKET_PREFIX}-${DATE_NOW}" +export WIN_EC2_TAG_KEY="aws-lc" +export WIN_EC2_TAG_VALUE="aws-lc-windows-docker-image-build-${DATE_NOW}" +export WIN_DOCKER_BUILD_SSM_DOCUMENT="windows-ssm-document-${DATE_NOW}" + +# Clean up all Windows docker image build resources. +cdk destroy aws-lc-docker-image-build-windows --force +aws s3 rm "s3://${S3_FOR_WIN_DOCKER_IMG_BUILD}" --recursive +aws s3api delete-bucket --bucket "${S3_FOR_WIN_DOCKER_IMG_BUILD}" + +# Deploy Windows docker image build resources. +cdk deploy aws-lc-docker-image-build-windows --require-approval never + +# Sleep 10 minutes so Windows EC2 is ready to execute SSM commands. +sleep 600 + +# Trigger SSM commands to build Windows Docker Images. +instance_id=$(aws ec2 describe-instances \ + --filters "Name=tag:${WIN_EC2_TAG_KEY},Values=${WIN_EC2_TAG_VALUE}" | jq -r '.Reservations[0].Instances[0].InstanceId') +aws ssm send-command \ + --instance-ids "${instance_id}" \ + --document-name "${WIN_DOCKER_BUILD_SSM_DOCUMENT}" \ + --output-s3-bucket-name "${S3_FOR_WIN_DOCKER_IMG_BUILD}" \ + --output-s3-key-prefix 'ssm-runcommand-logs' + +# Go to AWS console, you can check run command by clicking `AWS Systems Manager > Run Command`. +``` diff --git a/tests/ci/cdk/app.py b/tests/ci/cdk/app.py index 67c62fcce0..73cd49fd4b 100644 --- a/tests/ci/cdk/app.py +++ b/tests/ci/cdk/app.py @@ -5,170 +5,29 @@ from aws_cdk import core -from cdk.github_codebuild_stack import GitHubCodeBuildStack -from cdk.linux_docker_images_build_stack import LinuxDockerImagesBuildStack +from cdk.aws_lc_github_ci_stack import AwsLcGitHubCIStack +from cdk.linux_docker_image_batch_build_stack import LinuxDockerImageBatchBuildStack from cdk.windows_docker_image_build_stack import WindowsDockerImageBuildStack from cdk.ecr_stack import EcrStack -from util.util import EnvUtil +from util.metadata import LINUX_X86_ECR_REPO, LINUX_AARCH_ECR_REPO, WINDOWS_ECR_REPO # Initialize app. app = core.App() -# Fetch environment variables. -aws_account = EnvUtil.get("CDK_DEPLOY_ACCOUNT", "620771051181") -aws_region = EnvUtil.get("CDK_DEPLOY_REGION", "us-west-2") -env = {"account": aws_account, "region": aws_region} - # Define AWS ECR stacks. # ECR holds the docker images, which are pre-built to accelerate the code builds/tests of git pull requests. -linux_aarch_ecr_repo = EnvUtil.get("ECR_LINUX_AARCH_REPO_NAME", "aws-lc-test-docker-images-linux-aarch") -linux_x86_ecr_repo = EnvUtil.get("ECR_LINUX_X86_REPO_NAME", "aws-lc-test-docker-images-linux-x86") -windows_ecr_repo = EnvUtil.get("ECR_WINDOWS_REPO_NAME", "aws-lc-test-docker-images-windows") -EcrStack(app, linux_aarch_ecr_repo, env=env) -EcrStack(app, linux_x86_ecr_repo, env=env) -EcrStack(app, windows_ecr_repo, env=env) +EcrStack(app, "aws-lc-ecr-linux-x86", LINUX_X86_ECR_REPO) +EcrStack(app, "aws-lc-ecr-linux-aarch", LINUX_AARCH_ECR_REPO) +EcrStack(app, "aws-lc-ecr-windows", WINDOWS_ECR_REPO) + +# Define CodeBuild Batch job for testing code. +LinuxDockerImageBatchBuildStack(app, "aws-lc-docker-image-build-linux") -# Define stacks used to build Docker images. -linux_docker_img_build_stacks = [ - { - "id": "aws-lc-test-docker-images-build-linux-aarch", - "repo_name": linux_aarch_ecr_repo, - "env_type": "ARM", - "build_spec": "linux-aarch-docker-img-build.yml" - }, - { - "id": "aws-lc-test-docker-images-build-linux-x86", - "repo_name": linux_x86_ecr_repo, - "env_type": "Linux", - "build_spec": "linux-x86-docker-img-build.yml" - } -] -code_build_dir = "./tests/ci/codebuild" -for stack in linux_docker_img_build_stacks: - LinuxDockerImagesBuildStack(app, stack["id"], stack["repo_name"], - "./tests/ci/codebuild/{}".format(stack["build_spec"]), env_type=stack["env_type"], - env=env) # DIND is not supported on Windows and, therefore, AWS CodeBuild is not used to build Windows Server container images. # Windows Docker images are created by running commands in Windows EC2 instance. -WindowsDockerImageBuildStack(app, "aws-lc-test-docker-images-build-windows", windows_ecr_repo, env=env) +WindowsDockerImageBuildStack(app, "aws-lc-docker-image-build-windows") -# Define CodeBuild stacks used to run test cases. -is_windows = True -# Define CodeBuild running on Linux aarch. -linux_aarch_test_stacks = [ - { - "id": "aws-lc-test-ubuntu-19-10--gcc-9x", - "img": "ubuntu-19.10_gcc-9x_latest", - "spec": "ubuntu-19.10_gcc-9x_aarch64.yml", - }, - { - "id": "aws-lc-test-ubuntu-19-10--clang-9x", - "img": "ubuntu-19.10_clang-9x_latest", - "spec": "ubuntu-19.10_clang-9x_aarch64.yml", - }, -] -for stack in linux_aarch_test_stacks: - GitHubCodeBuildStack(app, stack["id"], linux_aarch_ecr_repo, stack["img"], - "./tests/ci/codebuild/{}".format(stack["spec"]), env_type="ARM", privileged=True, env=env) -# Define CodeBuild running on Linux x86-64. -linux_x86_test_stacks = [ - { - "id": "aws-lc-test-pre-push", - "img": "ubuntu-18.04_gcc-7x_latest", - "spec": "pre-push.yml", - }, - { - "id": "aws-lc-test-ubuntu-16-04--gcc-5x--x86", - "img": "ubuntu-16.04_gcc-5x_latest", - "spec": "ubuntu-16.04_gcc-5x_x86.yml", - }, - { - "id": "aws-lc-test-ubuntu-18-04--clang-6x--x86-64", - "img": "ubuntu-18.04_clang-6x_latest", - "spec": "ubuntu-18.04_clang-6x_x86-64.yml", - }, - { - "id": "aws-lc-test-ubuntu-18-04--gcc-7x--x86-64", - "img": "ubuntu-18.04_gcc-7x_latest", - "spec": "ubuntu-18.04_gcc-7x_x86-64.yml", - }, - { - "id": "aws-lc-test-ubuntu-19-10--gcc-9x--x86-64", - "img": "ubuntu-19.10_gcc-9x_latest", - "spec": "ubuntu-19.10_gcc-9x_x86-64.yml", - }, - { - "id": "aws-lc-test-ubuntu-19-10--clang-9x--x86-64", - "img": "ubuntu-19.10_clang-9x_latest", - "spec": "ubuntu-19.10_clang-9x_x86-64.yml", - }, - { - "id": "aws-lc-test-ubuntu-19-04--gcc-8x--x86-64", - "img": "ubuntu-19.04_gcc-8x_latest", - "spec": "ubuntu-19.04_gcc-8x_x86-64.yml", - }, - { - "id": "aws-lc-test-ubuntu-19-04--clang-8x--x86-64", - "img": "ubuntu-19.04_clang-8x_latest", - "spec": "ubuntu-19.04_clang-8x_x86-64.yml", - }, - { - "id": "aws-lc-test-centos-7--gcc-4x--x86", - "img": "centos-7_gcc-4x_latest", - "spec": "centos-7_gcc-4x-x86.yml", - }, - { - "id": "aws-lc-test-centos-7--gcc-4x--x86-64", - "img": "centos-7_gcc-4x_latest", - "spec": "centos-7_gcc-4x-x86-64.yml", - }, - { - "id": "aws-lc-test-amazonlinux-2--gcc-7x--x86-64", - "img": "amazonlinux-2_gcc-7x_latest", - "spec": "amazonlinux-2_gcc-7x-x86-64.yml", - }, - { - "id": "aws-lc-test-s2n--integration", - "img": "s2n_integration_clang-9x_latest", - "spec": "s2n_integration.yml", - }, - { - "id": "aws-lc-test-fedora-31--gcc-9x", - "img": "fedora-31_gcc-9x_latest", - "spec": "fedora-31_gcc-9x_x86-64.yml", - }, - { - "id": "aws-lc-test-fedora-31--clang-9x", - "img": "fedora-31_clang-9x_latest", - "spec": "fedora-31_clang-9x_x86-64.yml", - }, -] -for stack in linux_x86_test_stacks: - GitHubCodeBuildStack(app, stack["id"], linux_x86_ecr_repo, stack["img"], - "./tests/ci/codebuild/{}".format(stack["spec"]), privileged=True, env=env) -# Define CodeBuild for sanitizer tests. -linux_sanitizer_test_stacks = [ - { - "id": "aws-lc-test-ubuntu-19-10--clang-9x--sanitizer", - "img": "ubuntu-19.10_clang-9x_sanitizer_latest", - "spec": "ubuntu-19.10_clang-9x_aarch64_sanitizer.yml", - "env_type": "ARM", - "repo": linux_aarch_ecr_repo, - }, - { - "id": "aws-lc-test-ubuntu-19-10--clang-9x--x86-64--sanitizer", - "img": "ubuntu-19.10_clang-9x_sanitizer_latest", - "spec": "ubuntu-19.10_clang-9x_x86-64_sanitizer.yml", - "env_type": "Linux", - "repo": linux_x86_ecr_repo, - }, -] -for stack in linux_sanitizer_test_stacks: - GitHubCodeBuildStack(app, stack["id"], stack["repo"], stack["img"], - "./tests/ci/codebuild/{}".format(stack["spec"]), env_type=stack["env_type"], privileged=True, - env=env) -# Define CodeBuild running on Windows. -GitHubCodeBuildStack(app, "aws-lc-test-windows-msvc2015-x64--vs2015", windows_ecr_repo, "vs2015_latest", - "./tests/ci/codebuild/windows-msvc2015-x64.yml", env_type="Windows", env=env) +# Define CodeBuild Batch job for testing code. +AwsLcGitHubCIStack(app, "aws-lc-ci") app.synth() diff --git a/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py new file mode 100644 index 0000000000..5777dab4b9 --- /dev/null +++ b/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py @@ -0,0 +1,75 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from aws_cdk import core, aws_codebuild as codebuild, aws_iam as iam +from util.iam_policies import codebuild_batch_policy_in_json +from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, LINUX_X86_ECR_REPO, \ + LINUX_AARCH_ECR_REPO, WINDOWS_ECR_REPO +from util.yml_loader import YmlLoader + + +class AwsLcGitHubCIStack(core.Stack): + """Define a stack used to batch execute AWS-LC tests in GitHub.""" + + def __init__(self, + scope: core.Construct, + id: str, + **kwargs) -> None: + super().__init__(scope, id, **kwargs) + + # Define CodeBuild resource. + git_hub_source = codebuild.Source.git_hub( + owner=GITHUB_REPO_OWNER, + repo=GITHUB_REPO_NAME, + webhook=True, + webhook_filters=[ + codebuild.FilterGroup.in_event_of( + codebuild.EventAction.PULL_REQUEST_CREATED, + codebuild.EventAction.PULL_REQUEST_UPDATED, + codebuild.EventAction.PULL_REQUEST_REOPENED) + ], + clone_depth=1) + + # Define a IAM role for this stack. + codebuild_batch_policy = iam.PolicyDocument.from_json( + codebuild_batch_policy_in_json([id]) + ) + inline_policies = {"codebuild_batch_policy": codebuild_batch_policy} + role = iam.Role(scope=self, + id="{}-role".format(id), + assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"), + inline_policies=inline_policies, + managed_policies=[ + iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEC2ContainerRegistryReadOnly") + ]) + + # Create build spec. + placeholder_map = {"AWS_ACCOUNT_ID_PLACEHOLDER": AWS_ACCOUNT, "AWS_REGION_PLACEHOLDER": AWS_REGION, + "ECR_REPO_X86_PLACEHOLDER": LINUX_X86_ECR_REPO, + "ECR_REPO_AARCH_PLACEHOLDER": LINUX_AARCH_ECR_REPO, + "ECR_REPO_WINDOWS_PLACEHOLDER": WINDOWS_ECR_REPO} + build_spec_content = YmlLoader.load("./cdk/codebuild/github_build_omnibus.yaml", placeholder_map) + + # Define CodeBuild. + project = codebuild.Project( + scope=self, + id=id, + project_name=id, + source=git_hub_source, + role=role, + timeout=core.Duration.minutes(120), + environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.SMALL, + privileged=False, + build_image=codebuild.LinuxBuildImage.STANDARD_4_0), + build_spec=codebuild.BuildSpec.from_object(build_spec_content)) + + # TODO: add build type BUILD_BATCH when CFN finishes the feature release. See CryptoAlg-575. + + # Add 'BuildBatchConfig' property, which is not supported in CDK. + # CDK raw overrides: https://docs.aws.amazon.com/cdk/latest/guide/cfn_layer.html#cfn_layer_raw + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-codebuild-project.html#aws-resource-codebuild-project-properties + cfn_build = project.node.default_child + cfn_build.add_override("Properties.BuildBatchConfig", { + "ServiceRole": role.role_arn, + "TimeoutInMins": 120 + }) diff --git a/tests/ci/cdk/cdk/codebuild/github_build_omnibus.yaml b/tests/ci/cdk/cdk/codebuild/github_build_omnibus.yaml new file mode 100644 index 0000000000..6d4b4187b2 --- /dev/null +++ b/tests/ci/cdk/cdk/codebuild/github_build_omnibus.yaml @@ -0,0 +1,169 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +version: 0.2 + +# Doc for batch https://docs.aws.amazon.com/codebuild/latest/userguide/batch-build-buildspec.html#build-spec.batch.build-list +batch: + build-list: + + ############# + # Linux x86 # + ############# + - identifier: pre_push_checker + buildspec: ./tests/ci/codebuild/pre-push.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_SMALL + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:ubuntu-18.04_gcc-7x_latest + + - identifier: ubuntu1604_gcc5x_x86 + buildspec: ./tests/ci/codebuild/ubuntu-16.04_gcc-5x_x86.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:ubuntu-16.04_gcc-5x_latest + + - identifier: ubuntu1804_clang6x_x86_64 + buildspec: ./tests/ci/codebuild/ubuntu-18.04_clang-6x_x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:ubuntu-18.04_clang-6x_latest + + - identifier: ubuntu1804_gcc7x_x86_64 + buildspec: ./tests/ci/codebuild/ubuntu-18.04_gcc-7x_x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:ubuntu-18.04_gcc-7x_latest + + - identifier: ubuntu1910_gcc9x_x86_64 + buildspec: ./tests/ci/codebuild/ubuntu-19.10_gcc-9x_x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:ubuntu-19.10_gcc-9x_latest + + - identifier: ubuntu1910_clang9x_x86_64 + buildspec: ./tests/ci/codebuild/ubuntu-19.10_clang-9x_x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:ubuntu-19.10_clang-9x_latest + + - identifier: ubuntu1904_gcc8x_x86_64 + buildspec: ./tests/ci/codebuild/ubuntu-19.04_gcc-8x_x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:ubuntu-19.04_gcc-8x_latest + + - identifier: ubuntu1904_clang8x_x86_64 + buildspec: ./tests/ci/codebuild/ubuntu-19.04_clang-8x_x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:ubuntu-19.04_clang-8x_latest + + - identifier: centos7_gcc4x_x86 + buildspec: ./tests/ci/codebuild/centos-7_gcc-4x-x86.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:centos-7_gcc-4x_latest + + - identifier: centos7_gcc4x_x86_64 + buildspec: ./tests/ci/codebuild/centos-7_gcc-4x-x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:centos-7_gcc-4x_latest + + - identifier: amazonlinux2_gcc7x_x86_64 + buildspec: ./tests/ci/codebuild/amazonlinux-2_gcc-7x-x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:amazonlinux-2_gcc-7x_latest + + - identifier: s2n_integration + buildspec: ./tests/ci/codebuild/s2n_integration.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:s2n_integration_clang-9x_latest + + - identifier: fedora31_gcc9x_x86_64 + buildspec: ./tests/ci/codebuild/fedora-31_gcc-9x_x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:fedora-31_gcc-9x_latest + + - identifier: fedora31_clang9x_x86_64 + buildspec: ./tests/ci/codebuild/fedora-31_clang-9x_x86-64.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:fedora-31_clang-9x_latest + + - identifier: ubuntu1910_clang9x_x86_64_sanitizer + buildspec: ./tests/ci/codebuild/ubuntu-19.10_clang-9x_x86-64_sanitizer.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_X86_PLACEHOLDER:ubuntu-19.10_clang-9x_sanitizer_latest + + ############### + # Linux aarch # + ############### + - identifier: ubuntu1910_gcc9x_aarch + buildspec: ./tests/ci/codebuild/ubuntu-19.10_gcc-9x_aarch64.yml + env: + type: ARM_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_AARCH_PLACEHOLDER:ubuntu-19.10_gcc-9x_latest + + - identifier: ubuntu1910_clang9x_aarch + buildspec: ./tests/ci/codebuild/ubuntu-19.10_clang-9x_aarch64.yml + env: + type: ARM_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_AARCH_PLACEHOLDER:ubuntu-19.10_clang-9x_latest + + - identifier: ubuntu1910_clang9x_aarch_sanitizer + buildspec: ./tests/ci/codebuild/ubuntu-19.10_clang-9x_aarch64_sanitizer.yml + env: + type: ARM_CONTAINER + privileged-mode: true + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_AARCH_PLACEHOLDER:ubuntu-19.10_clang-9x_sanitizer_latest + + ########### + # Windows # + ########### + - identifier: windows_msvc2015_x64 + buildspec: ./tests/ci/codebuild/windows-msvc2015-x64.yml + env: + type: WINDOWS_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_LARGE + image: AWS_ACCOUNT_ID_PLACEHOLDER.dkr.ecr.AWS_REGION_PLACEHOLDER.amazonaws.com/ECR_REPO_WINDOWS_PLACEHOLDER:vs2015_latest diff --git a/tests/ci/cdk/cdk/codebuild/linux_img_build_omnibus.yaml b/tests/ci/cdk/cdk/codebuild/linux_img_build_omnibus.yaml new file mode 100644 index 0000000000..3b1875d1d1 --- /dev/null +++ b/tests/ci/cdk/cdk/codebuild/linux_img_build_omnibus.yaml @@ -0,0 +1,30 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +version: 0.2 + +# Doc for batch https://docs.aws.amazon.com/codebuild/latest/userguide/batch-build-buildspec.html#build-spec.batch.build-list +batch: + build-list: + + ############# + # Linux x86 # + ############# + - identifier: linux_x86_docker_img_build + buildspec: ./tests/ci/codebuild/linux-x86-docker-img-build.yml + env: + type: LINUX_CONTAINER + privileged-mode: true + compute-type: BUILD_GENERAL1_LARGE + image: aws/codebuild/standard:4.0 + + ############### + # Linux aarch # + ############### + - identifier: linux_aarch_docker_img_build + buildspec: ./tests/ci/codebuild/linux-aarch-docker-img-build.yml + env: + type: ARM_CONTAINER + privileged-mode: true + compute-type: BUILD_GENERAL1_LARGE + image: aws/codebuild/amazonlinux2-aarch64-standard:1.0 diff --git a/tests/ci/cdk/cdk/ecr_stack.py b/tests/ci/cdk/cdk/ecr_stack.py index a3d3c6c1b6..8d8f4e65b9 100644 --- a/tests/ci/cdk/cdk/ecr_stack.py +++ b/tests/ci/cdk/cdk/ecr_stack.py @@ -7,8 +7,8 @@ class EcrStack(core.Stack): """Define a stack of ECR to store pre-built Docker Images.""" - def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: + def __init__(self, scope: core.Construct, id: str, repo_name: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) - ecr.Repository(scope=self, id=id, repository_name=id).grant_pull_push( + ecr.Repository(scope=self, id=id, repository_name=repo_name).grant_pull_push( iam.ServicePrincipal("codebuild.amazonaws.com")) diff --git a/tests/ci/cdk/cdk/github_codebuild_stack.py b/tests/ci/cdk/cdk/github_codebuild_stack.py deleted file mode 100644 index 5c616ade09..0000000000 --- a/tests/ci/cdk/cdk/github_codebuild_stack.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -import typing -from util.util import EnvUtil - -from aws_cdk import core, aws_codebuild as codebuild, aws_iam as iam, aws_ecr as ecr - - -class GitHubCodeBuildStack(core.Stack): - """Define a stack used to run AWS-LC tests.""" - - def __init__(self, - scope: core.Construct, - id: str, - ecr_repo_name: str, - docker_img_tag: str, - build_spec_file: str, - env_type: typing.Optional[str] = 'Linux', - privileged: typing.Optional[bool] = False, - **kwargs) -> None: - super().__init__(scope, id, **kwargs) - - # Fetch environment variables. - github_repo_owner = EnvUtil.get("GITHUB_REPO_OWNER", "awslabs") - github_repo = EnvUtil.get("GITHUB_REPO", "aws-lc") - - # Define CodeBuild resource. - git_hub_source = codebuild.Source.git_hub( - owner=github_repo_owner, - repo=github_repo, - webhook=True, - webhook_filters=[ - codebuild.FilterGroup.in_event_of( - codebuild.EventAction.PULL_REQUEST_CREATED, - codebuild.EventAction.PULL_REQUEST_UPDATED, - codebuild.EventAction.PULL_REQUEST_REOPENED) - ], - clone_depth=1) - - # Define CodeBuild environment. - ecr_repo = ecr.Repository.from_repository_name(scope=self, id=ecr_repo_name, repository_name=ecr_repo_name) - build_image = codebuild.LinuxBuildImage.from_ecr_repository(repository=ecr_repo, tag=docker_img_tag) - if env_type is 'Windows': - build_image = codebuild.WindowsBuildImage.from_ecr_repository(repository=ecr_repo, tag=docker_img_tag) - - # Define a role. - role = iam.Role(scope=self, - id="{}-role".format(id), - assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"), - managed_policies=[ - iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEC2ContainerRegistryReadOnly") - ]) - - # Define timeout. - if env_type is 'ARM': - # ARM sanitizer code build takes 90 minutes to complete. - timeout = core.Duration.minutes(120) - else: - timeout = core.Duration.minutes(60) - - # Define CodeBuild. - build = codebuild.Project( - scope=self, - id=id, - project_name=id, - source=git_hub_source, - role=role, - timeout=timeout, - environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.LARGE, - privileged=privileged, - build_image=build_image), - build_spec=codebuild.BuildSpec.from_source_filename(build_spec_file)) - - if env_type is 'ARM': - # Workaround to change environment type. - # see: https://github.com/aws/aws-cdk/issues/5517 - cfn_build = build.node.default_child - cfn_build.add_override("Properties.Environment.Type", "ARM_CONTAINER") diff --git a/tests/ci/cdk/cdk/linux_docker_image_batch_build_stack.py b/tests/ci/cdk/cdk/linux_docker_image_batch_build_stack.py new file mode 100644 index 0000000000..ffbb488c3c --- /dev/null +++ b/tests/ci/cdk/cdk/linux_docker_image_batch_build_stack.py @@ -0,0 +1,62 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from aws_cdk import core, aws_codebuild as codebuild, aws_iam as iam +from util.metadata import AWS_ACCOUNT, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, GITHUB_SOURCE_VERSION, LINUX_AARCH_ECR_REPO, \ + LINUX_X86_ECR_REPO +from util.iam_policies import codebuild_batch_policy_in_json, ecr_power_user_policy_in_json +from util.yml_loader import YmlLoader + + +class LinuxDockerImageBatchBuildStack(core.Stack): + """Define a temporary stack used to batch build Linux Docker images. After build, this stack will be destroyed.""" + + def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: + super().__init__(scope, id, **kwargs) + + # Define CodeBuild resource. + git_hub_source = codebuild.Source.git_hub( + owner=GITHUB_REPO_OWNER, + repo=GITHUB_REPO_NAME, + branch_or_ref=GITHUB_SOURCE_VERSION, + clone_depth=1) + + # Define a role. + codebuild_batch_policy = iam.PolicyDocument.from_json(codebuild_batch_policy_in_json([id])) + ecr_power_user_policy = iam.PolicyDocument.from_json(ecr_power_user_policy_in_json()) + inline_policies = {"codebuild_batch_policy": codebuild_batch_policy, + "ecr_power_user_policy": ecr_power_user_policy} + role = iam.Role(scope=self, + id="{}-role".format(id), + assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"), + inline_policies=inline_policies) + + # Create build spec. + build_spec_content = YmlLoader.load("./cdk/codebuild/linux_img_build_omnibus.yaml") + + # Define CodeBuild project. + project = codebuild.Project( + scope=self, + id=id, + project_name=id, + source=git_hub_source, + environment=codebuild.BuildEnvironment( + compute_type=codebuild.ComputeType.SMALL, + privileged=False, + build_image=codebuild.LinuxBuildImage.STANDARD_4_0), + environment_variables={ + "AWS_ACCOUNT_ID": codebuild.BuildEnvironmentVariable(value=AWS_ACCOUNT), + "AWS_ECR_REPO_X86": codebuild.BuildEnvironmentVariable(value=LINUX_X86_ECR_REPO), + "AWS_ECR_REPO_AARCH": codebuild.BuildEnvironmentVariable(value=LINUX_AARCH_ECR_REPO) + }, + role=role, + build_spec=codebuild.BuildSpec.from_object(build_spec_content)) + + # Add 'BuildBatchConfig' property, which is not supported in CDK. + # CDK raw overrides: https://docs.aws.amazon.com/cdk/latest/guide/cfn_layer.html#cfn_layer_raw + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-codebuild-project.html#aws-resource-codebuild-project-properties + cfn_build = project.node.default_child + cfn_build.add_override("Properties.BuildBatchConfig", { + "ServiceRole": role.role_arn, + "TimeoutInMins": 120 + }) diff --git a/tests/ci/cdk/cdk/linux_docker_images_build_stack.py b/tests/ci/cdk/cdk/linux_docker_images_build_stack.py deleted file mode 100644 index 97e66a91e6..0000000000 --- a/tests/ci/cdk/cdk/linux_docker_images_build_stack.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -import typing -from util.util import EnvUtil - -from aws_cdk import core, aws_codebuild as codebuild, aws_iam as iam - - -class LinuxDockerImagesBuildStack(core.Stack): - """Define a stack used to build Linux Docker images.""" - - def __init__(self, scope: core.Construct, id: str, ecr_repo: str, - build_spec_file: str, env_type: typing.Optional[str] = 'Linux', **kwargs) -> None: - super().__init__(scope, id, **kwargs) - - # Fetch environment variables. - github_repo_owner = EnvUtil.get("GITHUB_REPO_OWNER", "awslabs") - github_repo = EnvUtil.get("GITHUB_REPO", "aws-lc") - - # Define CodeBuild resource. - git_hub_source = codebuild.Source.git_hub( - owner=github_repo_owner, - repo=github_repo, - webhook=True, - clone_depth=1) - - # Define a role. - env = kwargs['env'] - ecr_power_user_policy = iam.PolicyDocument.from_json( - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:DescribeImages", - "ecr:BatchGetImage", - "ecr:GetLifecyclePolicy", - "ecr:GetLifecyclePolicyPreview", - "ecr:ListTagsForResource", - "ecr:DescribeImageScanFindings", - "ecr:InitiateLayerUpload", - "ecr:UploadLayerPart", - "ecr:CompleteLayerUpload", - "ecr:PutImage" - ], - "Resource": "arn:aws:ecr:{}:{}:repository/{}".format(env['region'], env['account'], ecr_repo) - } - ] - } - ) - inline_policies = {"ecr_power_user_policy": ecr_power_user_policy} - role = iam.Role(scope=self, - id="{}-role".format(id), - assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"), - inline_policies=inline_policies) - - # Define build img. - build_image = codebuild.LinuxBuildImage.STANDARD_2_0 - if env_type is 'ARM': - build_image = codebuild.LinuxBuildImage.AMAZON_LINUX_2_ARM - - # Define CodeBuild project. - project = codebuild.Project( - scope=self, - id=id, - project_name=ecr_repo, - source=git_hub_source, - environment=codebuild.BuildEnvironment( - compute_type=codebuild.ComputeType.LARGE, - privileged=True, - build_image=build_image), - environment_variables={ - "AWS_ACCOUNT_ID": codebuild.BuildEnvironmentVariable(value=kwargs['env']['account']), - "AWS_ECR_REPO": codebuild.BuildEnvironmentVariable(value=ecr_repo) - }, - role=role, - build_spec=codebuild.BuildSpec.from_source_filename( - build_spec_file)) diff --git a/tests/ci/cdk/cdk/ssm/windows_docker_build_ssm_document.yaml b/tests/ci/cdk/cdk/ssm/windows_docker_build_ssm_document.yaml new file mode 100644 index 0000000000..19445bb64e --- /dev/null +++ b/tests/ci/cdk/cdk/ssm/windows_docker_build_ssm_document.yaml @@ -0,0 +1,26 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +schemaVersion: '2.2' +description: aws-lc:buildWindowsDockerImages +mainSteps: +- action: aws:runPowerShellScript + name: runPowerShellScript + inputs: + timeoutSeconds: '7200' + runCommand: + - mkdir docker-images + - cd docker-images + # Install choco and git + - Set-ExecutionPolicy Bypass -Scope Process -Force; [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; $env:chocolateyUseWindowsCompression = 'true'; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) | Out-Null + - choco install git --version 2.23.0 -y + - $env:path+='C:\Program Files\Git\cmd' + # Git clone aws-lc repo. + - git clone https://github.com/GITHUB_OWNER_PLACEHOLDER/aws-lc.git + # Build Windows docker images. + - cd .\aws-lc\tests\ci\docker_images\windows + - git checkout GITHUB_SOURCE_VERSION_PLACEHOLDER + - Invoke-Expression -Command (Get-ECRLoginCommand -Region REGION_PLACEHOLDER).Command + - .\build_images.ps1 + - .\push_images.ps1 ECR_PLACEHOLDER diff --git a/tests/ci/cdk/cdk/windows_docker_build_ssm_document.yaml b/tests/ci/cdk/cdk/windows_docker_build_ssm_document.yaml deleted file mode 100644 index f9f4459734..0000000000 --- a/tests/ci/cdk/cdk/windows_docker_build_ssm_document.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -schemaVersion: '2.2' -description: aws-lc:buildWindowsDockerImages -mainSteps: -- action: aws:runPowerShellScript - name: runPowerShellScript - inputs: - timeoutSeconds: '7200' - runCommand: - - mkdir docker-images - - cd docker-images - - Read-S3Object -BucketName S3_BUCKET_PLACEHOLDER -Key windows.zip -File windows.zip - - Expand-Archive .\windows.zip -DestinationPath .\ - - cd .\windows - - Invoke-Expression -Command (Get-ECRLoginCommand -Region REGION_PLACEHOLDER).Command - - .\build_images.ps1 - - .\push_images.ps1 ECR_PLACEHOLDER diff --git a/tests/ci/cdk/cdk/windows_docker_image_build_stack.py b/tests/ci/cdk/cdk/windows_docker_image_build_stack.py index be5302cce8..cd030782da 100644 --- a/tests/ci/cdk/cdk/windows_docker_image_build_stack.py +++ b/tests/ci/cdk/cdk/windows_docker_image_build_stack.py @@ -1,103 +1,43 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from aws_cdk import core, aws_ec2 as ec2, aws_iam as iam, aws_s3 as s3, aws_ssm as ssm -import yaml -from util.util import EnvUtil +from aws_cdk import core, aws_ec2 as ec2, aws_s3 as s3, aws_iam as iam, aws_ssm as ssm +from util.iam_policies import ecr_power_user_policy_in_json, s3_read_write_policy_in_json +from util.metadata import AWS_ACCOUNT, AWS_REGION, WINDOWS_ECR_REPO, S3_BUCKET_NAME, GITHUB_REPO_OWNER, WIN_EC2_TAG_KEY, \ + WIN_EC2_TAG_VALUE, SSM_DOCUMENT_NAME, GITHUB_SOURCE_VERSION +from util.yml_loader import YmlLoader class WindowsDockerImageBuildStack(core.Stack): - """Define a stack used to build Windows Docker images.""" + """Define a temporary stack used to build Windows Docker images. After build, this stack will be destroyed.""" def __init__(self, scope: core.Construct, id: str, - ecr_repo: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) - # Fetch environment variables. - s3_bucket_name = EnvUtil.get("S3_FOR_WIN_DOCKER_IMG_BUILD", "windows-docker-images") - win_ec2_tag_key = EnvUtil.get("WIN_EC2_TAG_KEY", "aws-lc") - win_ec2_tag_value = EnvUtil.get("WIN_EC2_TAG_VALUE", "aws-lc-windows") - ssm_document_name = EnvUtil.get("WIN_DOCKER_BUILD_SSM_DOCUMENT", "aws-lc-windows-docker-ssm-doc") + # Define SSM command document. + ecr_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format(AWS_ACCOUNT, AWS_REGION, WINDOWS_ECR_REPO) + placeholder_map = {"ECR_PLACEHOLDER": ecr_repo, "GITHUB_OWNER_PLACEHOLDER": GITHUB_REPO_OWNER, + "REGION_PLACEHOLDER": AWS_REGION, "GITHUB_SOURCE_VERSION_PLACEHOLDER": GITHUB_SOURCE_VERSION} + content = YmlLoader.load("./cdk/ssm/windows_docker_build_ssm_document.yaml", placeholder_map) + ssm.CfnDocument(scope=self, + id="{}-ssm-document".format(id), + name=SSM_DOCUMENT_NAME, + content=content, + document_type="Command") # Define a S3 bucket to store windows docker files and build scripts. s3.Bucket(scope=self, id="{}-s3".format(id), - bucket_name=s3_bucket_name, + bucket_name=S3_BUCKET_NAME, block_public_access=s3.BlockPublicAccess.BLOCK_ALL) - # Define SSM command document. - aws_account_id = kwargs["env"]["account"] - aws_region = kwargs["env"]["region"] - ecr_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format(aws_account_id, aws_region, ecr_repo) - with open('./cdk/windows_docker_build_ssm_document.yaml') as file: - file_text = file.read().replace("ECR_PLACEHOLDER", ecr_repo) \ - .replace("S3_BUCKET_PLACEHOLDER", s3_bucket_name) \ - .replace("REGION_PLACEHOLDER", aws_region) - content = yaml.load(file_text, Loader=yaml.FullLoader) - ssm.CfnDocument(scope=self, - id="{}-ssm-document".format(id), - name=ssm_document_name, - content=content, - document_type="Command") - # Define a role for EC2. - s3_read_write_policy = iam.PolicyDocument.from_json( - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:Put*", - "s3:Get*" - ], - "Resource": [ - "arn:aws:s3:::{}/*".format(s3_bucket_name) - ] - } - ] - } - ) - env = kwargs['env'] - ecr_power_user_policy = iam.PolicyDocument.from_json( - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:DescribeImages", - "ecr:BatchGetImage", - "ecr:GetLifecyclePolicy", - "ecr:GetLifecyclePolicyPreview", - "ecr:ListTagsForResource", - "ecr:DescribeImageScanFindings", - "ecr:InitiateLayerUpload", - "ecr:UploadLayerPart", - "ecr:CompleteLayerUpload", - "ecr:PutImage" - ], - "Resource": "arn:aws:ecr:{}:{}:repository/{}".format(env['region'], env['account'], ecr_repo) - } - ] - } - ) - inline_policies = {"s3_read_write_policy": s3_read_write_policy, "ecr_power_user_policy": ecr_power_user_policy} + ecr_power_user_policy = iam.PolicyDocument.from_json(ecr_power_user_policy_in_json()) + s3_read_write_policy = iam.PolicyDocument.from_json(s3_read_write_policy_in_json(S3_BUCKET_NAME)) + inline_policies = {"ecr_power_user_policy": ecr_power_user_policy, "s3_read_write_policy": s3_read_write_policy} role = iam.Role(scope=self, id="{}-role".format(id), assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), inline_policies=inline_policies, @@ -119,4 +59,4 @@ def __init__(self, vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), machine_image=machine_image) - core.Tag.add(instance, win_ec2_tag_key, win_ec2_tag_value) + core.Tags.of(instance).add(WIN_EC2_TAG_KEY, WIN_EC2_TAG_VALUE) diff --git a/tests/ci/cdk/run-cdk.sh b/tests/ci/cdk/run-cdk.sh index 83cb6bb9ba..3c9c800c00 100755 --- a/tests/ci/cdk/run-cdk.sh +++ b/tests/ci/cdk/run-cdk.sh @@ -25,31 +25,45 @@ shift # Export other environment variables. export DATE_NOW="$(date +%Y-%m-%d-%H-%M)" -export ECR_LINUX_AARCH_REPO_NAME="aws-lc-test-docker-images-linux-aarch" -export ECR_LINUX_X86_REPO_NAME="aws-lc-test-docker-images-linux-x86" -export ECR_WINDOWS_REPO_NAME="aws-lc-test-docker-images-windows" -export S3_FOR_WIN_DOCKER_IMG_BUILD="windows-docker-images-${DATE_NOW}" +export ECR_LINUX_AARCH_REPO_NAME="aws-lc-docker-images-linux-aarch" +export ECR_LINUX_X86_REPO_NAME="aws-lc-docker-images-linux-x86" +export ECR_WINDOWS_REPO_NAME="aws-lc-docker-images-windows" +export AWS_LC_S3_BUCKET_PREFIX="aws-lc-windows-docker-image-build" +export S3_FOR_WIN_DOCKER_IMG_BUILD="${AWS_LC_S3_BUCKET_PREFIX}-${DATE_NOW}" export WIN_EC2_TAG_KEY="aws-lc" -export WIN_EC2_TAG_VALUE="windows-docker-img-${DATE_NOW}" +export WIN_EC2_TAG_VALUE="aws-lc-windows-docker-image-build-${DATE_NOW}" export WIN_DOCKER_BUILD_SSM_DOCUMENT="windows-ssm-document-${DATE_NOW}" # Functions +function delete_s3_buckets() { + aws s3api list-buckets --query "Buckets[].Name" | jq '.[]' | while read -r i; do + # Delete the bucket if its name uses AWS_LC_S3_BUCKET_PREFIX. + if [[ ${i} == "${AWS_LC_S3_BUCKET_PREFIX}"* ]]; then + aws s3 rm "s3://${i}" --recursive + aws s3api delete-bucket --bucket "${i}" + fi + done +} + +function destroy_all() { + cdk destroy aws-lc-* --force + # CDK stack destroy does not delete s3 bucket automatically. + delete_s3_buckets +} + function create_aws_resources() { - cdk deploy aws-lc-test-* --require-approval never + # Clean up resources before deployment. + destroy_all + cdk deploy aws-lc-* --require-approval never + # Need to use aws cli to change webhook build type because CFN is not ready yet. + aws codebuild update-webhook --project-name aws-lc-ci --build-type BUILD_BATCH } function build_linux_img() { - aws codebuild start-build --project-name ${ECR_LINUX_AARCH_REPO_NAME} - aws codebuild start-build --project-name ${ECR_LINUX_X86_REPO_NAME} + aws codebuild start-build-batch --project-name aws-lc-docker-image-build-linux } function build_windows_img() { - # Upload windows docker file and build scripts to S3. - cd ../docker_images - zip -r -X "windows.zip" "./windows" - aws s3 cp "windows.zip" "s3://${S3_FOR_WIN_DOCKER_IMG_BUILD}/" - rm windows.zip - # EC2 takes 10 min to be ready for running command. echo "Wait 10 min for EC2 ready for SSM command execution." sleep 600 @@ -81,7 +95,7 @@ function build_windows_img() { function destroy_docker_img_build_stack() { # Destroy all temporary resources created for all docker image build. - cdk destroy aws-lc-test-docker-images-build-* --force + cdk destroy aws-lc-docker-image-build-* --force } function images_pushed_to_ecr() { @@ -89,7 +103,8 @@ function images_pushed_to_ecr() { shift target_images=("$@") ecr_repo_name="${CDK_DEPLOY_ACCOUNT}.dkr.ecr.${CDK_DEPLOY_REGION}.amazonaws.com/${repo_name}" - echo "Checking if docker images [${target_images[@]}] are pushed to ${ecr_repo_name}." + docker_images=$* + echo "Checking if docker images [${docker_images}] are pushed to ${ecr_repo_name}." # Every 5 min, this function checks if the target docker img is created. # Normally, docker img build can take up to 1 hour. Here, we wait up to 30 * 5 min. @@ -97,7 +112,7 @@ function images_pushed_to_ecr() { images_in_ecr=$(aws ecr describe-images --repository-name ${repo_name}) images_pushed=0 for target_image in "${target_images[@]}"; do - if [[ ${images_in_ecr} != *"$target_image"* ]]; then + if [[ ${images_in_ecr} != *"${target_image}"* ]]; then images_pushed=1 break fi @@ -148,10 +163,6 @@ function deploy() { images_pushed_to_ecr "${ECR_WINDOWS_REPO_NAME}" "${windows_img_tags[@]}" } -function destroy() { - cdk destroy aws-lc-test-* --force -} - # Main logics case ${ACTION} in @@ -165,7 +176,7 @@ SYNTH) cdk synth aws-lc* ;; DESTROY) - destroy + destroy_all ;; *) echo "Action: ${ACTION} is not supported." diff --git a/tests/ci/cdk/setup.py b/tests/ci/cdk/setup.py index ce0f1e7c09..8bbfa7ad04 100644 --- a/tests/ci/cdk/setup.py +++ b/tests/ci/cdk/setup.py @@ -20,10 +20,10 @@ install_requires=[ # CDK dependencies. - "aws-cdk.core==1.48.0", - "aws-cdk.aws-codebuild==1.48.0", - "aws-cdk.aws-ecr==1.48.0", - "aws-cdk.aws-iam==1.48.0", + "aws-cdk.core==1.64.0", + "aws-cdk.aws-codebuild==1.64.0", + "aws-cdk.aws-ecr==1.64.0", + "aws-cdk.aws-iam==1.64.0", # PyYAML is a YAML parser and emitter for Python. Used to read build_spec.yaml. "pyyaml==5.3.1", # A formatter for Python code. diff --git a/tests/ci/cdk/util/util.py b/tests/ci/cdk/util/env_util.py similarity index 79% rename from tests/ci/cdk/util/util.py rename to tests/ci/cdk/util/env_util.py index 636bc0dc56..a17990e827 100644 --- a/tests/ci/cdk/util/util.py +++ b/tests/ci/cdk/util/env_util.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + import os import typing diff --git a/tests/ci/cdk/util/iam_policies.py b/tests/ci/cdk/util/iam_policies.py new file mode 100644 index 0000000000..dee65a0ea3 --- /dev/null +++ b/tests/ci/cdk/util/iam_policies.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from util.metadata import AWS_REGION, AWS_ACCOUNT, LINUX_AARCH_ECR_REPO, LINUX_X86_ECR_REPO, WINDOWS_ECR_REPO + + +def codebuild_batch_policy_in_json(project_ids): + """ + Define an IAM policy statement for CodeBuild batch operation. + :param project_ids: a list of CodeBuild project id. + :return: an IAM policy statement in json. + """ + resources = [] + for project_id in project_ids: + resources.append("arn:aws:codebuild:{}:{}:project/{}*".format(AWS_REGION, AWS_ACCOUNT, project_id)) + return { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "codebuild:StartBuild", + "codebuild:StopBuild", + "codebuild:RetryBuild" + ], + "Resource": resources + } + ] + } + + +def s3_read_write_policy_in_json(s3_bucket_name): + """ + Define an IAM policy statement for reading and writing to S3 bucket. + :return: an IAM policy statement in json. + """ + return { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:Put*", + "s3:Get*" + ], + "Resource": [ + "arn:aws:s3:::{}/*".format(s3_bucket_name) + ] + } + ] + } + + +def ecr_power_user_policy_in_json(): + """ + Define an AWS-LC specific IAM policy statement for AWS ECR power user used to create new docker images. + :return: an IAM policy statement in json. + """ + ecr_arn_prefix = "arn:aws:ecr:{}:{}:repository".format(AWS_REGION, AWS_ACCOUNT) + linux_x86_ecr_arn = "{}/{}".format(ecr_arn_prefix, LINUX_X86_ECR_REPO) + linux_aarch_ecr_arn = "{}/{}".format(ecr_arn_prefix, LINUX_AARCH_ECR_REPO) + windows_ecr_arn = "{}/{}".format(ecr_arn_prefix, WINDOWS_ECR_REPO) + return { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages", + "ecr:BatchGetImage", + "ecr:GetLifecyclePolicy", + "ecr:GetLifecyclePolicyPreview", + "ecr:ListTagsForResource", + "ecr:DescribeImageScanFindings", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:PutImage" + ], + "Resource": [ + linux_x86_ecr_arn, + linux_aarch_ecr_arn, + windows_ecr_arn + ] + } + ] + } diff --git a/tests/ci/cdk/util/metadata.py b/tests/ci/cdk/util/metadata.py new file mode 100644 index 0000000000..4f1ca3ddb7 --- /dev/null +++ b/tests/ci/cdk/util/metadata.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from util.env_util import EnvUtil + +# Used when AWS CDK defines AWS resources. +AWS_ACCOUNT = EnvUtil.get("CDK_DEPLOY_ACCOUNT", "620771051181") +AWS_REGION = EnvUtil.get("CDK_DEPLOY_REGION", "us-west-2") + +# Used when AWS CDK defines ECR repos. +LINUX_AARCH_ECR_REPO = EnvUtil.get("ECR_LINUX_AARCH_REPO_NAME", "aws-lc-docker-images-linux-aarch") +LINUX_X86_ECR_REPO = EnvUtil.get("ECR_LINUX_X86_REPO_NAME", "aws-lc-docker-images-linux-x86") +WINDOWS_ECR_REPO = EnvUtil.get("ECR_WINDOWS_REPO_NAME", "aws-lc-docker-images-windows") + +# Used when AWS CodeBuild needs to create web_hooks. +GITHUB_REPO_OWNER = EnvUtil.get("GITHUB_REPO_OWNER", "awslabs") +GITHUB_REPO_NAME = EnvUtil.get("GITHUB_REPO", "aws-lc") +GITHUB_SOURCE_VERSION = EnvUtil.get("GITHUB_SOURCE_VERSION", "main") + +# Used when AWS CDK defines resources for Windows docker image build. +S3_BUCKET_NAME = EnvUtil.get("S3_FOR_WIN_DOCKER_IMG_BUILD", "aws-lc-windows-docker-image-build") +WIN_EC2_TAG_KEY = EnvUtil.get("WIN_EC2_TAG_KEY", "aws-lc") +WIN_EC2_TAG_VALUE = EnvUtil.get("WIN_EC2_TAG_VALUE", "aws-lc-windows-docker-image-build") +SSM_DOCUMENT_NAME = EnvUtil.get("WIN_DOCKER_BUILD_SSM_DOCUMENT", "windows-ssm-document") diff --git a/tests/ci/cdk/util/yml_loader.py b/tests/ci/cdk/util/yml_loader.py new file mode 100644 index 0000000000..c0c1e27983 --- /dev/null +++ b/tests/ci/cdk/util/yml_loader.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + + +import typing +import yaml + + +class YmlLoader(object): + """Responsible for loading yml file as python object.""" + + @staticmethod + def load(file_path, placeholder_map: typing.Optional[typing.Mapping[str, str]] = {}): + """ + Used to load yml file and replace some placeholders if needed. + :param file_path: path to the yml file. + :param placeholder_map: a mapping from placeholder to corresponding value. + :return: python object. + """ + with open(file_path) as file: + file_text = file.read() + for key in placeholder_map.keys(): + file_text = file_text.replace(key, placeholder_map[key]) + return yaml.safe_load(file_text) diff --git a/tests/ci/codebuild/linux-aarch-docker-img-build.yml b/tests/ci/codebuild/linux-aarch-docker-img-build.yml index 8f0d0395ab..819fe8394f 100644 --- a/tests/ci/codebuild/linux-aarch-docker-img-build.yml +++ b/tests/ci/codebuild/linux-aarch-docker-img-build.yml @@ -17,4 +17,4 @@ phases: commands: - cd ./tests/ci/docker_images/linux-aarch - ./build_images.sh - - ./push_images.sh $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$AWS_ECR_REPO + - ./push_images.sh $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$AWS_ECR_REPO_AARCH diff --git a/tests/ci/codebuild/linux-x86-docker-img-build.yml b/tests/ci/codebuild/linux-x86-docker-img-build.yml index 2cb79c051b..489483fa8e 100644 --- a/tests/ci/codebuild/linux-x86-docker-img-build.yml +++ b/tests/ci/codebuild/linux-x86-docker-img-build.yml @@ -16,4 +16,4 @@ phases: commands: - cd ./tests/ci/docker_images/linux-x86 - ./build_images.sh - - ./push_images.sh $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$AWS_ECR_REPO + - ./push_images.sh $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$AWS_ECR_REPO_X86 diff --git a/tests/ci/codebuild/windows-msvc2015-x64.yml b/tests/ci/codebuild/windows-msvc2015-x64.yml index 9b2bf6a50b..e83fe2d76d 100644 --- a/tests/ci/codebuild/windows-msvc2015-x64.yml +++ b/tests/ci/codebuild/windows-msvc2015-x64.yml @@ -6,5 +6,9 @@ version: 0.2 phases: build: commands: + # Installing cyg-get and quilt during Docker image build can result in https://github.com/microsoft/hcsshim/issues/835. + - Set-ExecutionPolicy Bypass -Scope Process -Force; [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; $env:chocolateyUseWindowsCompression = 'true'; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) | Out-Null + - choco install cyg-get -y + - cyg-get install quilt # vcvarsall will set the required lib and libpath for MSVC to compile everything # - .\tests\ci\run_windows_tests.bat "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" diff --git a/tests/ci/docker_images/windows/windows_base/Dockerfile b/tests/ci/docker_images/windows/windows_base/Dockerfile index 6d662b5470..1c851c36de 100644 --- a/tests/ci/docker_images/windows/windows_base/Dockerfile +++ b/tests/ci/docker_images/windows/windows_base/Dockerfile @@ -3,7 +3,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -FROM mcr.microsoft.com/windows/servercore:1607-KB4556813 +FROM mcr.microsoft.com/windows/servercore:ltsc2016 ADD https://download.microsoft.com/download/6/A/A/6AA4EDFF-645B-48C5-81CC-ED5963AEAD48/vc_redist.x64.exe /vc_redist.x64.exe RUN start /wait C:\vc_redist.x64.exe /quiet /norestart @@ -17,9 +17,6 @@ choco install ninja --version 1.9.0.20190208 -y && ` choco install activeperl --version 5.24.3.2404 -y && ` choco install nasm --version 2.14.02 -y && ` choco install golang --version 1.13.1 -y && ` -choco install cmake --version 3.15.4 --installargs 'ADD_CMAKE_TO_PATH=""System""' -y && ` -choco install cyg-get -y - -RUN cyg-get install quilt +choco install cmake --version 3.15.4 --installargs 'ADD_CMAKE_TO_PATH=""System""' -y CMD [ "cmd.exe" ] diff --git a/tests/ci/run_posix_sanitizers.sh b/tests/ci/run_posix_sanitizers.sh index fd3f8f1575..9d5c753852 100755 --- a/tests/ci/run_posix_sanitizers.sh +++ b/tests/ci/run_posix_sanitizers.sh @@ -27,9 +27,9 @@ fi if [ $(dpkg --print-architecture) == "amd64" ]; then # x86 TSAN runs get stuck on PoolTest.Threads for over an hour https://github.com/awslabs/aws-lc/issues/13 - echo "Building AWS-LC in ${build_type} mode with memory sanitizer." + echo "Building AWS-LC in ${build_type} mode with thread sanitizer." run_build -DTSAN=1 -DUSE_CUSTOM_LIBCXX=1 "${cflags[@]}" else - echo "Testing AWS-LC in ${build_type} mode with memory sanitizer." + echo "Testing AWS-LC in ${build_type} mode with thread sanitizer." build_and_test -DTSAN=1 -DUSE_CUSTOM_LIBCXX=1 "${cflags[@]}" fi