diff --git a/.github/workflows/e2e-test-trusted.yaml b/.github/workflows/e2e-test-trusted.yaml new file mode 100644 index 00000000..8156581f --- /dev/null +++ b/.github/workflows/e2e-test-trusted.yaml @@ -0,0 +1,20 @@ +name: "E2E Tests (Trusted)" + +on: + push: + branches: [ "main", "feature/**", "release-**", "workflow/**" ] + merge_group: + types: [ "checks_requested" ] + +permissions: + id-token: write + contents: read + +jobs: + e2e: + name: E2E Tests + uses: ./.github/workflows/e2e-tests.yaml + with: + environment: "trusted" + ref: ${{ github.event_name == 'push' && github.sha || github.event.pull_request.head.sha }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/e2e-test-untrusted.yaml b/.github/workflows/e2e-test-untrusted.yaml new file mode 100644 index 00000000..6ce0570c --- /dev/null +++ b/.github/workflows/e2e-test-untrusted.yaml @@ -0,0 +1,17 @@ +name: "E2E Tests (Untrusted)" + +on: + pull_request_target: + +permissions: + id-token: write + contents: read + +jobs: + e2e: + name: E2E Tests + uses: ./.github/workflows/e2e-tests.yaml + with: + environment: "untrusted" + ref: ${{ github.event.pull_request.head.sha }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 6217745a..199b7c08 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -1,72 +1,58 @@ name: E2E Tests on: - push: - branches: ["main", "release-**", "feature/*"] - pull_request: - branches: ["main", "feature/*"] - paths: - - "tests/**" - - "pkg/**" - - "cmd/**" - - "charts/**" - - ".github/workflows/**" - - "Dockerfile" + workflow_call: + inputs: + environment: + required: true + type: string + ref: + required: true + type: string -# This workflow runs e2e tests and relies on existence of EKS cluster with a `s3-csi-driver-sa` service account -# already deployed to it, which provides the driver with access to s3. -# -# Since we have a single cluster for e2e tests, we ensure that no more than one instance of this workflow is -# running by `concurrency: e2e-cluster` option. -# -# Successful workflows triggered by push to main will upload tested image to the private repository "PROMOTED_IMAGE_NAME": -# - uploaded images will be tagged with main branch commit number -# - uploaded images will be later promoted to public repository by "release" workflow concurrency: e2e-cluster env: - AWS_REGION: "us-east-1" - COMMIT_ID: ${{ github.event_name == 'push' && github.sha || github.event.pull_request.head.sha }} - TMP_IMAGE_NAME: "s3-csi-driver-tmp" - PROMOTED_IMAGE_NAME: "s3-csi-driver" - BENCHMARK_RESULTS_BUCKET: "s3://mountpoint-s3-csi-driver-benchmark" - BENCHMARK_RESULTS_REGION: "us-east-1" + IMAGE_NAME: "s3-csi-driver" BENCHMARK_ARTIFACTS_FOLDER: ".github/artifacts" BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + KOPS_STATE_FILE: "s3://${{ vars.KOPS_STATE_FILE }}" + BENCHMARK_BUCKET: "s3://${{ vars.BENCHMARK_BUCKET }}" + TAG_UNTESTED: "untested_${{ inputs.ref }}" + TAG_PASSED: "test_passed_${{ inputs.ref }}" jobs: build: - # this is to prevent the job to run at forked projects - if: github.repository == 'awslabs/mountpoint-s3-csi-driver' runs-on: ubuntu-latest + environment: ${{ inputs.environment }} permissions: id-token: write contents: read steps: - name: Checkout uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false - name: Set up Go uses: actions/setup-go@v4 with: go-version-file: "go.mod" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@master with: - role-to-assume: ${{ secrets.TEST_IAM_ROLE }} - aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ vars.IAM_ROLE }} + aws-region: ${{ vars.AWS_REGION }} - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - name: Build, tag, and push docker image to Amazon ECR Private Repository env: REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_NAME: ${{ env.TMP_IMAGE_NAME }} + PLATFORM: "linux/amd64,linux/arm64" + TAG: "${{ env.TAG_UNTESTED }}" run: | - export PLATFORM=linux/amd64,linux/arm64 - export TAG=${{ env.COMMIT_ID }} - make -j `nproc` all-push + make -j `nproc` all-push-skip-if-present test: needs: build strategy: @@ -113,12 +99,26 @@ jobs: - cluster-type: "kops" kubernetes-version: "1.31.0" runs-on: ubuntu-latest + environment: ${{ inputs.environment }} permissions: id-token: write contents: read + env: + AWS_REGION: "${{ vars.AWS_REGION }}" + CLUSTER_TYPE: "${{ matrix.cluster-type }}" + ARCH: "${{ matrix.arch }}" + AMI_FAMILY: "${{ matrix.family }}" + TAG: "untested_${{ inputs.ref }}" + # envtest doesn't support all versions, here K8S_VERSION is a full version like 1.28.13, + # and in order to get latest supported version by envtest we convert it to 1.28. + K8S_VERSION: "${{ matrix.kubernetes-version }}" + ENVTEST_K8S_VERSION: "${K8S_VERSION%.*}" steps: - name: Checkout uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false - name: Set up Go uses: actions/setup-go@v4 with: @@ -127,77 +127,49 @@ jobs: with: python-version: "3.10" - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@master + uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.TEST_IAM_ROLE }} - aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ vars.IAM_ROLE }} + aws-region: ${{ vars.AWS_REGION }} - name: Install tools + env: + ACTION: "install_tools" run: | - export ACTION=install_tools - export K8S_VERSION=${{ matrix.kubernetes-version }} tests/e2e-kubernetes/scripts/run.sh - name: Run Controller Tests run: | - # envtest doesn't support all versions, here K8S_VERSION is a full version like 1.28.13, - # and in order to get latest supported version by envtest we convert it to 1.28. - export K8S_VERSION=${{ matrix.kubernetes-version }} - export ENVTEST_K8S_VERSION="${K8S_VERSION%.*}" make e2e-controller - name: Create cluster + env: + ACTION: "create_cluster" run: | - export ACTION=create_cluster - export AWS_REGION=${{ env.AWS_REGION }} - export CLUSTER_TYPE=${{ matrix.cluster-type }} - export ARCH=${{ matrix.arch }} - export AMI_FAMILY=${{ matrix.family }} - export K8S_VERSION=${{ matrix.kubernetes-version }} tests/e2e-kubernetes/scripts/run.sh - name: Update kubeconfig + env: + ACTION: "update_kubeconfig" run: | - export ACTION=update_kubeconfig - export AWS_REGION=${{ env.AWS_REGION }} - export CLUSTER_TYPE=${{ matrix.cluster-type }} - export ARCH=${{ matrix.arch }} - export AMI_FAMILY=${{ matrix.family }} - export K8S_VERSION=${{ matrix.kubernetes-version }} tests/e2e-kubernetes/scripts/run.sh - name: Install the driver + env: + ACTION: "install_driver" run: | - export ACTION=install_driver - export AWS_REGION=${{ env.AWS_REGION }} - export CLUSTER_TYPE=${{ matrix.cluster-type }} - export IMAGE_NAME=${{ env.TMP_IMAGE_NAME }} - export TAG=${{ env.COMMIT_ID }} - export ARCH=${{ matrix.arch }} - export AMI_FAMILY=${{ matrix.family }} - export K8S_VERSION=${{ matrix.kubernetes-version }} tests/e2e-kubernetes/scripts/run.sh - name: Run E2E Tests + env: + ACTION: "run_tests" run: | - export ACTION=run_tests - export AWS_REGION=${{ env.AWS_REGION }} - export CLUSTER_TYPE=${{ matrix.cluster-type }} - export TAG=${{ env.COMMIT_ID }} - export ARCH=${{ matrix.arch }} - export AMI_FAMILY=${{ matrix.family }} - export K8S_VERSION=${{ matrix.kubernetes-version }} tests/e2e-kubernetes/scripts/run.sh - name: Run Performance Tests if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' + env: + ACTION: "run_perf" run: | - export ACTION=run_perf - export AWS_REGION=${{ env.AWS_REGION }} - export CLUSTER_TYPE=${{ matrix.cluster-type }} - export TAG=${{ env.COMMIT_ID }} - export ARCH=${{ matrix.arch }} - export AMI_FAMILY=${{ matrix.family }} - export K8S_VERSION=${{ matrix.kubernetes-version }} tests/e2e-kubernetes/scripts/run.sh - name: Download previous benchmark results if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' run: | mkdir -p ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} - aws s3 cp --region ${{ env.BENCHMARK_RESULTS_REGION }} ${{ env.BENCHMARK_RESULTS_BUCKET }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json || true + aws s3 cp --region ${{ vars.BENCHMARK_RESULTS_REGION }} ${{ vars.BENCHMARK_BUCKET }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json || true - name: Update benchmark result file if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' uses: benchmark-action/github-action-benchmark@v1 @@ -212,59 +184,45 @@ jobs: if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' run: | tests/e2e-kubernetes/scripts/format_benchmark_data.py ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/quicksight-data.json - aws s3 cp ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} s3://mountpoint-s3-csi-driver-benchmark --recursive + aws s3 cp ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} ${{ env.BENCHMARK_BUCKET }} --recursive - name: Post e2e cleanup if: always() + env: + ACTION: "e2e_cleanup" run: | - export ACTION=e2e_cleanup - export AWS_REGION=${{ env.AWS_REGION }} - export CLUSTER_TYPE=${{ matrix.cluster-type }} - export ARCH=${{ matrix.arch }} - export AMI_FAMILY=${{ matrix.family }} - export K8S_VERSION=${{ matrix.kubernetes-version }} tests/e2e-kubernetes/scripts/run.sh - name: Uninstall the driver if: always() + env: + ACTION: "uninstall_driver" run: | - export ACTION=uninstall_driver - export AWS_REGION=${{ env.AWS_REGION }} - export CLUSTER_TYPE=${{ matrix.cluster-type }} - export ARCH=${{ matrix.arch }} - export AMI_FAMILY=${{ matrix.family }} - export K8S_VERSION=${{ matrix.kubernetes-version }} tests/e2e-kubernetes/scripts/run.sh - name: Delete cluster if: always() + env: + ACTION: "delete_cluster" run: | - export ACTION=delete_cluster - export AWS_REGION=${{ env.AWS_REGION }} - export CLUSTER_TYPE=${{ matrix.cluster-type }} - export ARCH=${{ matrix.arch }} - export AMI_FAMILY=${{ matrix.family }} - export K8S_VERSION=${{ matrix.kubernetes-version }} tests/e2e-kubernetes/scripts/run.sh - promote: - if: startsWith(github.ref_name, 'release') + post_test: needs: test runs-on: ubuntu-latest + environment: ${{ inputs.environment }} permissions: id-token: write contents: read steps: - - name: Set up crane - uses: imjasonh/setup-crane@v0.1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@master + uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.TEST_IAM_ROLE }} - aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ vars.IAM_ROLE }} + aws-region: ${{ vars.AWS_REGION }} - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Promote image for release branch + - name: Mark tests as passed env: - REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: "${{ steps.login-ecr.outputs.registry }}/${{ env.IMAGE_NAME }}" run: | - export TMP_IMAGE_NAME=${REGISTRY}/${{ env.TMP_IMAGE_NAME }}:${{ env.COMMIT_ID }} - export NEW_IMAGE_NAME=${REGISTRY}/${{ env.PROMOTED_IMAGE_NAME }}:${{ env.COMMIT_ID }} - crane copy ${TMP_IMAGE_NAME} ${NEW_IMAGE_NAME} + docker buildx imagetools create --tag ${REPOSITORY}:${TAG_PASSED} ${REPOSITORY}:${TAG_UNTESTED} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0909a3ab..f86551e4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,16 +10,16 @@ on: # 1) private ARS source repository # 2) public repository env: - AWS_SRC_REGION : "us-east-1" # this is where our testing repo exists - AWS_DEST_REGION : "us-west-2" # this is where our prod repos exist - COMMIT_ID: ${{ github.sha }} GIT_TAG: ${{ github.ref_name }} + TAG_PASSED: "test_passed_${{ github.sha }}" IMAGE_NAME: "s3-csi-driver" - PUBLIC_REGISTRY: "public.ecr.aws/mountpoint-s3-csi-driver/aws-mountpoint-s3-csi-driver" + PUBLIC_REGISTRY: ${{ vars.PUBLIC_REGISTRY }} + ARS_REGISTRY: ${{ vars.ARS_REGISTRY }} jobs: build: # this is to prevent the job to run at forked projects - if: github.repository == 'awslabs/mountpoint-s3-csi-driver' + if: ${{ ! github.repository.fork }} + environment: release runs-on: ubuntu-latest permissions: id-token: write @@ -32,40 +32,52 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Set up crane uses: imjasonh/setup-crane@v0.1 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Configure AWS Credentials from Test acccount - uses: aws-actions/configure-aws-credentials@master + + - name: Configure AWS Credentials from CI Trusted account + uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.TEST_IAM_ROLE }} - aws-region: ${{ env.AWS_SRC_REGION }} - - name: Login to Amazon ECR test - id: login-ecr-test + role-to-assume: ${{ vars.CI_TRUSTED_IAM_ROLE }} + aws-region: ${{ vars.AWS_CI_TRUSTED_REGION }} + - name: Login to Amazon ECR (trusted) + id: login-ecr-trusted uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: true + - name: Configure AWS Credentials from Prod account - uses: aws-actions/configure-aws-credentials@master + uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.PROD_IAM_IMAGE_ROLE }} - aws-region: ${{ env.AWS_DEST_REGION }} + role-to-assume: ${{ vars.PROD_IAM_IMAGE_ROLE }} + aws-region: ${{ vars.AWS_PROD_ECR_REGION }} - name: Login to Amazon ECR prod id: login-ecr-prod uses: aws-actions/amazon-ecr-login@v1 - - name: Configure AWS Credentials from Prod account (for public repo) - uses: aws-actions/configure-aws-credentials@master with: - role-to-assume: ${{ secrets.PROD_IAM_IMAGE_ROLE }} + mask-password: true + + - name: Configure AWS Credentials from Prod account (for ECR public) + if: ${{ env.PUBLIC_REGISTRY != '' }} + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.PROD_IAM_IMAGE_ROLE }} aws-region: us-east-1 - name: Login to Amazon ECR Public + if: ${{ env.PUBLIC_REGISTRY != '' }} id: login-ecr-public uses: aws-actions/amazon-ecr-login@v1 with: registry-type: public + mask-password: true + - name: Promote image env: - ARS_REGISTRY: ${{ secrets.ARS_REGISTRY }} + SOURCE_REGISTRY: ${{ steps.login-ecr-trusted.outputs.registry }} run: | - crane copy ${{ steps.login-ecr-test.outputs.registry }}/${{ env.IMAGE_NAME }}:${{ env.COMMIT_ID }} ${ARS_REGISTRY}:${{ env.GIT_TAG }} - crane copy ${{ steps.login-ecr-test.outputs.registry }}/${{ env.IMAGE_NAME }}:${{ env.COMMIT_ID }} ${{ env.PUBLIC_REGISTRY }}:${{ env.GIT_TAG }} + crane copy ${SOURCE_REGISTRY}/${IMAGE_NAME}:${TAG_PASSED} ${ARS_REGISTRY}:${GIT_TAG} + if [ -n "${PUBLIC_REGISTRY}" ]; then + crane copy ${SOURCE_REGISTRY}/${IMAGE_NAME}:${TAG_PASSED} ${PUBLIC_REGISTRY}:${GIT_TAG} + fi + - name: Create Release id: create-release uses: actions/create-release@v1 diff --git a/Makefile b/Makefile index 1deab877..e030dd70 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,11 @@ all: all-image-docker all-push: create-manifest-and-images docker manifest push --purge $(IMAGE):$(TAG) +# Builds images only if not present with the tag +.PHONY: all-push-skip-if-present +all-push-skip-if-present: + docker manifest inspect $(IMAGE):$(TAG) > /dev/null || $(MAKE) all-push + .PHONY: build_image build_image: DOCKER_BUILDKIT=1 docker buildx build -f ${DOCKERFILE} -t=${IMAGE}:${TAG} --platform=${PLATFORM} .