Update SELinux configuration and environment variables in e2e tests w… #734
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| environment: | ||
| required: true | ||
| type: string | ||
| ref: | ||
| required: true | ||
| type: string | ||
| concurrency: | ||
| group: e2e-cluster-${{ inputs.environment }} | ||
| env: | ||
| IMAGE_NAME: "s3-csi-driver" | ||
| BENCHMARK_ARTIFACTS_FOLDER: ".github/artifacts" | ||
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | ||
| KOPS_STATE_FILE: "s3://${{ vars.KOPS_STATE_FILE }}" | ||
| BENCHMARK_BUCKET: "s3://${{ vars.BENCHMARK_BUCKET }}" | ||
| TAG_UNTESTED: "untested_${{ inputs.ref }}" | ||
| TAG_PASSED: "test_passed_${{ inputs.ref }}" | ||
| jobs: | ||
| build: | ||
| runs-on: ubuntu-latest | ||
| environment: ${{ inputs.environment }} | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| ref: ${{ inputs.ref }} | ||
| persist-credentials: false | ||
| - name: Set up Go | ||
| uses: actions/setup-go@v4 | ||
| with: | ||
| go-version-file: "go.mod" | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v2 | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@master | ||
| with: | ||
| role-to-assume: ${{ vars.IAM_ROLE }} | ||
| aws-region: ${{ vars.AWS_REGION }} | ||
| - name: Login to Amazon ECR | ||
| id: login-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| - name: Build, tag, and push docker image to Amazon ECR Private Repository | ||
| env: | ||
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | ||
| PLATFORM: "linux/amd64,linux/arm64" | ||
| TAG: "${{ env.TAG_UNTESTED }}" | ||
| run: | | ||
| make -j `nproc` all-push-skip-if-present | ||
| test: | ||
| needs: build | ||
| strategy: | ||
| # Failing fast causes some resources created during the test to leak, | ||
| # so we disable it to ensure all resources created during test are properly cleaned up. | ||
| fail-fast: false | ||
| matrix: | ||
| cluster-type: ["eksctl", "kops"] | ||
| arch: ["x86", "arm"] | ||
| family: | ||
| ["AmazonLinux2", "AmazonLinux2023", "Bottlerocket", "Ubuntu2004"] | ||
| kubernetes-version: ["1.28.13", "1.29.8", "1.30.4", "1.31.0"] | ||
| include: | ||
| # Ubuntu2004 supported for EKS <= 1.29 and Ubuntu2204 supported for EKS >= 1.29. | ||
| # See https://eksctl.io/usage/custom-ami-support/?h=ubuntu#setting-the-node-ami-family. | ||
| - cluster-type: "eksctl" | ||
| arch: "x86" | ||
| family: "Ubuntu2004" | ||
| kubernetes-version: "1.28.13" | ||
| - cluster-type: "eksctl" | ||
| arch: "arm" | ||
| family: "Ubuntu2004" | ||
| kubernetes-version: "1.29.8" | ||
| - cluster-type: "eksctl" | ||
| arch: "x86" | ||
| family: "Ubuntu2204" | ||
| kubernetes-version: "1.30.4" | ||
| - cluster-type: "eksctl" | ||
| arch: "arm" | ||
| family: "Ubuntu2204" | ||
| kubernetes-version: "1.31.0" | ||
| # Since we only enable enforcing mode for SELinux in AL2023, it's easier to list it in "include" | ||
| # field rather than trying to exclude all other variants. | ||
| - cluster-type: "kops" | ||
| arch: "arm" | ||
| family: "AmazonLinux2023" | ||
| selinux-mode: "enforcing" | ||
| exclude: | ||
| - cluster-type: "kops" | ||
| family: "Bottlerocket" | ||
| - cluster-type: "eksctl" | ||
| arch: "arm" | ||
| family: "Bottlerocket" | ||
| - cluster-type: "eksctl" | ||
| family: "Ubuntu2004" | ||
| # We're mainly using AL2023 to test our CSI driver with enforced SELinux, | ||
| # AL2023 supports switching to enforcing mode via cloud-init, but, | ||
| # passing user-data (the way we configure cloud-init) is not supported on eksctl for AL2023. | ||
| # So, we're only running SELinux tests with kops. | ||
| - cluster-type: "eksctl" | ||
| family: "AmazonLinux2023" | ||
| # Our tests are failing on clusters created with kops 1.29+. | ||
| # Until we fix that issue, we use kops 1.28 which only supports k8s versions up to 1.28. | ||
| # So, we only run our tests in k8s versions 1.29 and 1.30 on eksctl. | ||
| - cluster-type: "kops" | ||
| kubernetes-version: "1.29.8" | ||
| - cluster-type: "kops" | ||
| kubernetes-version: "1.30.4" | ||
| - cluster-type: "kops" | ||
| kubernetes-version: "1.31.0" | ||
| runs-on: ubuntu-latest | ||
| environment: ${{ inputs.environment }} | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| env: | ||
| AWS_REGION: "${{ vars.AWS_REGION }}" | ||
| CLUSTER_TYPE: "${{ matrix.cluster-type }}" | ||
| ARCH: "${{ matrix.arch }}" | ||
| AMI_FAMILY: "${{ matrix.family }}" | ||
| TAG: "untested_${{ inputs.ref }}" | ||
| # envtest doesn't support all versions, here K8S_VERSION is a full version like 1.28.13, | ||
| # and in order to get latest supported version by envtest we convert it to 1.28. | ||
| K8S_VERSION: "${{ matrix.kubernetes-version }}" | ||
| ENVTEST_K8S_VERSION: "${K8S_VERSION%.*}" | ||
| SELINUX_MODE=${{ matrix.selinux-mode }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| ref: ${{ inputs.ref }} | ||
| persist-credentials: false | ||
| - name: Set up Go | ||
| uses: actions/setup-go@v4 | ||
| with: | ||
| go-version-file: "go.mod" | ||
| - uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: "3.10" | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: ${{ vars.IAM_ROLE }} | ||
| aws-region: ${{ vars.AWS_REGION }} | ||
| role-duration-seconds: 7200 | ||
| - name: Install tools | ||
| env: | ||
| ACTION: "install_tools" | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Run Controller Tests | ||
| run: | | ||
| make e2e-controller | ||
| - name: Create cluster | ||
| env: | ||
| ACTION: "create_cluster" | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Update kubeconfig | ||
| env: | ||
| ACTION: "update_kubeconfig" | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Install the driver | ||
| env: | ||
| ACTION: "install_driver" | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Run E2E Tests | ||
| env: | ||
| ACTION: "run_tests" | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Run Performance Tests | ||
| if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' | ||
| env: | ||
| ACTION: "run_perf" | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Download previous benchmark results | ||
| if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' | ||
| run: | | ||
| mkdir -p ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} | ||
| aws s3 cp --region ${{ vars.BENCHMARK_RESULTS_REGION }} ${{ vars.BENCHMARK_BUCKET }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json || true | ||
| - name: Update benchmark result file | ||
| if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' | ||
| uses: benchmark-action/github-action-benchmark@v1 | ||
| with: | ||
| tool: "customBiggerIsBetter" | ||
| output-file-path: tests/e2e-kubernetes/csi-test-artifacts/output.json | ||
| alert-threshold: "200%" | ||
| fail-on-alert: true | ||
| external-data-json-path: ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json | ||
| max-items-in-chart: 20 | ||
| - name: Store benchmark result | ||
| if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/format_benchmark_data.py ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/quicksight-data.json | ||
| aws s3 cp ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} ${{ env.BENCHMARK_BUCKET }} --recursive | ||
| - name: Post e2e cleanup | ||
| if: always() | ||
| env: | ||
| ACTION: "e2e_cleanup" | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Uninstall the driver | ||
| if: always() | ||
| env: | ||
| ACTION: "uninstall_driver" | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Delete cluster | ||
| if: always() | ||
| env: | ||
| ACTION: "delete_cluster" | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| post_test: | ||
| needs: test | ||
| runs-on: ubuntu-latest | ||
| environment: ${{ inputs.environment }} | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v1 | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: ${{ vars.IAM_ROLE }} | ||
| aws-region: ${{ vars.AWS_REGION }} | ||
| - name: Login to Amazon ECR | ||
| id: login-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| - name: Mark tests as passed | ||
| env: | ||
| REPOSITORY: "${{ steps.login-ecr.outputs.registry }}/${{ env.IMAGE_NAME }}" | ||
| run: | | ||
| docker buildx imagetools create --tag ${REPOSITORY}:${TAG_PASSED} ${REPOSITORY}:${TAG_UNTESTED} | ||
