From 363a8e6b07b15968c98eb58c1eb6644bd7a5ae46 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 13:44:07 -0800 Subject: [PATCH] build: Parallelize the CI image builds (continued) (#26698) --- .github/workflows/docker-release.yml | 18 +- .github/workflows/docker.yml | 41 ++++- scripts/docker_build_push.sh | 162 ++++++------------ tests/unit_tests/fixtures/bash_mock.py | 6 +- .../scripts/docker_build_push_test.py | 34 ++-- 5 files changed, 126 insertions(+), 135 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 7cfba73299b43..3e205fdeffaaf 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -22,21 +22,29 @@ jobs: if: needs.config.outputs.has-secrets name: docker-release runs-on: ubuntu-latest + strategy: + matrix: + target: ["dev", "lean", "lean310", "websocket", "dockerize"] + platform: ["linux/amd64", "linux/arm64"] + fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false submodules: recursive ref: ${{ github.ref }} + - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - shell: bash + uses: docker/setup-buildx-action@v3 + + - name: Build Docker Image env: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}" - ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME" + ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME" ${{ matrix.target }} ${{ matrix.platform }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6160d3cc1f595..b0d12be103c11 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,27 +24,52 @@ jobs: echo "has-secrets=0" >> "$GITHUB_OUTPUT" echo "no secrets!" fi - docker-build: needs: config if: needs.config.outputs.has-secrets name: docker-build runs-on: ubuntu-latest + strategy: + matrix: + target: ["dev", "lean", "lean310", "websocket", "dockerize"] + platform: ["linux/amd64", "linux/arm64"] + fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false + - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - shell: bash + uses: docker/setup-buildx-action@v3 + + - name: Build Docker Image + shell: bash env: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | - ./scripts/docker_build_push.sh + ./scripts/docker_build_push.sh "" ${{ matrix.target }} ${{ matrix.platform }} + + ephemeral-docker-build: + needs: config + if: needs.config.outputs.has-secrets + name: docker-build + runs-on: ubuntu-latest + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Build ephemeral env image if: github.event_name == 'pull_request' @@ -54,7 +79,7 @@ jobs: echo ${{ github.event.pull_request.number }} > ./build/PR-NUM docker buildx build --target ci \ --load \ - --cache-from=type=local,src=/tmp/superset \ + --cache-from=type=registry,ref=apache/superset:lean \ -t ${{ github.sha }} \ -t "pr-${{ github.event.pull_request.number }}" \ --platform linux/amd64 \ @@ -64,7 +89,7 @@ jobs: - name: Upload build artifacts if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: build path: build/ diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh index 8ae82faaeb5ee..10737447eaa6b 100755 --- a/scripts/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -18,9 +18,15 @@ set -eo pipefail GITHUB_RELEASE_TAG_NAME="$1" +TARGET="$2" +BUILD_PLATFORM="$3" # should be either 'linux/amd64' or 'linux/arm64' +# Common variables SHA=$(git rev-parse HEAD) REPO_NAME="apache/superset" +DOCKER_ARGS="--load" # default args, change as needed +DOCKER_CONTEXT="." + if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40) @@ -54,6 +60,45 @@ if [[ "${TEST_ENV}" == "true" ]]; then exit 0 fi +# for the dev image, it's ok to tag master as latest-dev +# for production, we only want to tag the latest official release as latest +if [ "${LATEST_TAG}" = "master" ]; then + DEV_TAG="${REPO_NAME}:latest-dev" +else + DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev" +fi + +BUILD_ARG="3.9-slim-bookworm" + +case "${TARGET}" in + "dev") + DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}" + BUILD_TARGET="dev" + ;; + "lean") + DOCKER_TAGS="-t ${REPO_NAME}:${SHA} -t ${REPO_NAME}:${REFSPEC} -t ${REPO_NAME}:${LATEST_TAG}" + BUILD_TARGET="lean" + ;; + "lean310") + DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-py310 -t ${REPO_NAME}:${REFSPEC}-py310 -t ${REPO_NAME}:${LATEST_TAG}-py310" + BUILD_TARGET="lean" + BUILD_ARG="3.10-slim-bookworm" + ;; + "websocket") + DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-websocket -t ${REPO_NAME}:${REFSPEC}-websocket -t ${REPO_NAME}:${LATEST_TAG}-websocket" + BUILD_TARGET="" + DOCKER_CONTEXT="superset-websocket" + ;; + "dockerize") + DOCKER_TAGS="-t ${REPO_NAME}:dockerize" + BUILD_TARGET="" + DOCKER_CONTEXT="-f dockerize.Dockerfile ." + ;; + *) + echo "Invalid TARGET: ${TARGET}" + exit 1 + ;; +esac cat<