From b0ba28c6bc513a6d087680d32142290f7c93c1d3 Mon Sep 17 00:00:00 2001 From: Danny Mccormick Date: Wed, 26 Feb 2025 10:28:12 -0500 Subject: [PATCH 1/2] Enable splitting up docker publish workflow --- .../republish_released_docker_containers.yml | 12 +++++++++-- build.gradle.kts | 20 ++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/republish_released_docker_containers.yml b/.github/workflows/republish_released_docker_containers.yml index f87f4af220cc..dc9228e3371a 100644 --- a/.github/workflows/republish_released_docker_containers.yml +++ b/.github/workflows/republish_released_docker_containers.yml @@ -39,6 +39,14 @@ jobs: build: runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + images_to_publish: [ + {"gradle_task": ":pushAllRunnersDockerImages", "skip_flags": ""}, + {"gradle_task": ":pushAllSdkDockerImages", "skip_flags": ""}, + {"gradle_task": ":pushAllDockerImages", "skip_flags": "-Pskip-runner-images -Pskip-sdk-images"} + ] steps: - name: Checkout uses: actions/checkout@v4 @@ -73,10 +81,10 @@ jobs: gcloud auth configure-docker ${{ env.docker_registry }} - name: Push docker images run: | - ./gradlew :pushAllDockerImages \ + ./gradlew ${{ matrix.images_to_publish.gradle_task }} \ -PisRelease \ -Pdocker-pull-licenses \ - -Pprune-images \ + -Pprune-images ${{ matrix.images_to_publish.skip_flags }} \ -Pdocker-repository-root=gcr.io/apache-beam-testing/updated_released_container_images \ -Pdocker-tag-list=${{ env.release }},${{ github.sha }},$(date +'%Y-%m-%d') \ --no-daemon \ diff --git a/build.gradle.kts b/build.gradle.kts index d230f32ea9bb..209cb2b95931 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -598,7 +598,9 @@ tasks.register("pushAllRunnersDockerImages") { tasks.register("pushAllSdkDockerImages") { // Enforce ordering to allow the prune step to happen between runs. // This will ensure we don't use up too much space (especially in CI environments) - mustRunAfter(":pushAllRunnersDockerImages") + if (!project.hasProperty("skip-runner-images")) { + mustRunAfter(":pushAllRunnersDockerImages") + } dependsOn(":sdks:java:container:pushAll") dependsOn(":sdks:python:container:pushAll") @@ -618,7 +620,9 @@ tasks.register("pushAllSdkDockerImages") { tasks.register("pushAllXlangDockerImages") { // Enforce ordering to allow the prune step to happen between runs. // This will ensure we don't use up too much space (especially in CI environments) - mustRunAfter(":pushAllSdkDockerImages") + if (!project.hasProperty("skip-sdk-images")) { + mustRunAfter(":pushAllSdkDockerImages") + } dependsOn(":sdks:java:expansion-service:container:docker") dependsOn(":sdks:java:transform-service:controller-container:docker") @@ -635,9 +639,15 @@ tasks.register("pushAllXlangDockerImages") { } tasks.register("pushAllDockerImages") { - dependsOn(":pushAllRunnersDockerImages") - dependsOn(":pushAllSdkDockerImages") - dependsOn(":pushAllXlangDockerImages") + if (!project.hasProperty("skip-runner-images")) { + dependsOn(":pushAllRunnersDockerImages") + } + if (!project.hasProperty("skip-sdk-images")) { + dependsOn(":pushAllSdkDockerImages") + } + if (!project.hasProperty("skip-xlang-images")) { + dependsOn(":pushAllXlangDockerImages") + } } // Use this task to validate the environment set up for Go, Python and Java From e06c7533b4f8705ca2db93a2a0eb7ce27db83d7b Mon Sep 17 00:00:00 2001 From: Danny Mccormick Date: Wed, 26 Feb 2025 13:50:25 -0500 Subject: [PATCH 2/2] Just make build.gradle.kts changes for now --- .../republish_released_docker_containers.yml | 12 ++---------- build.gradle.kts | 4 +++- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/republish_released_docker_containers.yml b/.github/workflows/republish_released_docker_containers.yml index dc9228e3371a..f87f4af220cc 100644 --- a/.github/workflows/republish_released_docker_containers.yml +++ b/.github/workflows/republish_released_docker_containers.yml @@ -39,14 +39,6 @@ jobs: build: runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - images_to_publish: [ - {"gradle_task": ":pushAllRunnersDockerImages", "skip_flags": ""}, - {"gradle_task": ":pushAllSdkDockerImages", "skip_flags": ""}, - {"gradle_task": ":pushAllDockerImages", "skip_flags": "-Pskip-runner-images -Pskip-sdk-images"} - ] steps: - name: Checkout uses: actions/checkout@v4 @@ -81,10 +73,10 @@ jobs: gcloud auth configure-docker ${{ env.docker_registry }} - name: Push docker images run: | - ./gradlew ${{ matrix.images_to_publish.gradle_task }} \ + ./gradlew :pushAllDockerImages \ -PisRelease \ -Pdocker-pull-licenses \ - -Pprune-images ${{ matrix.images_to_publish.skip_flags }} \ + -Pprune-images \ -Pdocker-repository-root=gcr.io/apache-beam-testing/updated_released_container_images \ -Pdocker-tag-list=${{ env.release }},${{ github.sha }},$(date +'%Y-%m-%d') \ --no-daemon \ diff --git a/build.gradle.kts b/build.gradle.kts index 209cb2b95931..8dcdc14f04e7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -603,7 +603,9 @@ tasks.register("pushAllSdkDockerImages") { } dependsOn(":sdks:java:container:pushAll") - dependsOn(":sdks:python:container:pushAll") + if (!project.hasProperty("skip-python-images")) { + dependsOn(":sdks:python:container:pushAll") + } dependsOn(":sdks:go:container:pushAll") dependsOn(":sdks:typescript:container:pushAll")