From 8b9c537efed159436e7bba13ee5751f82060963e Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 26 Apr 2023 09:46:58 -0400 Subject: [PATCH 1/7] Add GitHub action for publishing java artifacts --- .github/workflows/build_release_candidate.yml | 54 +++++++++++++++++++ .../main/scripts/build_release_candidate.sh | 39 ++------------ .../content/en/contribute/release-guide.md | 36 +++++++++---- 3 files changed, 83 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/build_release_candidate.yml diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml new file mode 100644 index 000000000000..52677a9e88e9 --- /dev/null +++ b/.github/workflows/build_release_candidate.yml @@ -0,0 +1,54 @@ +name: build_release_candidate + +on: + workflow_dispatch: + inputs: + RELEASE: + description: Beam version of current release (e.g. 2.XX.0) + required: true + default: '2.XX.0' + RC: + description: Integer RC version for the release (e.g. 3 for RC3) + required: true + PUBLISH_JAVA_ARTIFACTS: + description: Whether to publish java artifacts to https://repository.apache.org/#stagingRepositories (yes/no) + required: true + default: 'no' + +jobs: + publish_java_artifacts: + if: ${{github.event.inputs.PUBLISH_JAVA_ARTIFACTS == 'yes'}} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" + repository: apache/beam + - name: Import GPG key + id: import_gpg + uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.PASSPHRASE }} + - name: Auth for nexus + run: | + mkdir -p ${HOME}/.m2 + echo " + + + apache.releases.https + ${{ secrets.APACHE_TOKEN_NAME }} + ${{ secrets.APACHE_TOKEN_PASSWORD }} + + + apache.snapshots.https + ${{ secrets.APACHE_TOKEN_NAME }} + ${{ secrets.APACHE_TOKEN_PASSWORD }} + + + " > ${HOME}/.m2/settings.xml + - name: Build Java Artifacts with Gradle + run: git config credential.helper store + - name: Stage Java Artifacts into Maven + run: ./gradlew publish -Psigning.gnupg.keyName=FC383FCDE7D7E86699954EF2509872C8031C4DFB -PisRelease --no-daemon --no-parallel \ No newline at end of file diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh index d5430c7b4bbc..861a9f6dba6f 100755 --- a/release/src/main/scripts/build_release_candidate.sh +++ b/release/src/main/scripts/build_release_candidate.sh @@ -17,11 +17,10 @@ # # This script will create a Release Candidate, includes: -# 1. Build and stage java artifacts -# 2. Stage source release on dist.apache.org -# 3. Stage python source distribution and wheels on dist.apache.org -# 4. Stage SDK docker images -# 5. Create a PR to update beam-site +# 1. Stage source release on dist.apache.org +# 2. Stage python source distribution and wheels on dist.apache.org +# 3. Stage SDK docker images +# 4. Create a PR to update beam-site set -e @@ -176,36 +175,6 @@ if [[ $confirmation != "y" ]]; then exit fi -echo "[Current Step]: Build and stage java artifacts" -echo "Do you want to proceed? [y|N]" -read confirmation -if [[ $confirmation = "y" ]]; then - echo "============Building and Staging Java Artifacts=============" - echo "--------Cloning Beam Repo and Checkout Release Tag-------" - cd ~ - wipe_local_clone_dir - mkdir -p ${LOCAL_CLONE_DIR} - cd ${LOCAL_CLONE_DIR} - git clone --depth 1 --branch "${RC_TAG}" ${GIT_REPO_URL} "${BEAM_ROOT_DIR}" - cd ${BEAM_ROOT_DIR} - - echo "-------------Building Java Artifacts with Gradle-------------" - git config credential.helper store - - echo "-------------Staging Java Artifacts into Maven---------------" - # Cache the key/passphrase in gpg-agent by signing an arbitrary file. - gpg --local-user ${SIGNING_KEY} --output /dev/null --sign ~/.bashrc - # Too many workers can overload (?) gpg-agent, causing gpg to prompt for a - # passphrase, and gradle doesn't play nice with pinentry. - # https://github.com/gradle/gradle/issues/11706 - # --max-workers=6 works, but parallelism also seems to cause - # multiple Nexus repos to be created, so parallelism is disabled. - # https://issues.apache.org/jira/browse/BEAM-11813 - ./gradlew publish -Psigning.gnupg.keyName=${SIGNING_KEY} -PisRelease --no-daemon --no-parallel - echo "You need to close the staging repository manually on Apache Nexus. See the release guide for instructions." - wipe_local_clone_dir -fi - echo "[Current Step]: Stage source release on dist.apache.org" echo "Do you want to proceed? [y|N]" read confirmation diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md index 59e8b90a0181..2f9d9460f55d 100644 --- a/website/www/site/content/en/contribute/release-guide.md +++ b/website/www/site/content/en/contribute/release-guide.md @@ -548,6 +548,31 @@ is perfectly safe since the script does not depend on the current working tree. See the source of the script for more details, or to run commands manually in case of a problem. +### Run build_release_candidate GitHub Action to create a release candidate + +Note: This step is partially automated (in progress), so part of the rc creation is done by GitHub Actions and the rest is done by a script. +You don't need to wait for the action to complete to start running the script. + +* **Action** [build_release_candidate](https://github.com/damccorm/beam/actions/workflows/build_release_candidate.yml) + +* **The script will:** + 1. Clone the repo at the selected RC tag. + 1. Run gradle publish to push java artifacts into Maven staging repo. + +#### Tasks you need to do manually + + 1. Publish staging artifacts + 1. Log in to the [Apache Nexus](https://repository.apache.org/#stagingRepositories) website. + 1. Navigate to Build Promotion -> Staging Repositories (in the left sidebar). + 1. Select repository `orgapachebeam-NNNN`. + 1. Click the Close button. + 1. When prompted for a description, enter “Apache Beam, version X, release candidate Y”. + 1. Review all staged artifacts on `https://repository.apache.org/content/repositories/orgapachebeam-NNNN/`. + They should contain all relevant parts for each module, including `pom.xml`, jar, test jar, javadoc, etc. + Artifact names should follow [the existing format](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) in which artifact name mirrors directory structure, e.g., `beam-sdks-java-io-kafka`. + Carefully review any new artifacts. + Some additional validation should be done during the rc validation step. + ### Run build_release_candidate.sh to create a release candidate * **Script:** [build_release_candidate.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/build_release_candidate.sh) @@ -558,7 +583,6 @@ See the source of the script for more details, or to run commands manually in ca * **The script will:** 1. Clone the repo at the selected RC tag. - 1. Run gradle publish to push java artifacts into Maven staging repo. 1. Stage source release into dist.apache.org dev [repo](https://dist.apache.org/repos/dist/dev/beam/). 1. Stage, sign and hash python source distribution and wheels into dist.apache.org dev repo python dir 1. Stage SDK docker images to [docker hub Apache organization](https://hub.docker.com/search?q=apache%2Fbeam&type=image). @@ -588,16 +612,6 @@ help with this step. Please email `dev@` and ask a member of the `beammaintainer docker run --rm -it --entrypoint=/bin/bash apache/beam_java${ver}_sdk:${RELEASE_VERSION}_rc{RC_NUM} ls -al /opt/apache/beam/third_party_licenses/ | wc -l ``` - 1. Publish staging artifacts - 1. Log in to the [Apache Nexus](https://repository.apache.org/#stagingRepositories) website. - 1. Navigate to Build Promotion -> Staging Repositories (in the left sidebar). - 1. Select repository `orgapachebeam-NNNN`. - 1. Click the Close button. - 1. When prompted for a description, enter “Apache Beam, version X, release candidate Y”. - 1. Review all staged artifacts on `https://repository.apache.org/content/repositories/orgapachebeam-NNNN/`. - They should contain all relevant parts for each module, including `pom.xml`, jar, test jar, javadoc, etc. - Artifact names should follow [the existing format](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) in which artifact name mirrors directory structure, e.g., `beam-sdks-java-io-kafka`. - Carefully review any new artifacts. ### Upload release candidate to PyPi From 47c6144770e43fc598f2c8c8d6f8deebcd75e13a Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 26 Apr 2023 09:53:37 -0400 Subject: [PATCH 2/7] Info on break glass mode --- .github/workflows/build_release_candidate.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index 52677a9e88e9..08995fca9357 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -1,5 +1,8 @@ name: build_release_candidate +# Workflow added after https://github.com/apache/beam/commit/4183e747becebd18becee5fff547af365910fc9c +# If help is needed debugging issues, you can view the release guide at that commit for guidance on how to do this manually. +# (https://github.com/apache/beam/blob/4183e747becebd18becee5fff547af365910fc9c/website/www/site/content/en/contribute/release-guide.md) on: workflow_dispatch: inputs: From 9ffbcd6444358547949e7d5051cccb2f165ad1fd Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 26 Apr 2023 09:55:06 -0400 Subject: [PATCH 3/7] More explicit instruction --- website/www/site/content/en/contribute/release-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md index 2f9d9460f55d..77a1af513852 100644 --- a/website/www/site/content/en/contribute/release-guide.md +++ b/website/www/site/content/en/contribute/release-guide.md @@ -553,7 +553,7 @@ See the source of the script for more details, or to run commands manually in ca Note: This step is partially automated (in progress), so part of the rc creation is done by GitHub Actions and the rest is done by a script. You don't need to wait for the action to complete to start running the script. -* **Action** [build_release_candidate](https://github.com/damccorm/beam/actions/workflows/build_release_candidate.yml) +* **Action** [build_release_candidate](https://github.com/damccorm/beam/actions/workflows/build_release_candidate.yml) (click `run workflow`) * **The script will:** 1. Clone the repo at the selected RC tag. From e5bdf8112af240c66d50dbe0fc94f031dc069ca2 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 12 May 2023 11:58:33 -0400 Subject: [PATCH 4/7] Updates from https://issues.apache.org/jira/browse/INFRA-24520 --- .github/workflows/build_release_candidate.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index 08995fca9357..f6f8398e5d03 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -33,7 +33,6 @@ jobs: uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} - passphrase: ${{ secrets.PASSPHRASE }} - name: Auth for nexus run: | mkdir -p ${HOME}/.m2 @@ -41,17 +40,17 @@ jobs: apache.releases.https - ${{ secrets.APACHE_TOKEN_NAME }} - ${{ secrets.APACHE_TOKEN_PASSWORD }} + ${{ secrets.NEXUS_USER }} + ${{ secrets.NEXUS_PW }} apache.snapshots.https - ${{ secrets.APACHE_TOKEN_NAME }} - ${{ secrets.APACHE_TOKEN_PASSWORD }} + ${{ secrets.NEXUS_USER }} + ${{ secrets.NEXUS_PW }} " > ${HOME}/.m2/settings.xml - name: Build Java Artifacts with Gradle run: git config credential.helper store - name: Stage Java Artifacts into Maven - run: ./gradlew publish -Psigning.gnupg.keyName=FC383FCDE7D7E86699954EF2509872C8031C4DFB -PisRelease --no-daemon --no-parallel \ No newline at end of file + run: ./gradlew publish -Psigning.gnupg.keyName=FC383FCDE7D7E86699954EF2509872C8031C4DFB -PisRelease --no-daemon --no-parallel From 9e65a0e170e4d36e447e426016caf0979d47a360 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 12 May 2023 12:01:07 -0400 Subject: [PATCH 5/7] Force java 8 --- .github/workflows/build_release_candidate.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index f6f8398e5d03..76cf59cdc5e9 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -28,6 +28,11 @@ jobs: with: ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" repository: apache/beam + - name: Install Java 8 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '8' - name: Import GPG key id: import_gpg uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 From 14d6631026759ab35e9a0624d7bcb48aaa2cd69a Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 12 May 2023 12:13:01 -0400 Subject: [PATCH 6/7] Update build_release_candidate.yml --- .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index 76cf59cdc5e9..b856d6671532 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -55,7 +55,7 @@ jobs: " > ${HOME}/.m2/settings.xml - - name: Build Java Artifacts with Gradle + - name: Configure git run: git config credential.helper store - name: Stage Java Artifacts into Maven run: ./gradlew publish -Psigning.gnupg.keyName=FC383FCDE7D7E86699954EF2509872C8031C4DFB -PisRelease --no-daemon --no-parallel From 02efdcab4d58f5576f796a2ed116e47d4439751c Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 12 May 2023 14:52:14 -0400 Subject: [PATCH 7/7] Correct fingerprint --- .github/workflows/build_release_candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index b856d6671532..fbac8b97d50a 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -58,4 +58,4 @@ jobs: - name: Configure git run: git config credential.helper store - name: Stage Java Artifacts into Maven - run: ./gradlew publish -Psigning.gnupg.keyName=FC383FCDE7D7E86699954EF2509872C8031C4DFB -PisRelease --no-daemon --no-parallel + run: ./gradlew publish -Psigning.gnupg.keyName=${{steps.import_gpg.outputs.fingerprint}} -PisRelease --no-daemon --no-parallel