From bb58b3751ef9f5f5f069701482cd2fc4e74cd212 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 9 Jan 2025 16:49:40 +0900 Subject: [PATCH 01/28] GH-20: Add release scripts Fixes GH-20. This is based on https://github.com/apache/arrow-go/pull/122 . Workflow: Cut a RC: 1. Bump version by `dev/release/bump_version.sh 19.0.0` 2. Run `dev/release/release_rc.sh` 1. `dev/release/release_rc.sh` pushes `v${version}-rc${rc}` tag 2. `.github/workflows/rc.yml` creates `apache-arrow-java-${version}.tar.gz{,.sha256,.sha512}` 3. `.github/workflows/rc.yml` uploads `apache-arrow-java-${version}.tar.gz{,.sha256,.sha512}` to GitHub Releases 4. `dev/release/release_rc.sh` downloads `apache-arrow-java-${version}.tar.gz` from GitHub Releases 5. `dev/release/release_rc.sh` signs `apache-arrow-java-${version}.tar.gz` as `apache-arrow-go-${version}.tar.gz.asc` 6. `dev/release/release_rc.sh` uploads `apache-arrow-java-${version}.tar.gz.asc` to GitHub Releases 3. Start a vote (GitHub Actions instead of https://dist.apache.org/repos/dist/dev/arrow/ is used like ADBC.) Verify a RC: 1. Run `dev/release/verify_rc.sh` Release an approved RC: 1. Run `dev/release/release.sh` 1. `dev/release/release.sh` pushes `v${version}` tag that refers that same commit ID as `v${version}-rc${rc}` 2. `dev/release/release.sh` downloads `apache-arrow-java-${version}.tar.gz{,.asc,.sha256,.sha512}` from GitHub Actions 3. `dev/release/release.sh` uploads `apache-arrow-java-${version}.tar.gz{,.asc,.sha256,.sha512}` to https://dist.apache.org/repos/dist/release/arrow 4. `dev/release/release.sh` removes old releases from https://dist.apache.org/repos/dist/release/arrow 2. Add this release to ASF's report database: https://reporter.apache.org/addrelease.html?arrow Follow-up tasks: * Add support for building binary packages (.jar) * Add support for releasing binary packages (.jar) * Add support for running JNI test in the verification script * Add support for running integration test in the verification script * Add support for verifying binary packages (.jar) in the verification script * Add support for releasing a release note --- .github/workflows/rc.yml | 141 ++++++++++++++++++++++++++++++ .github/workflows/release.yml | 54 ++++++++++++ dev/release/README.md | 122 ++++++++++++++++++++++++++ dev/release/bump_version.sh | 51 +++++++++++ dev/release/release.sh | 83 ++++++++++++++++++ dev/release/release_rc.sh | 152 ++++++++++++++++++++++++++++++++ dev/release/verify_rc.sh | 158 ++++++++++++++++++++++++++++++++++ 7 files changed, 761 insertions(+) create mode 100644 .github/workflows/rc.yml create mode 100644 .github/workflows/release.yml create mode 100644 dev/release/README.md create mode 100755 dev/release/bump_version.sh create mode 100755 dev/release/release.sh create mode 100755 dev/release/release_rc.sh create mode 100755 dev/release/verify_rc.sh diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml new file mode 100644 index 000000000..9d1a97b9f --- /dev/null +++ b/.github/workflows/rc.yml @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: RC +on: + push: + branches: + - '**' + - '!dependabot/**' + tags: + - '*-rc*' + pull_request: +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +permissions: + contents: read +jobs: + archive: + name: Archive + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - name: Prepare for tag + if: github.ref_type == 'tag' + run: | + version=${GITHUB_REF_NAME%-rc*} + version=${version#v} + rc=${GITHUB_REF_NAME#*-rc} + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Prepare for branch + if: github.ref_type == 'branch' + run: | + version=$(grep -o '^ .*' "pom.xml" | + sed \ + -e 's,^ ,,' \ + -e 's,$,,') + rc=$(date +%Y%m%d) + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Archive + run: | + id="apache-arrow-java-${VERSION}" + tar_gz="${id}.tar.gz" + echo "TAR_GZ=${tar_gz}" >> ${GITHUB_ENV} + git archive HEAD --prefix "${id}/" --output "${tar_gz}" + sha256sum "${tar_gz}" > "${tar_gz}.sha256" + sha512sum "${tar_gz}" > "${tar_gz}.sha512" + - name: Audit + run: | + dev/release/run_rat.sh "${TAR_GZ}" + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: archive + path: | + apache-arrow-java-* + verify: + name: Verify + needs: + - archive + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: archive + - name: Verify + run: | + tar_gz=$(echo apache-arrow-java-*.tar.gz) + version=${tar_gz#apache-arrow-java-} + version=${version%.tar.gz} + # rc isn't used with VERIFY_DOWNLOAD=0 + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + rc="${GITHUB_REF_NAME#*-rc}" + else + rc=$(date +%Y%m%d) + fi + VERIFY_DEFAULT=0 \ + VERIFY_SOURCE=1 \ + dev/release/verify_rc.sh "${version}" "${rc}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + upload: + name: Upload + if: github.ref_type == 'tag' + needs: + - verify + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: archive + - name: Upload + run: | + # TODO: Add support for release note + version=${GITHUB_REF_NAME%-rc*} + version=${version#v} + rc=${GITHUB_REF_NAME#*-rc} + gh release create ${GITHUB_REF_NAME} \ + --notes "TODO" \ + --prerelease \ + --title "Apache Arrow Java ${version} RC${rc}" \ + --verify-tag \ + apache-arrow-java-*.tar.gz \ + apache-arrow-java-*.tar.gz.sha* + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..e0b866e84 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: RC +on: + push: + tags: + - '*' + - '!*-rc*' + pull_request: +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +permissions: + contents: write +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +jobs: + publish: + name: Publish + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Download RC contents + run: | + set -x + latest_rc_tag=$(gh release list --json tagName --jq '.[].tagName' | \ + grep -F "${GITHUB_REF_NAME}-rc" | \ + head -n1) + gh release download ${latest_rc_tag} --dir dists + - name: Create GitHub Release + run: | + version=${GITHUB_REF_NAME#v} + gh release create ${GITHUB_REF_NAME} \ + --notes "TODO" \ + --title "Apache Arrow Java ${version}" \ + --verify-tag \ + dists/* diff --git a/dev/release/README.md b/dev/release/README.md new file mode 100644 index 000000000..27142406f --- /dev/null +++ b/dev/release/README.md @@ -0,0 +1,122 @@ + + +# Release + +## Overview + + 1. Test the revision to be released + 2. Bump version by `dev/release/bump_version.sh X.Y.Z` + 3. Prepare RC and vote (detailed later) + 4. Publish (detailed later) + 5. Bump version by `dev/release/bump_version.sh X.Y.Z-SNAPSHOT` + +### Prepare RC and vote + +Run `dev/release/release_rc.sh` on a working copy of +`git@github.com:apache/arrow-java` not your fork: + +```console +$ git clone git@github.com:apache/arrow-java.git +$ dev/release/release_rc.sh ${RC} +(Send a vote email to dev@arrow.apache.org. + You can use a draft shown by release_rc.sh for the email.) +``` + +Here is an example to release RC1: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh 1 +``` + +The argument of `release_rc.sh` is the RC number. If RC1 has a +problem, we'll increment the RC number such as RC2, RC3 and so on. + +Requirements to run `release_rc.sh`: + + * You must be an Apache Arrow committer or PMC member + * You must prepare your PGP key for signing + +If you don't have a PGP key, +https://infra.apache.org/release-signing.html#generate may be helpful. + +Your PGP key must be registered to the followings: + + * https://dist.apache.org/repos/dist/dev/arrow/KEYS + * https://dist.apache.org/repos/dist/release/arrow/KEYS + +See the header comment of them how to add a PGP key. + +Apache arrow committers can update them by Subversion client with +their ASF account. e.g.: + +```console +$ svn co https://dist.apache.org/repos/dist/dev/arrow +$ cd arrow +$ editor KEYS +$ svn ci KEYS +``` + +### Publish + +We need to do the followings to publish a new release: + + * Publish to apache.org + +Run `dev/release/release.sh` on a working copy of +`git@github.com:apache/arrow-java` not your fork to publish to +apache.org: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh ${VERSION} ${RC} +``` + +Here is an example to release 19.0.0 RC1: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh 19.0.0 1 +``` + +Add the release to ASF's report database via [Apache Committee Report +Helper](https://reporter.apache.org/addrelease.html?arrow). + +### Verify + +We have a script to verify a RC. + +You must install the following commands to use the script: + + * `curl` + * `gpg` + * `shasum` or `sha256sum`/`sha512sum` + * `tar` + +To verify a RC, run the following command line: + +```console +$ dev/release/verify_rc.sh ${VERSION} ${RC} +``` + +Here is an example to verify the release 19.0.0 RC1: + +```console +$ dev/release/verify_rc.sh 19.0.0 1 +``` + +If the verification is successful, the message `RC looks good!` is shown. diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh new file mode 100755 index 000000000..84eded468 --- /dev/null +++ b/dev/release/bump_version.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo " e.g.: $0 19.0.1" + echo " e.g.: $0 20.0.0-SNAPSHOT" + exit 1 +fi + +version=$1 + +cd "${SOURCE_TOP_DIR}" + +git switch -c bump-version-${version} +mvn versions:set -DnewVersion=${version} -DprocessAllModules -DgenerateBackupPoms=false +case "${version}" in + *-SNAPSHOT) + tag=main + ;; + *) + tag=v${version} + ;; +esac +mvn versions:set-scm-tag -DnewTag=${tag} -DgenerateBackupPoms=false -pl :arrow-java-root +mvn versions:set-scm-tag -DnewTag=${tag} -DgenerateBackupPoms=false -pl :arrow-bom +git add "pom.xml" +git add "**/pom.xml" +git commit -m "MINOR: Bump version to ${version}" +gh pr create --fill diff --git a/dev/release/release.sh b/dev/release/release.sh new file mode 100755 index 000000000..ff3f854b1 --- /dev/null +++ b/dev/release/release.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo " e.g.: $0 19.0.1 1" + exit 1 +fi + +version=$1 +rc=$2 + +git_origin_url="$(git remote get-url origin)" +repository="${git_origin_url#*github.com?}" +repository="${repository%.git}" +if [ "${git_origin_url}" != "git@github.com:apache/arrow-java.git" ]; then + echo "This script must be ran with working copy of apache/arrow-java." + echo "The origin's URL: ${git_origin_url}" + exit 1 +fi + +tag="v${version}" +rc_tag="${tag}-rc${rc}" +echo "Tagging for release: ${tag}" +git tag -a -m "${version}" "${tag}" "${rc_tag}" +git push origin "${tag}" + +dist_url="https://dist.apache.org/repos/dist/release/arrow" +dist_dir="dev/release/dist" +echo "Checking out ${dist_url}" +rm -rf "${dist_dir}" +svn co --depth=empty "${dist_url}" "${dist_dir}" +gh release download "${rc_tag}" \ + --repo "${repository}" \ + --dir "${dist_dir}" \ + --skip-existing + +release_id="apache-arrow-java-${version}" +echo "Uploading to release/" +pushd "${dist_dir}" +svn add . +svn ci -m "Apache Arrow Java ${version}" +popd +rm -rf "${dist_dir}" + +echo "Keep only the latest versions" +old_releases=$( + svn ls https://dist.apache.org/repos/dist/release/arrow/ | + grep -E '^apache-arrow-java-' | + sort --version-sort --reverse | + tail -n +2 +) +for old_release_version in ${old_releases}; do + echo "Remove old release ${old_release_version}" + svn \ + delete \ + -m "Remove old Apache Arrow Java release: ${old_release_version}" \ + "https://dist.apache.org/repos/dist/release/arrow/${old_release_version}" +done + +echo "Success! The release is available here:" +echo " https://dist.apache.org/repos/dist/release/arrow/${release_id}" +echo +echo "Add this release to ASF's report database:" +echo " https://reporter.apache.org/addrelease.html?arrow" diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh new file mode 100755 index 000000000..7e820aab8 --- /dev/null +++ b/dev/release/release_rc.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo " e.g.: $0 1" + exit 1 +fi + +rc=$1 + +: "${RELEASE_DEFAULT:=1}" +: "${RELEASE_PULL:=${RELEASE_DEFAULT}}" +: "${RELEASE_PUSH_TAG:=${RELEASE_DEFAULT}}" +: "${RELEASE_SIGN:=${RELEASE_DEFAULT}}" +: "${RELEASE_UPLOAD:=${RELEASE_DEFAULT}}" + +cd "${SOURCE_TOP_DIR}" + +if [ "${RELEASE_PULL}" -gt 0 ] || [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then + git_origin_url="$(git remote get-url origin)" + if [ "${git_origin_url}" != "git@github.com:apache/arrow-java.git" ]; then + echo "This script must be ran with working copy of apache/arrow-java." + echo "The origin's URL: ${git_origin_url}" + exit 1 + fi +fi + +if [ "${RELEASE_PULL}" -gt 0 ]; then + echo "Ensure using the latest commit" + git checkout main + git pull --rebase --prune +fi + +version=$(grep -o '^ .*' "pom.xml" | + sed \ + -e 's,^ ,,' \ + -e 's,$,,') + +case "${version}" in + *-SNAPSHOT) + echo "Version isn't bumped: ${version}" + echo "Run dev/release/bump_version.sh before you run this script." + exit 1 + ;; +esac + +rc_tag="v${version}-rc${rc}" +if [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then + echo "Tagging for RC: ${rc_tag}" + git tag -a -m "${version} RC${rc}" "${rc_tag}" + git push origin "${rc_tag}" +fi + +rc_hash="$(git rev-list --max-count=1 "${rc_tag}")" + +id="apache-arrow-java-${version}" +tar_gz="${id}.tar.gz" + +if [ "${RELEASE_SIGN}" -gt 0 ]; then + git_origin_url="$(git remote get-url origin)" + repository="${git_origin_url#*github.com?}" + repository="${repository%.git}" + + echo "Looking for GitHub Actions workflow on ${repository}:${rc_tag}" + run_id="" + while [ -z "${run_id}" ]; do + echo "Waiting for run to start..." + run_id=$(gh run list \ + --branch "${rc_tag}" \ + --jq ".[].databaseId" \ + --json 'databaseId' \ + --limit 1 \ + --repo "${repository}" \ + --workflow rc.yml) + sleep 1 + done + + echo "Found GitHub Actions workflow with ID: ${run_id}" + gh run watch --repo "${repository}" --exit-status "${run_id}" + + echo "Downloading .tar.gz from GitHub Releases" + gh release download "${rc_tag}" \ + --dir . \ + --pattern "${tar_gz}" \ + --repo "${repository}" \ + --skip-existing + + echo "Signing tar.gz and creating checksums" + gpg --armor --output "${tar_gz}.asc" --detach-sig "${tar_gz}" +fi + +if [ "${RELEASE_UPLOAD}" -gt 0 ]; then + echo "Uploading signature" + gh release upload "${rc_tag}" \ + --clobber \ + --repo "${repository}" \ + "${tar_gz}.asc" +fi + +echo "Draft email for dev@arrow.apache.org mailing list" +echo "" +echo "---------------------------------------------------------" +cat < " + echo " e.g.: $0 19.0.1 1" + exit 1 +fi + +set -o pipefail +set -x + +VERSION="$1" +RC="$2" + +ARROW_DIST_BASE_URL="https://dist.apache.org/repos/dist/dev/arrow" +DOWNLOAD_RC_BASE_URL="https://github.com/apache/arrow-java/releases/download/v${VERSION}-rc${RC}" +ARCHIVE_BASE_NAME="apache-arrow-java-${VERSION}" + +: "${VERIFY_DEFAULT:=1}" +: "${VERIFY_DOWNLOAD:=${VERIFY_DEFAULT}}" +: "${VERIFY_SIGN:=${VERIFY_DEFAULT}}" +: "${VERIFY_SOURCE:=${VERIFY_DEFAULT}}" +: "${VERIFY_BINARY:=${VERIFY_DEFAULT}}" + +VERIFY_SUCCESS=no + +setup_tmpdir() { + cleanup() { + if [ "${VERIFY_SUCCESS}" = "yes" ]; then + rm -rf "${VERIFY_TMPDIR}" + else + echo "Failed to verify release candidate. See ${VERIFY_TMPDIR} for details." + fi + } + + if [ -z "${VERIFY_TMPDIR:-}" ]; then + VERIFY_TMPDIR="$(mktemp -d -t "$1.XXXXX")" + trap cleanup EXIT + else + mkdir -p "${VERIFY_TMPDIR}" + fi +} + +download() { + curl \ + --fail \ + --location \ + --remote-name \ + --show-error \ + --silent \ + "$1" +} + +download_rc_file() { + if [ "${VERIFY_DOWNLOAD}" -gt 0 ]; then + download "${DOWNLOAD_RC_BASE_URL}/$1" + else + cp "${TOP_SOURCE_DIR}/$1" "$1" + fi +} + +import_gpg_keys() { + if [ "${VERIFY_SIGN}" -gt 0 ]; then + download "${ARROW_DIST_BASE_URL}/KEYS" + gpg --import KEYS + fi +} + +if type shasum >/dev/null 2>&1; then + sha256_verify="shasum -a 256 -c" + sha512_verify="shasum -a 512 -c" +else + sha256_verify="sha256sum -c" + sha512_verify="sha512sum -c" +fi + +fetch_archive() { + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz" + if [ "${VERIFY_SIGN}" -gt 0 ]; then + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.asc" + gpg --verify "${ARCHIVE_BASE_NAME}.tar.gz.asc" "${ARCHIVE_BASE_NAME}.tar.gz" + fi + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.sha256" + ${sha256_verify} "${ARCHIVE_BASE_NAME}.tar.gz.sha256" + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.sha512" + ${sha512_verify} "${ARCHIVE_BASE_NAME}.tar.gz.sha512" +} + +ensure_source_directory() { + tar xf "${ARCHIVE_BASE_NAME}".tar.gz + + if [ -d "${TOP_SOURCE_DIR}/testing/data" ]; then + cp -a "${TOP_SOURCE_DIR}/testing" "${ARCHIVE_BASE_NAME}/" + else + git clone \ + https://github.com/apache/arrow-testing.git \ + "${ARCHIVE_BASE_NAME}/testing" + fi + + ARROW_TEST_DATA="${ARCHIVE_BASE_NAME}/testing/data" + export ARROW_TEST_DATA +} + +test_source_distribution() { + if [ ${VERIFY_SOURCE} -le 0 ]; then + return 0 + fi + + "${TOP_SOURCE_DIR}/ci/scripts/build.sh" "$(pwd)" + "${TOP_SOURCE_DIR}/ci/scripts/test.sh" "$(pwd)" + # TODO: JNI test + # TODO: Integration test +} + +test_binary_distribution() { + if [ ${VERIFY_SOURCE} -le 0 ]; then + return 0 + fi + + # TODO: jar test +} + +setup_tmpdir "arrow-java-${VERSION}-${RC}" +echo "Working in sandbox ${VERIFY_TMPDIR}" +cd "${VERIFY_TMPDIR}" + +import_gpg_keys +fetch_archive +ensure_source_directory +pushd "${ARCHIVE_BASE_NAME}" +test_source_distribution +test_binary_distribution +popd + +VERIFY_SUCCESS=yes +echo "RC looks good!" From 88a34566621a876ae06bd590ccbd3d86806bedd4 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 9 Jan 2025 17:35:52 +0900 Subject: [PATCH 02/28] Don't run release workflow for PR --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e0b866e84..f8d8eefb4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,13 +15,12 @@ # specific language governing permissions and limitations # under the License. -name: RC +name: Release on: push: tags: - '*' - '!*-rc*' - pull_request: concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true From b417364f00dd52f595c38fbf1e4798563fc6db86 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 9 Jan 2025 17:40:38 +0900 Subject: [PATCH 03/28] Fix style --- dev/release/bump_version.sh | 12 ++++++------ dev/release/release_rc.sh | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index 84eded468..6681b115d 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -36,12 +36,12 @@ cd "${SOURCE_TOP_DIR}" git switch -c bump-version-${version} mvn versions:set -DnewVersion=${version} -DprocessAllModules -DgenerateBackupPoms=false case "${version}" in - *-SNAPSHOT) - tag=main - ;; - *) - tag=v${version} - ;; +*-SNAPSHOT) + tag=main + ;; +*) + tag=v${version} + ;; esac mvn versions:set-scm-tag -DnewTag=${tag} -DgenerateBackupPoms=false -pl :arrow-java-root mvn versions:set-scm-tag -DnewTag=${tag} -DgenerateBackupPoms=false -pl :arrow-bom diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index 7e820aab8..3021b93b5 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -59,11 +59,11 @@ version=$(grep -o '^ .*' "pom.xml" | -e 's,$,,') case "${version}" in - *-SNAPSHOT) - echo "Version isn't bumped: ${version}" - echo "Run dev/release/bump_version.sh before you run this script." - exit 1 - ;; +*-SNAPSHOT) + echo "Version isn't bumped: ${version}" + echo "Run dev/release/bump_version.sh before you run this script." + exit 1 + ;; esac rc_tag="v${version}-rc${rc}" From c2a91fa748818c79294f44bac64217758c30712a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 9 Jan 2025 17:43:46 +0900 Subject: [PATCH 04/28] Add missing arguments --- dev/release/verify_rc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index debf2b916..f022b622b 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -128,8 +128,8 @@ test_source_distribution() { return 0 fi - "${TOP_SOURCE_DIR}/ci/scripts/build.sh" "$(pwd)" - "${TOP_SOURCE_DIR}/ci/scripts/test.sh" "$(pwd)" + "${TOP_SOURCE_DIR}/ci/scripts/build.sh" "$(pwd)" build jni_dist + "${TOP_SOURCE_DIR}/ci/scripts/test.sh" "$(pwd)" build jni_dist # TODO: JNI test # TODO: Integration test } From fed3ea55f9c6336a42ff80dc54ee818895999486 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 9 Jan 2025 17:49:00 +0900 Subject: [PATCH 05/28] Fix lint errors --- dev/release/bump_version.sh | 8 ++++---- dev/release/verify_rc.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index 6681b115d..d1b164f10 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -33,8 +33,8 @@ version=$1 cd "${SOURCE_TOP_DIR}" -git switch -c bump-version-${version} -mvn versions:set -DnewVersion=${version} -DprocessAllModules -DgenerateBackupPoms=false +git switch -c "bump-version-${version}" +mvn versions:set "-DnewVersion=${version}" -DprocessAllModules -DgenerateBackupPoms=false case "${version}" in *-SNAPSHOT) tag=main @@ -43,8 +43,8 @@ case "${version}" in tag=v${version} ;; esac -mvn versions:set-scm-tag -DnewTag=${tag} -DgenerateBackupPoms=false -pl :arrow-java-root -mvn versions:set-scm-tag -DnewTag=${tag} -DgenerateBackupPoms=false -pl :arrow-bom +mvn versions:set-scm-tag "-DnewTag=${tag}" -DgenerateBackupPoms=false -pl :arrow-java-root +mvn versions:set-scm-tag "-DnewTag=${tag}" -DgenerateBackupPoms=false -pl :arrow-bom git add "pom.xml" git add "**/pom.xml" git commit -m "MINOR: Bump version to ${version}" diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index f022b622b..c5942552a 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -124,7 +124,7 @@ ensure_source_directory() { } test_source_distribution() { - if [ ${VERIFY_SOURCE} -le 0 ]; then + if [ "${VERIFY_SOURCE}" -le 0 ]; then return 0 fi @@ -135,7 +135,7 @@ test_source_distribution() { } test_binary_distribution() { - if [ ${VERIFY_SOURCE} -le 0 ]; then + if [ "${VERIFY_BINARY}" -le 0 ]; then return 0 fi From 29aa2108af0e283f9f27ad829345313d0e497bf3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 9 Jan 2025 18:13:39 +0900 Subject: [PATCH 06/28] Add missing prefix --- dev/release/verify_rc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index c5942552a..e2de862a8 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -119,7 +119,7 @@ ensure_source_directory() { "${ARCHIVE_BASE_NAME}/testing" fi - ARROW_TEST_DATA="${ARCHIVE_BASE_NAME}/testing/data" + ARROW_TEST_DATA="$(pwd)/${ARCHIVE_BASE_NAME}/testing/data" export ARROW_TEST_DATA } From ed5261548880d6c34e729d7f41f3d6d1a860e23d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 12:56:37 +0900 Subject: [PATCH 07/28] Remove needless GITHUB_TOKEN --- .github/workflows/rc.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 9d1a97b9f..d2987c3d2 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -106,8 +106,6 @@ jobs: VERIFY_DEFAULT=0 \ VERIFY_SOURCE=1 \ dev/release/verify_rc.sh "${version}" "${rc}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} upload: name: Upload if: github.ref_type == 'tag' From 9a116397a627b5bd8bf22744ab45065e2984451a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:09:03 +0900 Subject: [PATCH 08/28] Don't accept addition commit in main --- dev/release/release_rc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index 3021b93b5..cd19d1293 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -50,7 +50,7 @@ fi if [ "${RELEASE_PULL}" -gt 0 ]; then echo "Ensure using the latest commit" git checkout main - git pull --rebase --prune + git pull --ff-only fi version=$(grep -o '^ .*' "pom.xml" | From 191e5846c1e4acd8839ae3f7ad4d577957310fb2 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:09:16 +0900 Subject: [PATCH 09/28] Accept https://github.com/apache/arrow-java.git --- dev/release/release_rc.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index cd19d1293..55c706715 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -40,10 +40,15 @@ cd "${SOURCE_TOP_DIR}" if [ "${RELEASE_PULL}" -gt 0 ] || [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then git_origin_url="$(git remote get-url origin)" - if [ "${git_origin_url}" != "git@github.com:apache/arrow-java.git" ]; then + case "${git_origin_url}" in + git@github.com:apache/arrow-java.git|https://github.com/apache/arrow-java.git) + : # OK + ;; + *) echo "This script must be ran with working copy of apache/arrow-java." echo "The origin's URL: ${git_origin_url}" exit 1 + ;; fi fi From ca473de486cb8b8e0ab8c8a3ce69eb507e0cdabe Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:09:27 +0900 Subject: [PATCH 10/28] Describe dev/release/bump_version.sh --- dev/release/README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/dev/release/README.md b/dev/release/README.md index 27142406f..018a04c10 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -22,11 +22,35 @@ ## Overview 1. Test the revision to be released - 2. Bump version by `dev/release/bump_version.sh X.Y.Z` + 2. Bump version (detailed later) 3. Prepare RC and vote (detailed later) 4. Publish (detailed later) 5. Bump version by `dev/release/bump_version.sh X.Y.Z-SNAPSHOT` +### Bump version + +Run `dev/release/bump_version.sh` on a working copy of your fork not +`git@github.com:apache/arrow-java`: + +```console +$ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} +$ cd arrow-java.${YOUR_GITHUB_ACCOUNT} +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh ${NEW_VERSION} +``` + +Here is an example to bump version to 19.0.0: + +``` +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0 +``` + +It creates a feature branch and adds a commit that bumps version. This +opens a pull request from the feature branch by `gh gr create`. So you +need `gh` command and GitHub personal access token. + +See also: +https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + ### Prepare RC and vote Run `dev/release/release_rc.sh` on a working copy of @@ -34,6 +58,7 @@ Run `dev/release/release_rc.sh` on a working copy of ```console $ git clone git@github.com:apache/arrow-java.git +$ cd arrow-java $ dev/release/release_rc.sh ${RC} (Send a vote email to dev@arrow.apache.org. You can use a draft shown by release_rc.sh for the email.) From 7c51faaec7639eb2b030fa3d2f0b794b0dbedd13 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:09:43 +0900 Subject: [PATCH 11/28] Use --generate-notes --- .github/workflows/rc.yml | 4 ++-- .github/workflows/release.yml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index d2987c3d2..71979e9fa 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -124,12 +124,12 @@ jobs: name: archive - name: Upload run: | - # TODO: Add support for release note + # GH-499: How to create release notes? version=${GITHUB_REF_NAME%-rc*} version=${version#v} rc=${GITHUB_REF_NAME#*-rc} gh release create ${GITHUB_REF_NAME} \ - --notes "TODO" \ + --generate-notes \ --prerelease \ --title "Apache Arrow Java ${version} RC${rc}" \ --verify-tag \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f8d8eefb4..e486cefe1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,9 +45,10 @@ jobs: gh release download ${latest_rc_tag} --dir dists - name: Create GitHub Release run: | + # GH-499: How to create release notes? version=${GITHUB_REF_NAME#v} gh release create ${GITHUB_REF_NAME} \ - --notes "TODO" \ + --generate-notes \ --title "Apache Arrow Java ${version}" \ --verify-tag \ dists/* From f08525b7c4ddb0fddd126d8a7259bc899997d345 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:09:50 +0900 Subject: [PATCH 12/28] Nightly --- .github/workflows/rc.yml | 8 +++++--- .github/workflows/release.yml | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 71979e9fa..5a82d686e 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -19,11 +19,13 @@ name: RC on: push: branches: - - '**' - - '!dependabot/**' + - "**" + - "!dependabot/**" tags: - - '*-rc*' + - "*-rc*" pull_request: + schedule: + - cron: "0 0 * * *" concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e486cefe1..4982c6b34 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,8 +19,8 @@ name: Release on: push: tags: - - '*' - - '!*-rc*' + - "*" + - "!*-rc*" concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true From ef414bf8118dbaee3008ebcf3bae779753695d83 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:12:23 +0900 Subject: [PATCH 13/28] Update end tag --- dev/release/release_rc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index 55c706715..75c81eb18 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -49,7 +49,7 @@ if [ "${RELEASE_PULL}" -gt 0 ] || [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then echo "The origin's URL: ${git_origin_url}" exit 1 ;; - fi + esac fi if [ "${RELEASE_PULL}" -gt 0 ]; then From cf645a687a86f62e2171787b91c6a2251770a279 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:12:36 +0900 Subject: [PATCH 14/28] Fix style --- dev/release/release_rc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index 75c81eb18..c8f1024a3 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -41,7 +41,7 @@ cd "${SOURCE_TOP_DIR}" if [ "${RELEASE_PULL}" -gt 0 ] || [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then git_origin_url="$(git remote get-url origin)" case "${git_origin_url}" in - git@github.com:apache/arrow-java.git|https://github.com/apache/arrow-java.git) + git@github.com:apache/arrow-java.git | https://github.com/apache/arrow-java.git) : # OK ;; *) From 405073e037765b86a0d8b1935b6a3eb14b6ecdc5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:15:12 +0900 Subject: [PATCH 15/28] Use main as the base branch --- dev/release/bump_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index d1b164f10..397d8897d 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -33,7 +33,7 @@ version=$1 cd "${SOURCE_TOP_DIR}" -git switch -c "bump-version-${version}" +git switch -c "bump-version-${version}" main mvn versions:set "-DnewVersion=${version}" -DprocessAllModules -DgenerateBackupPoms=false case "${version}" in *-SNAPSHOT) From 9ff0ff245e67701702616f6a0887ddf7ce0b8619 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:16:34 +0900 Subject: [PATCH 16/28] Specify repository explicitly --- dev/release/bump_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index 397d8897d..a8cb42c0f 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -48,4 +48,4 @@ mvn versions:set-scm-tag "-DnewTag=${tag}" -DgenerateBackupPoms=false -pl :arrow git add "pom.xml" git add "**/pom.xml" git commit -m "MINOR: Bump version to ${version}" -gh pr create --fill +gh pr create --fill --repo apache/arrow-java From b736ee2f728e1edbfc42b9141e57ebc4c0c6d378 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:19:13 +0900 Subject: [PATCH 17/28] Push explicitly --- dev/release/bump_version.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index a8cb42c0f..3102ed9e9 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -33,6 +33,17 @@ version=$1 cd "${SOURCE_TOP_DIR}" +git_origin_url="$(git remote get-url origin)" +case "${git_origin_url}" in +*apache/arrow-java*) + echo "You must use your fork: ${git_origin_url}" + exit 1 + ;; +*) + : # OK + ;; +esac + git switch -c "bump-version-${version}" main mvn versions:set "-DnewVersion=${version}" -DprocessAllModules -DgenerateBackupPoms=false case "${version}" in @@ -48,4 +59,5 @@ mvn versions:set-scm-tag "-DnewTag=${tag}" -DgenerateBackupPoms=false -pl :arrow git add "pom.xml" git add "**/pom.xml" git commit -m "MINOR: Bump version to ${version}" +git push origin gh pr create --fill --repo apache/arrow-java From e89edb1a3b06bc229a69f5111291d37348aca6cd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:20:11 +0900 Subject: [PATCH 18/28] Specify branch explicitly --- dev/release/bump_version.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index 3102ed9e9..a6600a1c9 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -44,7 +44,8 @@ case "${git_origin_url}" in ;; esac -git switch -c "bump-version-${version}" main +branch="bump-version-${version}" +git switch -c "${branch}" main mvn versions:set "-DnewVersion=${version}" -DprocessAllModules -DgenerateBackupPoms=false case "${version}" in *-SNAPSHOT) @@ -59,5 +60,5 @@ mvn versions:set-scm-tag "-DnewTag=${tag}" -DgenerateBackupPoms=false -pl :arrow git add "pom.xml" git add "**/pom.xml" git commit -m "MINOR: Bump version to ${version}" -git push origin +git push origin "${branch}" gh pr create --fill --repo apache/arrow-java From eb3c4297de0645e55132af388a6563cea6a8671f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 13:21:48 +0900 Subject: [PATCH 19/28] Set upstream explicitly --- dev/release/bump_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index a6600a1c9..d1c127de3 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -60,5 +60,5 @@ mvn versions:set-scm-tag "-DnewTag=${tag}" -DgenerateBackupPoms=false -pl :arrow git add "pom.xml" git add "**/pom.xml" git commit -m "MINOR: Bump version to ${version}" -git push origin "${branch}" +git push --set-upstream origin "${branch}" gh pr create --fill --repo apache/arrow-java From f996be69b92173ac909c0a585284a8afd8e4fa76 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 14:49:36 +0900 Subject: [PATCH 20/28] Add missing GH_TOKEN --- dev/release/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/README.md b/dev/release/README.md index 018a04c10..6627420ba 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -59,7 +59,7 @@ Run `dev/release/release_rc.sh` on a working copy of ```console $ git clone git@github.com:apache/arrow-java.git $ cd arrow-java -$ dev/release/release_rc.sh ${RC} +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh ${RC} (Send a vote email to dev@arrow.apache.org. You can use a draft shown by release_rc.sh for the email.) ``` From 6402ef4502a4c284f96ada79a22453cc274ea430 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 17:21:31 +0900 Subject: [PATCH 21/28] Accept https:// --- dev/release/release.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dev/release/release.sh b/dev/release/release.sh index ff3f854b1..3037dd6b0 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -31,11 +31,16 @@ rc=$2 git_origin_url="$(git remote get-url origin)" repository="${git_origin_url#*github.com?}" repository="${repository%.git}" -if [ "${git_origin_url}" != "git@github.com:apache/arrow-java.git" ]; then +case "${git_origin_url}" in +git@github.com:apache/arrow-java.git | https://github.com/apache/arrow-java.git) + : # OK + ;; +*) echo "This script must be ran with working copy of apache/arrow-java." echo "The origin's URL: ${git_origin_url}" exit 1 -fi + ;; +esac tag="v${version}" rc_tag="${tag}-rc${rc}" From d23d600370c88f27e628f99bd716de31089a23a5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 17:21:37 +0900 Subject: [PATCH 22/28] Fix release path --- dev/release/release.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dev/release/release.sh b/dev/release/release.sh index 3037dd6b0..911b561c6 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -48,19 +48,20 @@ echo "Tagging for release: ${tag}" git tag -a -m "${version}" "${tag}" "${rc_tag}" git push origin "${tag}" +release_id="apache-arrow-java-${version}" dist_url="https://dist.apache.org/repos/dist/release/arrow" -dist_dir="dev/release/dist" +dist_base_dir="dev/release/dist" +dist_dir="dev/release/dist/${release_id}" echo "Checking out ${dist_url}" -rm -rf "${dist_dir}" -svn co --depth=empty "${dist_url}" "${dist_dir}" +rm -rf "${dist_base_dir}" +svn co --depth=empty "${dist_url}" "${dist_base_dir}" gh release download "${rc_tag}" \ --repo "${repository}" \ --dir "${dist_dir}" \ --skip-existing -release_id="apache-arrow-java-${version}" echo "Uploading to release/" -pushd "${dist_dir}" +pushd "${dist_base_dir}" svn add . svn ci -m "Apache Arrow Java ${version}" popd From 13df24d8db5288c0a31a14086b4f986dd98544b8 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 17:31:58 +0900 Subject: [PATCH 23/28] Fix name --- dev/release/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/release.sh b/dev/release/release.sh index 911b561c6..df671108d 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -65,7 +65,7 @@ pushd "${dist_base_dir}" svn add . svn ci -m "Apache Arrow Java ${version}" popd -rm -rf "${dist_dir}" +rm -rf "${dist_base_dir}" echo "Keep only the latest versions" old_releases=$( From 711bf2e3b55295f05a2a6d26a5c7b38d854b31d5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 11 Jan 2025 05:21:46 +0900 Subject: [PATCH 24/28] Fix a typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Raúl Cumplido --- dev/release/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/README.md b/dev/release/README.md index 6627420ba..c984d401b 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -45,7 +45,7 @@ $ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0 ``` It creates a feature branch and adds a commit that bumps version. This -opens a pull request from the feature branch by `gh gr create`. So you +opens a pull request from the feature branch by `gh pr create`. So you need `gh` command and GitHub personal access token. See also: From e59a1b1cc96a2982b9547175e50cebf46283c10f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 11 Jan 2025 05:45:29 +0900 Subject: [PATCH 25/28] Fix how to update KEYS --- dev/release/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/release/README.md b/dev/release/README.md index c984d401b..7f0c07964 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -94,7 +94,8 @@ their ASF account. e.g.: ```console $ svn co https://dist.apache.org/repos/dist/dev/arrow $ cd arrow -$ editor KEYS +$ head KEYS +(This shows how to update KEYS) $ svn ci KEYS ``` From ab283539e48f55701ef4e45b0dcb126b46aba90f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 11 Jan 2025 05:49:44 +0900 Subject: [PATCH 26/28] Add a note about merging PR --- dev/release/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/release/README.md b/dev/release/README.md index 7f0c07964..576859fd7 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -51,6 +51,9 @@ need `gh` command and GitHub personal access token. See also: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens +We need to merge the pull request before we cut RC. If we try cut RC +without merging the pull request, the script to cut RC is failed. + ### Prepare RC and vote Run `dev/release/release_rc.sh` on a working copy of From 784fc63652a40446dfd5f838419072aaf9a072ec Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 11 Jan 2025 05:54:42 +0900 Subject: [PATCH 27/28] Describe -SNAPSHOT update --- dev/release/README.md | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index 576859fd7..eafbb61b8 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -22,12 +22,12 @@ ## Overview 1. Test the revision to be released - 2. Bump version (detailed later) + 2. Bump version for new release (detailed later) 3. Prepare RC and vote (detailed later) 4. Publish (detailed later) - 5. Bump version by `dev/release/bump_version.sh X.Y.Z-SNAPSHOT` + 5. Bump version for new development (detailed later) -### Bump version +### Bump version for new release Run `dev/release/bump_version.sh` on a working copy of your fork not `git@github.com:apache/arrow-java`: @@ -51,8 +51,8 @@ need `gh` command and GitHub personal access token. See also: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens -We need to merge the pull request before we cut RC. If we try cut RC -without merging the pull request, the script to cut RC is failed. +We need to merge the pull request before we cut a RC. If we try cut a +RC without merging the pull request, the script to cut a RC is failed. ### Prepare RC and vote @@ -125,7 +125,30 @@ $ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh 19.0.0 1 Add the release to ASF's report database via [Apache Committee Report Helper](https://reporter.apache.org/addrelease.html?arrow). -### Verify +### Bump version for new development + +We should bump version in the main branch for new development after we +release a new version. + +Run `dev/release/bump_version.sh` on a working copy of your fork not +`git@github.com:apache/arrow-java`: + +```console +$ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} +$ cd arrow-java.${YOUR_GITHUB_ACCOUNT} +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh ${NEW_VERSION}-SNAPSHOT +``` + +Here is an example to bump version to 19.0.1-SNAPSHOT: + +``` +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0-SNAPSHOT +``` + +It creates a feature branch and adds a commit that bumps version. This +opens a pull request from the feature branch by `gh pr create`. + +## Verify We have a script to verify a RC. From 4156c800b0574c9bdc9933c95cd0a7a34206ecab Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 11 Jan 2025 05:56:17 +0900 Subject: [PATCH 28/28] Use release/arrow/KEYS for verification --- dev/release/verify_rc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index e2de862a8..d7ed6fa23 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -34,7 +34,7 @@ set -x VERSION="$1" RC="$2" -ARROW_DIST_BASE_URL="https://dist.apache.org/repos/dist/dev/arrow" +ARROW_DIST_BASE_URL="https://dist.apache.org/repos/dist/release/arrow" DOWNLOAD_RC_BASE_URL="https://github.com/apache/arrow-java/releases/download/v${VERSION}-rc${RC}" ARCHIVE_BASE_NAME="apache-arrow-java-${VERSION}"