From 3593c6323ffad9b0d06209feeb609d76849b1e20 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 12 Sep 2024 11:00:12 +0900 Subject: [PATCH 1/6] GH-21: Add release CI Fix GH-21 This is based on https://github.com/apache/arrow-julia/tree/main/dev/release and https://github.com/apache/arrow-adbc/tree/main/dev/release . Workflow: Cut a RC: 1. Update `PkgVersion` in `arrow/doc.go` 2. Run `dev/release/release_rc.sh` 1. `dev/release/release_rc.sh` pushes `v${version}-rc${rc}` tag 2. `.github/workflows/rc.yml` creates `apache-arrow-go-${version}.tar.gz{,.sha256,.sha512}` 3. `.github/workflows/rc.yml` uploads `apache-arrow-go-${version}.tar.gz{,.sha256,.sha512}` to GitHub Releases 4. `dev/release/release_rc.sh` downloads `apache-arrow-go-${version}.tar.gz` from GitHub Releases 5. `dev/release/release_rc.sh` signs `apache-arrow-go-${version}.tar.gz` as `apache-arrow-go-${version}.tar.gz.asc` 6. `dev/release/release_rc.sh` uploads `apache-arrow-go-${version}.tar.gz.asc` to GitHub Releases 3. Start a vote (GitHub Actions instead of https://dist.apache.org/repos/dist/dev/arrow/ is used like ADBC.) Verify a RC: 1. Run `dev/release/verify_rc.sh` Release an approved RC: 1. Run `dev/release/release.sh` 1. `dev/release/release.sh` pushes `v${version}` tag that refers that same commit ID as `v${version}-rc${rc}` 2. `dev/release/release.sh` downloads `apache-arrow-go-${version}.tar.gz{,.asc,.sha256,.sha512}` from GitHub Actions 3. `dev/release/release.sh` uploads `apache-arrow-go-${version}.tar.gz{,.asc,.sha256,.sha512}` to https://dist.apache.org/repos/dist/release/arrow 4. `dev/release/release.sh` removes old releases from https://dist.apache.org/repos/dist/release/arrow 2. Add this release to ASF's report database: https://reporter.apache.org/addrelease.html?arrow Follow-up tasks: * Add support for running integration test in the verification script * Add support for releasing a release note * Add support for creating "v${version}"'s GitHub Releases * We can copy "v${version}-rc${rc}"'s GitHub Releases and adjust it for the official release --- .github/workflows/lint.yml | 7 +- .github/workflows/rc.yml | 133 ++++++++++++++++++++++ .github/workflows/test.yml | 24 ++-- arrow/doc.go | 2 +- ci/scripts/build.sh | 4 +- dev/release/README.md | 116 +++++++++++++++++++ dev/release/release.sh | 83 ++++++++++++++ dev/release/release_rc.sh | 141 +++++++++++++++++++++++ dev/release/verify_rc.sh | 226 +++++++++++++++++++++++++++++++++++++ 9 files changed, 718 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/rc.yml create mode 100644 dev/release/README.md create mode 100755 dev/release/release.sh create mode 100755 dev/release/release_rc.sh create mode 100755 dev/release/verify_rc.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index caf59d8d..872d404d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,8 +18,11 @@ name: Lint on: push: - branches-ignore: - - 'dependabot/**' + branches: + - '**' + - '!dependabot/**' + tags: + - '*' pull_request: concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml new file mode 100644 index 00000000..acd34ace --- /dev/null +++ b/.github/workflows/rc.yml @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: RC +on: + push: + branches: + - '**' + - '!dependabot/**' + tags: + - '*-rc*' + pull_request: +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +permissions: + contents: read +jobs: + archive: + name: Archive + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: recursive + - name: Prepare for tag + if: github.ref_type == 'tag' + run: | + version=${GITHUB_REF_NAME%-rc*} + version=${version#v} + rc=${GITHUB_REF_NAME#*-rc} + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Prepare for branch + if: github.ref_type == 'branch' + run: | + version=$(grep -o '^const PkgVersion = ".*"' "arrow/doc.go" | + sed -e 's/^const PkgVersion = "//' \ + -e 's/"$//') + rc=100 + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Archive + run: | + id="apache-arrow-go-${VERSION}" + tar_gz="${id}.tar.gz" + echo "TAR_GZ=${tar_gz}" >> ${GITHUB_ENV} + git archive HEAD --prefix "${id}/" --output "${tar_gz}" + sha256sum "${tar_gz}" > "${tar_gz}.sha256" + sha512sum "${tar_gz}" > "${tar_gz}.sha512" + - name: Audit + run: | + dev/release/run_rat.sh "${TAR_GZ}" + - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: archive + path: | + apache-arrow-go-* + verify: + name: Verify + needs: + - archive + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: recursive + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: archive + - name: Verify + run: | + tar_gz=$(echo apache-arrow-go-*.tar.gz) + version=${tar_gz#apache-arrow-go-} + version=${version%.tar.gz} + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + rc="${GITHUB_REF_NAME#*-rc}" + else + rc=100 + fi + VERIFY_DEFAULT=0 dev/release/verify_rc.sh "${version}" "${rc}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + upload: + name: Upload + if: github.ref_type == 'tag' + needs: + - verify + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: recursive + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: archive + - name: Upload + run: | + # TODO: Add support for release note + gh release create ${GITHUB_REF_NAME} \ + --prerelease \ + --title "Apache Arrow Go ${GITHUB_REF_NAME}" \ + --verify-tag \ + apache-arrow-go-*.tar.gz \ + apache-arrow-go-*.tar.gz.sha* + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99acc9bb..3a8e7b59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,8 +18,11 @@ name: Test on: push: - branches-ignore: - - 'dependabot/**' + branches: + - '**' + - '!dependabot/**' + tags: + - '*' pull_request: concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -202,15 +205,12 @@ jobs: go-version: ${{ matrix.go }} cache: true cache-dependency-path: go.sum - - name: Install bash via Homebrew - run: | - brew install bash - name: Build run: | - $(brew --prefix)/bin/bash ci/scripts/build.sh $(pwd) + ci/scripts/build.sh $(pwd) - name: Test run: | - $(brew --prefix)/bin/bash ci/scripts/test.sh $(pwd) + ci/scripts/test.sh $(pwd) macos-cgo: name: AMD64 macOS 12 Go ${{ matrix.go }} - CGO runs-on: macos-12 @@ -235,18 +235,16 @@ jobs: cache: true cache-dependency-path: go.sum - name: Brew Install Arrow and pkg-config - shell: bash - run: brew install apache-arrow pkg-config bash - - name: Add To pkg config path - shell: bash + run: brew install apache-arrow pkg-config + - name: Setup PKG_CONFIG_PATH run: | echo "PKG_CONFIG_PATH=$(brew --prefix openssl@3)/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV - name: Build run: | - $(brew --prefix)/bin/bash ci/scripts/build.sh $(pwd) + ci/scripts/build.sh $(pwd) - name: Test run: | - $(brew --prefix)/bin/bash ci/scripts/test.sh $(pwd) + ci/scripts/test.sh $(pwd) windows: name: AMD64 Windows 2019 Go ${{ matrix.go }} runs-on: windows-2019 diff --git a/arrow/doc.go b/arrow/doc.go index 30e6b7eb..44eda7ec 100644 --- a/arrow/doc.go +++ b/arrow/doc.go @@ -34,7 +34,7 @@ To build with tinygo include the noasm build tag. */ package arrow -const PkgVersion = "18.0.0-SNAPSHOT" +const PkgVersion = "18.0.0" //go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl //go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index d6e10355..acfca173 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -28,7 +28,7 @@ pushd "${source_dir}/arrow" : "${ARROW_GO_TESTCGO:=}" -go_install_arrow_options=() +go_install_arrow_options=(-v) if [[ -n "${ARROW_GO_TESTCGO}" ]]; then if [[ "${MSYSTEM:-}" = "MINGW64" ]]; then export PATH=${MINGW_PREFIX}/bin:${PATH} @@ -38,7 +38,7 @@ if [[ -n "${ARROW_GO_TESTCGO}" ]]; then go_install_arrow_options+=("-tags" "assert,test,ccalloc") fi -go install "${go_install_arrow_options[@]}" -v ./... +go install "${go_install_arrow_options[@]}" ./... popd diff --git a/dev/release/README.md b/dev/release/README.md new file mode 100644 index 00000000..310d3903 --- /dev/null +++ b/dev/release/README.md @@ -0,0 +1,116 @@ + + +# Release + +## Overview + + 1. Test the revision to be released + 2. Increment version number in `arrow/doc.go` + 3. Prepare RC and vote (detailed later) + 4. Publish (detailed later) + +### Prepare RC and vote + +Run `dev/release/release_rc.sh` on working copy of `git@github.com:apache/arrow-go` not your fork: + +```console +$ git clone git@github.com:apache/arrow-go.git +$ dev/release/release_rc.sh ${RC} +(Send a vote email to dev@arrow.apache.org. + You can use a draft shown by release_rc.sh for the email.) +``` + +Here is an example to release RC1: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh 1 +``` + +The argument of `release_rc.sh` is the RC number. If RC1 has a problem, we'll increment the RC number such as RC2, RC3 and so on. + +Requirements to run `release_rc.sh`: + + * You must be an Apache Arrow committer or PMC member + * You must prepare your PGP key for signing + +If you don't have a PGP key, https://infra.apache.org/release-signing.html#generate may be helpful. + +Your PGP key must be registered to the followings: + + * https://dist.apache.org/repos/dist/dev/arrow/KEYS + * https://dist.apache.org/repos/dist/release/arrow/KEYS + +See the header comment of them how to add a PGP key. + +Apache arrow committers can update them by Subversion client with their ASF account. e.g.: + +```console +$ svn co https://dist.apache.org/repos/dist/dev/arrow +$ cd arrow +$ editor KEYS +$ svn ci KEYS +``` + +### Publish + +We need to do the followings to publish a new release: + + * Publish to apache.org + +Run `dev/release/release.sh` to publish to apache.org: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh ${VERSION} ${RC} +``` + +Here is an example to release 18.0.0 RC1: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN}dev/release/release.sh 18.0.0 1 +``` + +Add the release to ASF's report database via [Apache Committee Report Helper](https://reporter.apache.org/addrelease.html?arrow). + +### Verify + +We have a script to verify a RC. + +You must install the following commands to use the script: + + * `curl` + * `gpg` + * `shasum` or `sha256sum`/`sha512sum` + * `tar` + +You don't need to install Go. If Go doesn't exist in system, the latest Go is automatically installed only for verification. + +To verify a RC, run the following command line: + +```console +$ dev/release/verify_rc.sh ${VERSION} ${RC} +``` + +Here is an example to release 18.0.0 RC1: + +```console +$ dev/release/verify_rc.sh 18.0.0 1 +``` + +If the verification is succeeded, `RC looks good!` is shown. diff --git a/dev/release/release.sh b/dev/release/release.sh new file mode 100755 index 00000000..3905a5ae --- /dev/null +++ b/dev/release/release.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo " e.g.: $0 18.0.0 1" + exit 1 +fi + +version=$1 +rc=$2 + +git_origin_url="$(git remote get-url origin)" +repository="${git_origin_url#*github.com?}" +repository="${repository%.git}" +if [ "${git_origin_url}" != "git@github.com:apache/arrow-go.git" ]; then + echo "This script must be ran with working copy of apache/arrow-go." + echo "The origin's URL: ${git_origin_url}" + exit 1 +fi + +tag="v${version}" +rc_tag="${tag}-rc${rc}" +echo "Tagging for release: ${tag}" +git tag "${tag}" "${rc_tag}" +git push origin "${tag}" + +dist_url="https://dist.apache.org/repos/dist/release/arrow" +dist_dir="dev/release/dist" +echo "Checking out ${dist_url}" +rm -rf "${dist_dir}" +svn co --depth=empty "${dist_url}" "${dist_dir}" +gh release download "${rc_tag}" \ + --repo "${repository}" \ + --dir "${dist_dir}" \ + --skip-existing + +release_id="apache-arrow-go-${version}" +echo "Uploading to release/" +pushd "${dist_dir}" +svn add . +svn ci -m "Apache Arrow Go ${version}" +popd +rm -rf "${dist_dir}" + +echo "Keep only the latest versions" +old_releases=$( + svn ls https://dist.apache.org/repos/dist/release/arrow/ | + grep -E '^apache-arrow-go-' | + sort --version-sort --reverse | + tail -n +2 +) +for old_release_version in ${old_releases}; do + echo "Remove old release ${old_release_version}" + svn \ + delete \ + -m "Remove old Apache Arrow Go release: ${old_release_version}" \ + "https://dist.apache.org/repos/dist/release/arrow/${old_release_version}" +done + +echo "Success! The release is available here:" +echo " https://dist.apache.org/repos/dist/release/arrow/${release_id}" +echo +echo "Add this release to ASF's report database:" +echo " https://reporter.apache.org/addrelease.html?arrow" diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh new file mode 100755 index 00000000..a66de0d2 --- /dev/null +++ b/dev/release/release_rc.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo " e.g.: $0 1" + exit 1 +fi + +rc=$1 + +: "${RELEASE_DEFAULT:=1}" +: "${RELEASE_PULL:=${RELEASE_DEFAULT}}" +: "${RELEASE_PUSH_TAG:=${RELEASE_DEFAULT}}" +: "${RELEASE_SIGN:=${RELEASE_DEFAULT}}" +: "${RELEASE_UPLOAD:=${RELEASE_DEFAULT}}" + +cd "${SOURCE_TOP_DIR}" + +if [ "${RELEASE_PULL}" -gt 0 ] || [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then + git_origin_url="$(git remote get-url origin)" + if [ "${git_origin_url}" != "git@github.com:apache/arrow-go.git" ]; then + echo "This script must be ran with working copy of apache/arrow-go." + echo "The origin's URL: ${git_origin_url}" + exit 1 + fi +fi + +if [ "${RELEASE_PULL}" -gt 0 ]; then + echo "Ensure using the latest commit" + git checkout main + git pull --rebase --prune +fi + +version=$(grep -o '^const PkgVersion = ".*"' "arrow/doc.go" | + sed -e 's/^const PkgVersion = "//' \ + -e 's/"$//') + +rc_tag="v${version}-rc${rc}" +if [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then + echo "Tagging for RC: ${rc_tag}" + git tag -a -m "${version} RC${rc}" "${rc_tag}" + git push origin "${rc_tag}" +fi + +rc_hash="$(git rev-list --max-count=1 "${rc_tag}")" + +id="apache-arrow-go-${version}" +tar_gz="${id}.tar.gz" + +if [ "${RELEASE_SIGN}" -gt 0 ]; then + git_origin_url="$(git remote get-url origin)" + repository="${git_origin_url#*github.com?}" + repository="${repository%.git}" + + echo "Looking for GitHub Actions workflow on ${repository}:${rc_tag}" + run_id="" + while [ -z "${run_id}" ]; do + echo "Waiting for run to start..." + run_id=$(gh run list \ + --repo "${repository}" \ + --workflow=rc.yml \ + --json 'databaseId,event,headBranch,status' \ + --jq ".[] | select(.event == \"push\" and .headBranch == \"${rc_tag}\") | .databaseId") + sleep 1 + done + + echo "Found GitHub Actions workflow with ID: ${run_id}" + gh run watch --repo "${repository}" --exit-status "${run_id}" + + echo "Downloading .tar.gz from GitHub Releases" + gh release download "${rc_tag}" \ + --dir . \ + --pattern "${tar_gz}" \ + --repo "${repository}" \ + --skip-existing + + echo "Signing tar.gz and creating checksums" + gpg --armor --output "${tar_gz}.asc" --detach-sig "${tar_gz}" +fi + +if [ "${RELEASE_UPLOAD}" -gt 0 ]; then + echo "Uploading signature" + gh release upload "${rc_tag}" \ + --clobber \ + --repo "${repository}" \ + "${tar_gz}.asc" +fi + +echo "Draft email for dev@arrow.apache.org mailing list" +echo "" +echo "---------------------------------------------------------" +cat < " + echo " e.g.: $0 18.0.0 1" + exit 1 +fi + +set -o pipefail +set -x + +VERSION="$1" +RC="$2" + +ARROW_DIST_BASE_URL="https://dist.apache.org/repos/dist/dev/arrow" +DOWNLOAD_RC_BASE_URL="https://github.com/apache/arrow-go/releases/download/v${VERSION}-rc${RC}" +ARCHIVE_BASE_NAME="apache-arrow-go-${VERSION}" + +: "${VERIFY_DEFAULT:=1}" +: "${VERIFY_DOWNLOAD:=${VERIFY_DEFAULT}}" +: "${VERIFY_FORCE_USE_GO_BINARY:=0}" +: "${VERIFY_SIGN:=${VERIFY_DEFAULT}}" + +VERIFY_SUCCESS=no + +setup_tmpdir() { + cleanup() { + go clean -modcache || : + if [ "${VERIFY_SUCCESS}" = "yes" ]; then + rm -rf "${VERIFY_TMPDIR}" + else + echo "Failed to verify release candidate. See ${VERIFY_TMPDIR} for details." + fi + } + + if [ -z "${VERIFY_TMPDIR:-}" ]; then + VERIFY_TMPDIR="$(mktemp -d -t "$1.XXXXX")" + trap cleanup EXIT + else + mkdir -p "${VERIFY_TMPDIR}" + fi +} + +download() { + curl \ + --fail \ + --location \ + --remote-name \ + --show-error \ + --silent \ + "$1" +} + +download_rc_file() { + if [ "${VERIFY_DOWNLOAD}" -gt 0 ]; then + download "${DOWNLOAD_RC_BASE_URL}/$1" + else + cp "${TOP_SOURCE_DIR}/$1" "$1" + fi +} + +import_gpg_keys() { + if [ "${VERIFY_SIGN}" -gt 0 ]; then + download "${ARROW_DIST_BASE_URL}/KEYS" + gpg --import KEYS + fi +} + +if type shasum >/dev/null 2>&1; then + sha256_verify="shasum -a 256 -c" + sha512_verify="shasum -a 512 -c" +else + sha256_verify="sha256sum -c" + sha512_verify="sha512sum -c" +fi + +fetch_archive() { + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz" + if [ "${VERIFY_SIGN}" -gt 0 ]; then + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.asc" + gpg --verify "${ARCHIVE_BASE_NAME}.tar.gz.asc" "${ARCHIVE_BASE_NAME}.tar.gz" + fi + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.sha256" + ${sha256_verify} "${ARCHIVE_BASE_NAME}.tar.gz.sha256" + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.sha512" + ${sha512_verify} "${ARCHIVE_BASE_NAME}.tar.gz.sha512" +} + +ensure_source_directory() { + tar xf "${ARCHIVE_BASE_NAME}".tar.gz + + if [ -d "${TOP_SOURCE_DIR}/arrow-testing/data" ]; then + cp -a "${TOP_SOURCE_DIR}/arrow-testing" "${ARCHIVE_BASE_NAME}/" + else + git clone \ + https://github.com/apache/arrow-testing.git \ + "${ARCHIVE_BASE_NAME}/arrow-testing" + fi + if [ -d "${TOP_SOURCE_DIR}/parquet-testing/data" ]; then + cp -a "${TOP_SOURCE_DIR}/parquet-testing" "${ARCHIVE_BASE_NAME}/" + else + git clone \ + https://github.com/apache/parquet-testing.git \ + "${ARCHIVE_BASE_NAME}/parquet-testing" + fi + + ARROW_TEST_DATA="${ARCHIVE_BASE_NAME}/arrow-testing/data" + export ARROW_TEST_DATA + PARQUET_TEST_DATA="${ARCHIVE_BASE_NAME}/parquet-testing/data" + export PARQUET_TEST_DATA +} + +latest_go_version() { + local -a options + options=( + --fail + --location + --show-error + --silent + ) + if [ -n "${GITHUB_TOKEN:-}" ]; then + options+=("--header" "Authorization: Bearer ${GITHUB_TOKEN}") + fi + curl \ + "${options[@]}" \ + https://api.github.com/repos/golang/go/git/matching-refs/tags/go | + grep -o '"ref": "refs/tags/go.*"' | + tail -n 1 | + sed -e 's,^"ref": "refs/tags/go,,g' \ + -e 's/"$//g' +} + +ensure_go() { + if [ "${VERIFY_FORCE_USE_GO_BINARY}" -le 0 ]; then + if go version; then + GOPATH="${VERIFY_TMPDIR}/gopath" + export GOPATH + mkdir -p "${GOPATH}" + return + fi + fi + + local go_version + go_version=$(latest_go_version) + local go_os + go_os="$(uname)" + case "${go_os}" in + Darwin) + go_os="darwin" + ;; + Linux) + go_os="linux" + ;; + esac + local go_arch + go_arch="$(arch)" + case "${go_arch}" in + i386 | x86_64) + go_arch="amd64" + ;; + aarch64) + go_arch="arm64" + ;; + esac + local go_binary_tar_gz + go_binary_tar_gz="go${go_version}.${go_os}-${go_arch}.tar.gz" + local go_binary_url + go_binary_url="https://go.dev/dl/${go_binary_tar_gz}" + curl \ + --fail \ + --location \ + --output "${go_binary_tar_gz}" \ + --show-error \ + --silent \ + "${go_binary_url}" + tar xf "${go_binary_tar_gz}" + GOROOT="$(pwd)/go" + export GOROOT + GOPATH="$(pwd)/gopath" + export GOPATH + mkdir -p "${GOPATH}" + PATH="${GOROOT}/bin:${GOPATH}/bin:${PATH}" +} + +test_source_distribution() { + "${TOP_SOURCE_DIR}/ci/scripts/build.sh" "$(pwd)" + "${TOP_SOURCE_DIR}/ci/scripts/test.sh" "$(pwd)" + # TODO: Integration test +} + +setup_tmpdir "arrow-go-${VERSION}-${RC}" +echo "Working in sandbox ${VERIFY_TMPDIR}" +cd "${VERIFY_TMPDIR}" + +import_gpg_keys +fetch_archive +ensure_source_directory +ensure_go +pushd "${ARCHIVE_BASE_NAME}" +test_source_distribution +popd + +VERIFY_SUCCESS=yes +echo "RC looks good!" From 8e4c9bdb364c92da2e447399eb7589e5c5916435 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Sep 2024 21:16:33 +0900 Subject: [PATCH 2/6] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Raúl Cumplido --- dev/release/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index 310d3903..1c1ed020 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -28,7 +28,7 @@ ### Prepare RC and vote -Run `dev/release/release_rc.sh` on working copy of `git@github.com:apache/arrow-go` not your fork: +Run `dev/release/release_rc.sh` on a working copy of `git@github.com:apache/arrow-go` not your fork: ```console $ git clone git@github.com:apache/arrow-go.git @@ -83,7 +83,7 @@ $ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh ${VERSION} ${RC} Here is an example to release 18.0.0 RC1: ```console -$ GH_TOKEN=${YOUR_GITHUB_TOKEN}dev/release/release.sh 18.0.0 1 +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh 18.0.0 1 ``` Add the release to ASF's report database via [Apache Committee Report Helper](https://reporter.apache.org/addrelease.html?arrow). @@ -107,10 +107,10 @@ To verify a RC, run the following command line: $ dev/release/verify_rc.sh ${VERSION} ${RC} ``` -Here is an example to release 18.0.0 RC1: +Here is an example to verify the release 18.0.0 RC1: ```console $ dev/release/verify_rc.sh 18.0.0 1 ``` -If the verification is succeeded, `RC looks good!` is shown. +If the verification is successful, the message `RC looks good!` is shown. From a0e1de0d23f21a4caa87109e974cd8089252113f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Sep 2024 21:18:23 +0900 Subject: [PATCH 3/6] Improve style --- .github/workflows/rc.yml | 3 ++- dev/release/release_rc.sh | 3 ++- dev/release/verify_rc.sh | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index acd34ace..c14ccf8b 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -51,7 +51,8 @@ jobs: if: github.ref_type == 'branch' run: | version=$(grep -o '^const PkgVersion = ".*"' "arrow/doc.go" | - sed -e 's/^const PkgVersion = "//' \ + sed \ + -e 's/^const PkgVersion = "//' \ -e 's/"$//') rc=100 echo "VERSION=${version}" >> ${GITHUB_ENV} diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index a66de0d2..b466050a 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -54,7 +54,8 @@ if [ "${RELEASE_PULL}" -gt 0 ]; then fi version=$(grep -o '^const PkgVersion = ".*"' "arrow/doc.go" | - sed -e 's/^const PkgVersion = "//' \ + sed \ + -e 's/^const PkgVersion = "//' \ -e 's/"$//') rc_tag="v${version}-rc${rc}" diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index 62cf8751..b742a90a 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -148,7 +148,8 @@ latest_go_version() { https://api.github.com/repos/golang/go/git/matching-refs/tags/go | grep -o '"ref": "refs/tags/go.*"' | tail -n 1 | - sed -e 's,^"ref": "refs/tags/go,,g' \ + sed \ + -e 's,^"ref": "refs/tags/go,,g' \ -e 's/"$//g' } From af2470bcc431648fbee62e43780cb0291b344627 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Sep 2024 21:21:11 +0900 Subject: [PATCH 4/6] Update ignores --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 55232367..ec2e3422 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ # under the License. .vscode +/apache-arrow-go-*.tar.gz +/apache-arrow-go-*.tar.gz.asc /apache-arrow-go.tar.gz /dev/release/apache-rat-*.jar /dev/release/filtered_rat.txt From c15bec78545f9b9ff75cec83433ddd63ba82c708 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 14 Sep 2024 06:12:17 +0900 Subject: [PATCH 5/6] Add PARQUET_TEST_BAD_DATA --- dev/release/verify_rc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index b742a90a..247f3065 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -130,6 +130,8 @@ ensure_source_directory() { export ARROW_TEST_DATA PARQUET_TEST_DATA="${ARCHIVE_BASE_NAME}/parquet-testing/data" export PARQUET_TEST_DATA + PARQUET_TEST_BAD_DATA="${ARCHIVE_BASE_NAME}/parquet-testing/bad_data" + export PARQUET_TEST_BAD_DATA } latest_go_version() { From 293a07c67551c4465693fad3aa4a673c391a3841 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 17 Sep 2024 10:42:09 +0900 Subject: [PATCH 6/6] Use today instead of "100" for RC --- .github/workflows/rc.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index c14ccf8b..eece3b77 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -54,7 +54,7 @@ jobs: sed \ -e 's/^const PkgVersion = "//' \ -e 's/"$//') - rc=100 + rc=$(date +%Y%m%d) echo "VERSION=${version}" >> ${GITHUB_ENV} echo "RC=${rc}" >> ${GITHUB_ENV} - name: Archive @@ -97,10 +97,11 @@ jobs: tar_gz=$(echo apache-arrow-go-*.tar.gz) version=${tar_gz#apache-arrow-go-} version=${version%.tar.gz} + # rc isn't used with VERIFY_DOWNLOAD=0 if [ "${GITHUB_REF_TYPE}" = "tag" ]; then rc="${GITHUB_REF_NAME#*-rc}" else - rc=100 + rc=$(date +%Y%m%d) fi VERIFY_DEFAULT=0 dev/release/verify_rc.sh "${version}" "${rc}" env: