From c6bdc9fecc4e8e28b055af28887f8cd7c19378ce Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 9 May 2024 12:17:39 -0600 Subject: [PATCH] chore: Update Python release process now that DataFusion is TLP (#674) --- dev/release/README.md | 59 +++---------- dev/release/create-tarball.sh | 34 ++++---- dev/release/generate-changelog.py | 2 +- dev/release/release-tarball.sh | 12 +-- .../update_change_log-datafusion-python.sh | 33 ------- dev/release/update_change_log.sh | 87 ------------------- dev/release/verify-release-candidate.sh | 28 +++--- 7 files changed, 50 insertions(+), 205 deletions(-) delete mode 100755 dev/release/update_change_log-datafusion-python.sh delete mode 100755 dev/release/update_change_log.sh diff --git a/dev/release/README.md b/dev/release/README.md index 6bd2c1eb2..c4372c832 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -63,7 +63,7 @@ We maintain a `CHANGELOG.md` so our users know what has been changed between rel The changelog is generated using a Python script: ```bash -$ GITHUB_TOKEN= ./dev/release/generate-changelog.py apache/arrow-datafusion-python 24.0.0 HEAD > dev/changelog/25.0.0.md +$ GITHUB_TOKEN= ./dev/release/generate-changelog.py apache/datafusion-python 24.0.0 HEAD > dev/changelog/25.0.0.md ``` This script creates a changelog from GitHub PRs based on the labels associated with them as well as looking for @@ -83,9 +83,9 @@ This process is not fully automated, so there are some additional manual steps: - Add the following content (copy from the previous version's changelog and update as appropriate: ``` -## [24.0.0](https://github.com/apache/arrow-datafusion-python/tree/24.0.0) (2023-05-06) +## [24.0.0](https://github.com/apache/datafusion-python/tree/24.0.0) (2023-05-06) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/23.0.0...24.0.0) +[Full Changelog](https://github.com/apache/datafusion-python/compare/23.0.0...24.0.0) ``` ### Preparing a Release Candidate @@ -103,42 +103,7 @@ git push apache 0.8.0-rc1 ./dev/release/create-tarball.sh 0.8.0 1 ``` -This will also create the email template to send to the mailing list. Here is an example: - -``` -To: dev@arrow.apache.org -Subject: [VOTE][RUST][DataFusion] Release DataFusion Python Bindings 0.7.0 RC2 -Hi, - -I would like to propose a release of Apache Arrow DataFusion Python Bindings, -version 0.7.0. - -This release candidate is based on commit: bd1b78b6d444b7ab172c6aec23fa58c842a592d7 [1] -The proposed release tarball and signatures are hosted at [2]. -The changelog is located at [3]. -The Python wheels are located at [4]. - -Please download, verify checksums and signatures, run the unit tests, and vote -on the release. The vote will be open for at least 72 hours. - -Only votes from PMC members are binding, but all members of the community are -encouraged to test the release and vote with "(non-binding)". - -The standard verification procedure is documented at https://github.com/apache/arrow-datafusion-python/blob/main/dev/release/README.md#verifying-release-candidates. - -[ ] +1 Release this as Apache Arrow DataFusion Python 0.7.0 -[ ] +0 -[ ] -1 Do not release this as Apache Arrow DataFusion Python 0.7.0 because... - -Here is my vote: - -+1 - -[1]: https://github.com/apache/arrow-datafusion-python/tree/bd1b78b6d444b7ab172c6aec23fa58c842a592d7 -[2]: https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-datafusion-python-0.7.0-rc2 -[3]: https://github.com/apache/arrow-datafusion-python/blob/bd1b78b6d444b7ab172c6aec23fa58c842a592d7/CHANGELOG.md -[4]: https://test.pypi.org/project/datafusion/0.7.0/ -``` +This will also create the email template to send to the mailing list. Create a draft email using this content, but do not send until after completing the next step. @@ -151,7 +116,7 @@ This section assumes some familiarity with publishing Python packages to PyPi. F Pushing an `rc` tag to the release branch will cause a GitHub Workflow to run that will build the Python wheels. -Go to https://github.com/apache/arrow-datafusion-python/actions and look for an action named "Python Release Build" +Go to https://github.com/apache/datafusion-python/actions and look for an action named "Python Release Build" that has run against the pushed tag. Click on the action and scroll down to the bottom of the page titled "Artifacts". Download `dist.zip`. It should @@ -266,10 +231,10 @@ git push apache 0.8.0 ### Add the release to Apache Reporter -Add the release to https://reporter.apache.org/addrelease.html?arrow with a version name prefixed with `RS-DATAFUSION-PYTHON`, -for example `RS-DATAFUSION-PYTHON-31.0.0`. +Add the release to https://reporter.apache.org/addrelease.html?datafusion with a version name prefixed with `DATAFUSION-PYTHON`, +for example `DATAFUSION-PYTHON-31.0.0`. -The release information is used to generate a template for a board report (see example +The release information is used to generate a template for a board report (see example from Apache Arrow [here](https://github.com/apache/arrow/pull/14357)). ### Delete old RCs and Releases @@ -284,13 +249,13 @@ Release candidates should be deleted once the release is published. Get a list of DataFusion release candidates: ```bash -svn ls https://dist.apache.org/repos/dist/dev/arrow | grep datafusion-python +svn ls https://dist.apache.org/repos/dist/dev/datafusion | grep datafusion-python ``` Delete a release candidate: ```bash -svn delete -m "delete old DataFusion RC" https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-datafusion-python-7.1.0-rc1/ +svn delete -m "delete old DataFusion RC" https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-python-7.1.0-rc1/ ``` #### Deleting old releases from `release` svn @@ -300,11 +265,11 @@ Only the latest release should be available. Delete old releases after publishin Get a list of DataFusion releases: ```bash -svn ls https://dist.apache.org/repos/dist/release/arrow | grep datafusion-python +svn ls https://dist.apache.org/repos/dist/release/datafusion | grep datafusion-python ``` Delete a release: ```bash -svn delete -m "delete old DataFusion release" https://dist.apache.org/repos/dist/release/arrow/arrow-datafusion-python-7.0.0 +svn delete -m "delete old DataFusion release" https://dist.apache.org/repos/dist/release/datafusion/datafusion-python-7.0.0 ``` diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh index c05da5b75..d6ca76561 100755 --- a/dev/release/create-tarball.sh +++ b/dev/release/create-tarball.sh @@ -21,9 +21,9 @@ # Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/create-tarball.sh # This script creates a signed tarball in -# dev/dist/apache-arrow-datafusion-python--.tar.gz and uploads it to +# dev/dist/apache-datafusion-python--.tar.gz and uploads it to # the "dev" area of the dist.apache.arrow repository and prepares an -# email for sending to the dev@arrow.apache.org list for a formal +# email for sending to the dev@datafusion.apache.org list for a formal # vote. # # See release/README.md for full release instructions @@ -65,25 +65,25 @@ tag="${version}-rc${rc}" echo "Attempting to create ${tarball} from tag ${tag}" release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag}) -release=apache-arrow-datafusion-python-${version} +release=apache-datafusion-python-${version} distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc} tarname=${release}.tar.gz tarball=${distdir}/${tarname} -url="https://dist.apache.org/repos/dist/dev/arrow/${release}-rc${rc}" +url="https://dist.apache.org/repos/dist/dev/datafusion/${release}-rc${rc}" if [ -z "$release_hash" ]; then echo "Cannot continue: unknown git tag: ${tag}" fi -echo "Draft email for dev@arrow.apache.org mailing list" +echo "Draft email for dev@datafusion.apache.org mailing list" echo "" echo "---------------------------------------------------------" cat < ${tarball}.sha256 (cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512 -echo "Uploading to apache dist/dev to ${url}" -svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow ${SOURCE_TOP_DIR}/dev/dist +echo "Uploading to datafusion dist/dev to ${url}" +svn co --depth=empty https://dist.apache.org/repos/dist/dev/datafusion ${SOURCE_TOP_DIR}/dev/dist svn add ${distdir} -svn ci -m "Apache Arrow DataFusion Python ${version} ${rc}" ${distdir} +svn ci -m "Apache DataFusion Python ${version} ${rc}" ${distdir} diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index 01d640669..af097ce98 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -102,7 +102,7 @@ def cli(args=None): parser = argparse.ArgumentParser() parser.add_argument( - "project", help="The project name e.g. apache/arrow-datafusion-python" + "project", help="The project name e.g. apache/datafusion-python" ) parser.add_argument("tag1", help="The previous release tag") parser.add_argument("tag2", help="The current release tag") diff --git a/dev/release/release-tarball.sh b/dev/release/release-tarball.sh index f5e8eb1bf..8c305a676 100755 --- a/dev/release/release-tarball.sh +++ b/dev/release/release-tarball.sh @@ -43,7 +43,7 @@ fi version=$1 rc=$2 -tmp_dir=tmp-apache-arrow-datafusion-python-dist +tmp_dir=tmp-apache-datafusion-python-dist echo "Recreate temporary directory: ${tmp_dir}" rm -rf ${tmp_dir} @@ -52,23 +52,23 @@ mkdir -p ${tmp_dir} echo "Clone dev dist repository" svn \ co \ - https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-datafusion-python-${version}-rc${rc} \ + https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-python-${version}-rc${rc} \ ${tmp_dir}/dev echo "Clone release dist repository" -svn co https://dist.apache.org/repos/dist/release/arrow ${tmp_dir}/release +svn co https://dist.apache.org/repos/dist/release/datafusion ${tmp_dir}/release echo "Copy ${version}-rc${rc} to release working copy" -release_version=arrow-datafusion-python-${version} +release_version=datafusion-python-${version} mkdir -p ${tmp_dir}/release/${release_version} cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/ svn add ${tmp_dir}/release/${release_version} echo "Commit release" -svn ci -m "Apache Arrow DataFusion Python ${version}" ${tmp_dir}/release +svn ci -m "Apache DataFusion Python ${version}" ${tmp_dir}/release echo "Clean up" rm -rf ${tmp_dir} echo "Success! The release is available here:" -echo " https://dist.apache.org/repos/dist/release/arrow/${release_version}" +echo " https://dist.apache.org/repos/dist/release/datafusion/${release_version}" diff --git a/dev/release/update_change_log-datafusion-python.sh b/dev/release/update_change_log-datafusion-python.sh deleted file mode 100755 index a11447f0b..000000000 --- a/dev/release/update_change_log-datafusion-python.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Usage: -# CHANGELOG_GITHUB_TOKEN= ./update_change_log-datafusion.sh main 8.0.0 7.1.0 -# CHANGELOG_GITHUB_TOKEN= ./update_change_log-datafusion.sh maint-7.x 7.1.0 7.0.0 - -RELEASE_BRANCH=$1 -RELEASE_TAG=$2 -BASE_TAG=$3 - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -${SOURCE_DIR}/update_change_log.sh \ - "${BASE_TAG}" \ - --future-release "${RELEASE_TAG}" \ - --release-branch "${RELEASE_BRANCH}" diff --git a/dev/release/update_change_log.sh b/dev/release/update_change_log.sh deleted file mode 100755 index a0b398131..000000000 --- a/dev/release/update_change_log.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/update_change_log.sh - -# invokes the changelog generator from -# https://github.com/github-changelog-generator/github-changelog-generator -# -# With the config located in -# arrow-datafusion/.github_changelog_generator -# -# Usage: -# CHANGELOG_GITHUB_TOKEN= ./update_change_log.sh - -set -e - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" - -if [[ "$#" -lt 1 ]]; then - echo "USAGE: $0 SINCE_TAG EXTRA_ARGS..." - exit 1 -fi - -SINCE_TAG=$1 -shift 1 - -OUTPUT_PATH="CHANGELOG.md" - -pushd ${SOURCE_TOP_DIR} - -# reset content in changelog -git checkout "${SINCE_TAG}" "${OUTPUT_PATH}" -# remove license header so github-changelog-generator has a clean base to append -sed -i.bak '1,18d' "${OUTPUT_PATH}" - -docker run -it --rm \ - --cpus "0.1" \ - -e CHANGELOG_GITHUB_TOKEN=$CHANGELOG_GITHUB_TOKEN \ - -v "$(pwd)":/usr/local/src/your-app \ - githubchangeloggenerator/github-changelog-generator \ - --user apache \ - --project arrow-datafusion-python \ - --since-tag "${SINCE_TAG}" \ - --base "${OUTPUT_PATH}" \ - --output "${OUTPUT_PATH}" \ - "$@" - -sed -i.bak "s/\\\n/\n\n/" "${OUTPUT_PATH}" - -echo ' -' | cat - "${OUTPUT_PATH}" > "${OUTPUT_PATH}".tmp -mv "${OUTPUT_PATH}".tmp "${OUTPUT_PATH}" diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index be86f69e0..14c0baee8 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -32,8 +32,8 @@ set -x set -o pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" -ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))" -ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow' +DATAFUSION_PYTHON_DIR="$(dirname $(dirname ${SOURCE_DIR}))" +DATAFUSION_PYTHON_DIST_URL='https://dist.apache.org/repos/dist/dev/datafusion' download_dist_file() { curl \ @@ -41,11 +41,11 @@ download_dist_file() { --show-error \ --fail \ --location \ - --remote-name $ARROW_DIST_URL/$1 + --remote-name $DATAFUSION_PYTHON_DIST_URL/$1 } download_rc_file() { - download_dist_file apache-arrow-datafusion-python-${VERSION}-rc${RC_NUMBER}/$1 + download_dist_file apache-datafusion-python-${VERSION}-rc${RC_NUMBER}/$1 } import_gpg_keys() { @@ -89,19 +89,19 @@ verify_dir_artifact_signatures() { setup_tempdir() { cleanup() { if [ "${TEST_SUCCESS}" = "yes" ]; then - rm -fr "${ARROW_TMPDIR}" + rm -fr "${DATAFUSION_PYTHON_TMPDIR}" else - echo "Failed to verify release candidate. See ${ARROW_TMPDIR} for details." + echo "Failed to verify release candidate. See ${DATAFUSION_PYTHON_TMPDIR} for details." fi } - if [ -z "${ARROW_TMPDIR}" ]; then - # clean up automatically if ARROW_TMPDIR is not defined - ARROW_TMPDIR=$(mktemp -d -t "$1.XXXXX") + if [ -z "${DATAFUSION_PYTHON_TMPDIR}" ]; then + # clean up automatically if DATAFUSION_PYTHON_TMPDIR is not defined + DATAFUSION_PYTHON_TMPDIR=$(mktemp -d -t "$1.XXXXX") trap cleanup EXIT else # don't clean up automatically - mkdir -p "${ARROW_TMPDIR}" + mkdir -p "${DATAFUSION_PYTHON_TMPDIR}" fi } @@ -142,11 +142,11 @@ test_source_distribution() { TEST_SUCCESS=no -setup_tempdir "arrow-${VERSION}" -echo "Working in sandbox ${ARROW_TMPDIR}" -cd ${ARROW_TMPDIR} +setup_tempdir "datafusion-python-${VERSION}" +echo "Working in sandbox ${DATAFUSION_PYTHON_TMPDIR}" +cd ${DATAFUSION_PYTHON_TMPDIR} -dist_name="apache-arrow-datafusion-python-${VERSION}" +dist_name="apache-datafusion-python-${VERSION}" import_gpg_keys fetch_archive ${dist_name} tar xf ${dist_name}.tar.gz