From d8c67543b69e02a754dccb95d01492cc3c3c45f7 Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Fri, 12 Jul 2019 14:12:13 -0700 Subject: [PATCH 1/6] Add get_contributors to release process --- dev/RELEASE_PROCESS.rst | 31 ++++++++++++------- dev/get_contributors.py | 66 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 11 deletions(-) create mode 100644 dev/get_contributors.py diff --git a/dev/RELEASE_PROCESS.rst b/dev/RELEASE_PROCESS.rst index 3b78cef5eda53..9dc35c43c3f13 100644 --- a/dev/RELEASE_PROCESS.rst +++ b/dev/RELEASE_PROCESS.rst @@ -37,6 +37,8 @@ This document describes the process for creating new releases. and make sure they pass. If they pass, it will be obvious that they passed. This will use the autoscaler to start a bunch of machines and run some tests. + Stress test will use significant aws resource. If you don't have enough quota, + please contact Robert for account. 5. **Resolve release-blockers:** If a release blocking issue arises, there are two ways the issue can be resolved: 1) Fix the issue on the master branch and @@ -63,13 +65,7 @@ This document describes the process for creating new releases. pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_6_intel.whl pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_6_intel.whl -7. **Final Testing:** Send a link to the wheels to the other contributors and - core members of the Ray project. Make sure the wheels are tested on Ubuntu - and MacOS (ideally multiple versions of Ubuntu and MacOS). This testing - should verify that the wheels are correct and that all release blockers have - been resolved. Should a new release blocker be found, repeat steps 5-7. - -8. **Upload to PyPI Test:** Upload the wheels to the PyPI test site using +7. **Upload to PyPI Test:** Upload the wheels to the PyPI test site using ``twine`` (ask Robert to add you as a maintainer to the PyPI project). You'll need to run a command like @@ -95,7 +91,7 @@ This document describes the process for creating new releases. Do this at least for MacOS and for Linux, as well as for Python 2 and Python 3. -9. **Upload to PyPI:** Now that you've tested the wheels on the PyPI test +8. **Upload to PyPI:** Now that you've tested the wheels on the PyPI test repository, they can be uploaded to the main PyPI repository. Be careful, **it will not be possible to modify wheels once you upload them**, so any mistake will require a new release. You can upload the wheels with a command @@ -114,7 +110,7 @@ This document describes the process for creating new releases. finds the correct Ray version, and successfully runs some simple scripts on both MacOS and Linux as well as Python 2 and Python 3. -10. **Create a GitHub release:** Create a GitHub release through the +9. **Create a GitHub release:** Create a GitHub release through the `GitHub website`_. The release should be created at the commit from the previous step. This should include **release notes**. Copy the style and formatting used by previous releases. Create a draft of the release notes @@ -127,7 +123,20 @@ This document describes the process for creating new releases. git pull origin master --tags git log $(git describe --tags --abbrev=0)..HEAD --pretty=format:"%s" | sort -11. **Update version numbers throughout codebase:** Suppose we just released + + At the end of the release note, you can add a list of contributors that help + creating this release. Use the ``dev/get_contributors.py`` to generate this + list. You will need to create a github token for this task. Example usage: + + .. code-block:: bash + python get_contributors.py --help + python get_contributors.py \ + --access-token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx \ + --ray-path=$HOME/ray/ray \ + --prev-branch="ray-0.7.1" \ + --curr-branch="ray-0.7.2" + +10. **Update version numbers throughout codebase:** Suppose we just released 0.7.1. The previous release version number (in this case 0.7.0) and the previous dev version number (in this case 0.8.0.dev0) appear in many places throughout the code base including the installation documentation, the @@ -137,7 +146,7 @@ This document describes the process for creating new releases. should be replaced. For example, ``0.7.0`` appears in this file but should not be updated. -12. **Improve the release process:** Find some way to improve the release +11. **Improve the release process:** Find some way to improve the release process so that whoever manages the release next will have an easier time. .. _`this example`: https://github.com/ray-project/ray/pull/4226 diff --git a/dev/get_contributors.py b/dev/get_contributors.py new file mode 100644 index 0000000000000..da349c4451acc --- /dev/null +++ b/dev/get_contributors.py @@ -0,0 +1,66 @@ +from github import Github +from subprocess import check_output +import shlex +from tqdm import tqdm +import click + + +@click.command() +@click.option( + "--access-token", + required=True, + help=""" +Github Access token that has repo:public_repo and user:read:user permission. + +Create them at https://github.com/settings/tokens/new +""", +) +@click.option( + "--ray-path", required=True, help="File system path to the ray directory") +@click.option( + "--prev-branch", + required=True, + help="Previous version branch like ray-0.7.1") +@click.option( + "--curr-branch", + required=True, + help="Current version branch like ray-0.7.2") +def run(access_token, ray_path, prev_branch, curr_branch): + # Generate command + cmd = [] + cmd.append(f"cd {ray_path}") + cmd.append(f'git log {prev_branch}..{curr_branch} --pretty=format:"%s" ' + ' | grep -Eo "#(\d+)"') + joined = " && ".join(cmd) + cmd = f"bash -c '{joined}'" + cmd = shlex.split(cmd) + print("Executing", cmd) + + # Sort the PR numbers + pr_numbers = [ + int(l.lstrip("#")) for l in check_output(cmd).decode().split() + ] + print("PR numbers", pr_numbers) + + # Use Github API to fetch the + g = Github(access_token) + ray_repo = g.get_repo("ray-project/ray") + logins = set() + for num in tqdm(pr_numbers): + try: + logins.add(ray_repo.get_pull(num).user.login) + except Exception as e: + print(e) + pass + + print() + print("Here's the list of contributors") + print("=" * 10) + print() + print("@" + ", @".join(logins)) + print() + print("=" * 10) + + +if __name__ == "__main__": + run() From 34307a5fa7d0ab19d40c3f2f9df36d09bf04fb0f Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Fri, 12 Jul 2019 14:13:57 -0700 Subject: [PATCH 2/6] Remove sgd tests; test impala in p36 only --- ci/stress_tests/run_application_stress_tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/stress_tests/run_application_stress_tests.sh b/ci/stress_tests/run_application_stress_tests.sh index 2935309287458..5951385bb00a8 100755 --- a/ci/stress_tests/run_application_stress_tests.sh +++ b/ci/stress_tests/run_application_stress_tests.sh @@ -148,10 +148,9 @@ test_sgd(){ } # RUN TESTS -for PYTHON_VERSION in "p27" "p36" +for PYTHON_VERSION in "p36" do test_impala $PYTHON_VERSION - test_sgd $PYTHON_VERSION done cat "$RESULT_FILE" From bd1c0abd9a5cb7ea84cf006d3dd1149c051fe6c0 Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Fri, 12 Jul 2019 17:39:35 -0700 Subject: [PATCH 3/6] Update dev/get_contributors.py Co-Authored-By: Richard Liaw --- dev/get_contributors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/get_contributors.py b/dev/get_contributors.py index da349c4451acc..82399940b3df8 100644 --- a/dev/get_contributors.py +++ b/dev/get_contributors.py @@ -51,7 +51,6 @@ def run(access_token, ray_path, prev_branch, curr_branch): logins.add(ray_repo.get_pull(num).user.login) except Exception as e: print(e) - pass print() print("Here's the list of contributors") From 9b8350c5050ce881b8cbce455b7c19cf01761b4b Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Fri, 12 Jul 2019 17:39:47 -0700 Subject: [PATCH 4/6] Update dev/RELEASE_PROCESS.rst Co-Authored-By: Richard Liaw --- dev/RELEASE_PROCESS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/RELEASE_PROCESS.rst b/dev/RELEASE_PROCESS.rst index 9dc35c43c3f13..1a09db05dc900 100644 --- a/dev/RELEASE_PROCESS.rst +++ b/dev/RELEASE_PROCESS.rst @@ -37,7 +37,7 @@ This document describes the process for creating new releases. and make sure they pass. If they pass, it will be obvious that they passed. This will use the autoscaler to start a bunch of machines and run some tests. - Stress test will use significant aws resource. If you don't have enough quota, + **Caution!**: By default, the stress tests will require expensive GPU instances. please contact Robert for account. 5. **Resolve release-blockers:** If a release blocking issue arises, there are From 43cd20b49ab3e4e6aea577423190a9982559e36f Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Fri, 12 Jul 2019 17:40:15 -0700 Subject: [PATCH 5/6] Update dev/RELEASE_PROCESS.rst Co-Authored-By: Richard Liaw --- dev/RELEASE_PROCESS.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/RELEASE_PROCESS.rst b/dev/RELEASE_PROCESS.rst index 1a09db05dc900..56a00d2da5424 100644 --- a/dev/RELEASE_PROCESS.rst +++ b/dev/RELEASE_PROCESS.rst @@ -38,7 +38,6 @@ This document describes the process for creating new releases. and make sure they pass. If they pass, it will be obvious that they passed. This will use the autoscaler to start a bunch of machines and run some tests. **Caution!**: By default, the stress tests will require expensive GPU instances. - please contact Robert for account. 5. **Resolve release-blockers:** If a release blocking issue arises, there are two ways the issue can be resolved: 1) Fix the issue on the master branch and From 5e245867ab99d2bb097dadadb76854deacefa4fa Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Fri, 12 Jul 2019 18:11:02 -0700 Subject: [PATCH 6/6] Update dev/RELEASE_PROCESS.rst Co-Authored-By: Richard Liaw --- dev/RELEASE_PROCESS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/RELEASE_PROCESS.rst b/dev/RELEASE_PROCESS.rst index 56a00d2da5424..a940fef3cf8c4 100644 --- a/dev/RELEASE_PROCESS.rst +++ b/dev/RELEASE_PROCESS.rst @@ -125,7 +125,7 @@ This document describes the process for creating new releases. At the end of the release note, you can add a list of contributors that help creating this release. Use the ``dev/get_contributors.py`` to generate this - list. You will need to create a github token for this task. Example usage: + list. You will need to create a GitHub token for this task. Example usage: .. code-block:: bash python get_contributors.py --help