From 155a609e70157b4a22be70bbfeeafa36588046a5 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 9 Aug 2022 14:34:51 +0100 Subject: [PATCH 01/10] Split out underlying functions so that they are callable --- scripts-dev/release.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 46220c4dd31e..4ff2762edd9e 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -90,6 +90,10 @@ def cli() -> None: @cli.command() def prepare() -> None: + _prepare() + + +def _prepare() -> None: """Do the initial stages of creating a release, including creating release branch, updating changelog and pushing to GitHub. """ @@ -284,6 +288,10 @@ def prepare() -> None: @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"]) def tag(gh_token: Optional[str]) -> None: + _tag(gh_token) + + +def _tag(gh_token: Optional[str]) -> None: """Tags the release and generates a draft GitHub release""" # Make sure we're in a git repo. @@ -374,6 +382,10 @@ def tag(gh_token: Optional[str]) -> None: @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) def publish(gh_token: str) -> None: + _publish(gh_token) + + +def _publish(gh_token: str) -> None: """Publish release on GitHub.""" # Make sure we're in a git repo. @@ -411,6 +423,10 @@ def publish(gh_token: str) -> None: @cli.command() def upload() -> None: + _upload() + + +def _upload() -> None: """Upload release to pypi.""" current_version = get_package_version() @@ -481,6 +497,10 @@ def _merge_into(repo: Repo, source: str, target: str) -> None: @cli.command() def merge_back() -> None: + _merge_back() + + +def _merge_back() -> None: """Merge the release branch back into the appropriate branches. All branches will be automatically pulled from the remote and the results will be pushed to the remote.""" @@ -519,6 +539,10 @@ def merge_back() -> None: @cli.command() def announce() -> None: + _announce() + + +def _announce() -> None: """Generate markdown to announce the release.""" current_version = get_package_version() From b7efe5d3d5ba60eb872e3cdea35febf53164f114 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 9 Aug 2022 14:35:37 +0100 Subject: [PATCH 02/10] Add a command for waiting for the actions to complete --- scripts-dev/release.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 4ff2762edd9e..8208956d49bc 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -18,10 +18,12 @@ """ import glob +import json import os import re import subprocess import sys +import time import urllib.request from os import path from tempfile import TemporaryDirectory @@ -495,6 +497,65 @@ def _merge_into(repo: Repo, source: str, target: str) -> None: repo.remote().push() +@cli.command() +def wait_for_actions() -> None: + _wait_for_actions() + + +def _wait_for_actions() -> None: + # Find out the version and tag name. + current_version = get_package_version() + tag_name = f"v{current_version}" + + # Authentication is optional on this endpoint. + url = f"https://api.github.com/repos/matrix-org/synapse/actions/runs?branch={tag_name}" + + req = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"}) + + time.sleep(10 * 60) + while True: + time.sleep(5 * 60) + response = urllib.request.urlopen(req) + resp = json.loads(response.read()) + + if len(resp["workflow_runs"]) == 0: + continue + + if all( + workflow["status"] != "in_progress" for workflow in resp["workflow_runs"] + ): + success = ( + workflow["status"] == "completed" for workflow in resp["workflow_runs"] + ) + if success: + _notify("Workflows successful. You can now continue the release.") + else: + _notify("Workflows failed.") + click.confirm("Continue anyway?", abort=True) + + break + + +def _notify(message: str) -> None: + # Send a bell character. Most terminals will play a sound or show a notification + # for this. + click.echo(f"\a{message}") + + # Try and run notify-send, but don't raise an Exception if this fails + # (This is best-effort) + # TODO Support other platforms? + subprocess.run( + [ + "notify-send", + "--app-name", + "Synapse Release Script", + "--expire-time", + "3600000", + message, + ] + ) + + @cli.command() def merge_back() -> None: _merge_back() From 74ec2089961cf94e7d853b1b751d9153453520de Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 9 Aug 2022 14:36:30 +0100 Subject: [PATCH 03/10] Note that the blog post and tweets exist --- scripts-dev/release.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 8208956d49bc..8b068d37d504 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -633,7 +633,9 @@ def _announce() -> None: - #homeowners:matrix.org (Synapse Announcements), bumping the version in the topic - #synapse:matrix.org (Synapse Admins), bumping the version in the topic - #synapse-dev:matrix.org -- #synapse-package-maintainers:matrix.org""" +- #synapse-package-maintainers:matrix.org + +Ask the designated people to do the blog and tweets.""" ) From 437ba669602c1b571a2e437547b9c13a685229a4 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 9 Aug 2022 14:36:46 +0100 Subject: [PATCH 04/10] Add 'full' command that does everything from start to finish --- scripts-dev/release.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 8b068d37d504..02993c81c3c9 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -639,6 +639,49 @@ def _announce() -> None: ) +@cli.command() +@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) +def full(gh_token: str) -> None: + click.echo("1. If this is a security release, read the security wiki page.") + click.echo("2. Check for any release blockers before proceeding.") + click.echo(" https://github.com/matrix-org/synapse/labels/X-Release-Blocker") + + click.confirm("Ready?", abort=True) + + click.echo("\n*** prepare ***") + _prepare() + + click.echo("Deploy to matrix.org and ensure that it hasn't fallen over.") + click.echo("Remember to silence the alerts to prevent alert spam.") + click.confirm("Deployed?", abort=True) + + click.echo("\n*** tag ***") + _tag(gh_token) + + click.echo("\n*** wait for actions ***") + _wait_for_actions() + + click.echo("\n*** publish ***") + _publish(gh_token) + + click.echo("\nUpdate the Debian repository") + click.confirm("Started updating Debian repository?", abort=True) + + click.echo("\n*** upload ***") + _upload() + + click.echo("\nWait for all release methods to be ready.") + # Docker should be ready because it was done by the workflows earlier + # PyPI should be ready because we just ran upload(). + click.confirm("Debs ready?", abort=True) + + click.echo("\n*** merge back ***") + _merge_back() + + click.echo("\n*** announce ***") + _announce() + + def get_package_version() -> version.Version: version_string = subprocess.check_output(["poetry", "version", "--short"]).decode( "utf-8" From 358024ae08221b4eff7347ce19fc2285a628c43f Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 9 Aug 2022 14:37:51 +0100 Subject: [PATCH 05/10] Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/13483.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/13483.misc diff --git a/changelog.d/13483.misc b/changelog.d/13483.misc new file mode 100644 index 000000000000..4fe6dbbea40d --- /dev/null +++ b/changelog.d/13483.misc @@ -0,0 +1 @@ +Extend the release script to wait for GitHub Actions to finish and to be usable as a guide for the whole process. \ No newline at end of file From 3072ff20437d2eb449abcc3931e5115d906ecefc Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 9 Aug 2022 14:41:37 +0100 Subject: [PATCH 06/10] Extend help text to mention the full command --- scripts-dev/release.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 02993c81c3c9..e9a694a4668c 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -73,18 +73,21 @@ def cli() -> None: ./scripts-dev/release.py tag - # ... wait for assets to build ... + # wait for assets to build, either manually or with: + ./scripts-dev/release.py wait-for-actions ./scripts-dev/release.py publish ./scripts-dev/release.py upload - # Optional: generate some nice links for the announcement - ./scripts-dev/release.py merge-back + # Optional: generate some nice links for the announcement ./scripts-dev/release.py announce + Alternatively, `./scripts-dev/release.py full` will do all the above + as well as guiding you through the manual steps. + If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the `tag`/`publish` command, then a new draft release will be created/published. """ From e89d840922f8c22097b4cd37615b52aa07fc3878 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 1 Sep 2022 14:17:36 +0100 Subject: [PATCH 07/10] Shuffle order around --- scripts-dev/release.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index e9a694a4668c..7c5e36259e99 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -667,20 +667,20 @@ def full(gh_token: str) -> None: click.echo("\n*** publish ***") _publish(gh_token) - click.echo("\nUpdate the Debian repository") - click.confirm("Started updating Debian repository?", abort=True) - click.echo("\n*** upload ***") _upload() + click.echo("\n*** merge back ***") + _merge_back() + + click.echo("\nUpdate the Debian repository") + click.confirm("Started updating Debian repository?", abort=True) + click.echo("\nWait for all release methods to be ready.") # Docker should be ready because it was done by the workflows earlier # PyPI should be ready because we just ran upload(). click.confirm("Debs ready?", abort=True) - click.echo("\n*** merge back ***") - _merge_back() - click.echo("\n*** announce ***") _announce() From 54a0e49128d2855187e3c31c326b7c45070ab9ba Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 1 Sep 2022 14:18:16 +0100 Subject: [PATCH 08/10] TODO debs ready automation --- scripts-dev/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 7c5e36259e99..d09f10424541 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -679,6 +679,7 @@ def full(gh_token: str) -> None: click.echo("\nWait for all release methods to be ready.") # Docker should be ready because it was done by the workflows earlier # PyPI should be ready because we just ran upload(). + # TODO Automatically poll until the Debs have made it to packages.matrix.org click.confirm("Debs ready?", abort=True) click.echo("\n*** announce ***") From 2078dc9f2f7af4f3a7874abe3510d7e09705e23f Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 1 Sep 2022 14:42:34 +0100 Subject: [PATCH 09/10] Use GH_TOKEN for checking workflows if we have one --- scripts-dev/release.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index d09f10424541..93deb24618d7 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -501,19 +501,23 @@ def _merge_into(repo: Repo, source: str, target: str) -> None: @cli.command() -def wait_for_actions() -> None: - _wait_for_actions() +@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) +def wait_for_actions(gh_token: Optional[str]) -> None: + _wait_for_actions(gh_token) -def _wait_for_actions() -> None: +def _wait_for_actions(gh_token: Optional[str]) -> None: # Find out the version and tag name. current_version = get_package_version() tag_name = f"v{current_version}" - # Authentication is optional on this endpoint. + # Authentication is optional on this endpoint, + # but use a token if we have one to reduce the chance of being rate-limited. url = f"https://api.github.com/repos/matrix-org/synapse/actions/runs?branch={tag_name}" - - req = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"}) + headers = {"Accept": "application/vnd.github+json"} + if gh_token is not None: + headers["authorization"] = f"token {gh_token}" + req = urllib.request.Request(url, headers=headers) time.sleep(10 * 60) while True: @@ -662,7 +666,7 @@ def full(gh_token: str) -> None: _tag(gh_token) click.echo("\n*** wait for actions ***") - _wait_for_actions() + _wait_for_actions(gh_token) click.echo("\n*** publish ***") _publish(gh_token) From 9ff49810baf581c526c13e6763befb19ff6c10e0 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Fri, 2 Sep 2022 14:32:14 +0100 Subject: [PATCH 10/10] Make gh_token not required --- scripts-dev/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 93deb24618d7..6603bc593b67 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -501,7 +501,7 @@ def _merge_into(repo: Repo, source: str, target: str) -> None: @cli.command() -@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) +@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=False) def wait_for_actions(gh_token: Optional[str]) -> None: _wait_for_actions(gh_token)