From 48858792ccf4e58929de6cdcb3475d0854fe2b6e Mon Sep 17 00:00:00 2001 From: rexwangcc Date: Thu, 10 Sep 2020 19:25:16 -0400 Subject: [PATCH 1/5] Implement the building blocks for the release automation. --- misc/ci_release_pipeline.py | 340 ++++++++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 misc/ci_release_pipeline.py diff --git a/misc/ci_release_pipeline.py b/misc/ci_release_pipeline.py new file mode 100644 index 0000000000000..af355ab25d128 --- /dev/null +++ b/misc/ci_release_pipeline.py @@ -0,0 +1,340 @@ +import argparse +import functools +import logging +import re +import subprocess +from functools import partial, reduce +from pathlib import Path +from typing import Callable, List, Match, Tuple, Union + +import git +import requests +from requests.auth import HTTPBasicAuth + + +API_PREFIX = 'https://api.github.com/repos/taichi-dev/taichi' +logger = logging.getLogger(__name__) + + +def make_api_url(p): + return f'{API_PREFIX}/{p}' + + +def generate_changelog() -> str: + # redirect the stdout to DEVNULL to make it less noisy + subprocess.run(["ti changelog --save"], + shell=True, + check=True, + stdout=subprocess.DEVNULL) + # need to make sure call from the root of the taichi repo + with open("CHANGELOG.md", "r") as fp: + changelog = fp.read() + return changelog + + +def regenerate_docs(): + # redirect both stdout and stderr to DEVNULL to make it less noisy + subprocess.run(["cmake ."], + shell=True, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + +def _semver_matcher() -> Callable: + """Return a func performs regex matching for semantic versions. + The regex uses word_boundry and is sensitive to the changes + to the contents of the CMakeLists.txt file. The match should + have 3 matched groups: + 1. `SET(TI_VERSION_MAJOR ` + 2. the corresponding version number + 3. `)` + """ + return lambda s, w: re.search(rf"(\bSET\(TI_VERSION_{w} )(\d+)(\b\))", s) + + +def _intify_version(v: Match) -> int: + """Convert matched group v to integer.""" + return int(v.group(2)) + + +def parse_semver( + cmakelist_path: str, + return_match_groups: bool = False +) -> Union[Tuple[int, int, int], Tuple[Match[str], Match[str], Match[str]]]: + """Parse and return the major, minor and patch version numbers + (or matched groups) from CMakeLists.txt given CMAKELIST_PATH. + """ + with open(cmakelist_path, "r") as fp: + cmakelist = fp.read() + matcher = partial(_semver_matcher(), cmakelist) + major, minor, patch = map(matcher, ["MAJOR", "MINOR", "PATCH"]) + if return_match_groups: + return major, minor, patch + return tuple(map(_intify_version, (major, minor, patch))) + + +def bump_major(cmakelist_path: str) -> Tuple[int, int, int]: + """Semantically bump the major version in CMakeLists.txt given CMAKELIST_PATH + in-place as side-effect, return the result semVer.""" + major, minor, patch = parse_semver(cmakelist_path=cmakelist_path, + return_match_groups=True) + patterns = ( + (major.re, rf"\g<1>{_intify_version(major) + 1}\g<3>"), + (minor.re, rf"\g<1>0\g<3>"), + (patch.re, rf"\g<1>0\g<3>"), + ) + new_cmakelist_content = reduce( + lambda content, pattern: re.sub(*pattern, content), patterns, + major.string) + with open(cmakelist_path, "w") as fp: + fp.write(new_cmakelist_content) + return _intify_version(major) + 1, 0, 0 + + +def bump_minor(cmakelist_path: str) -> Tuple[int, int, int]: + """Semantically bump the minor version in CMakeLists.txt given CMAKELIST_PATH + in-place as side-effect, return the result semVer.""" + major, minor, patch = parse_semver(cmakelist_path=cmakelist_path, + return_match_groups=True) + patterns = ( + (minor.re, rf"\g<1>{_intify_version(minor) + 1}\g<3>"), + (patch.re, rf"\g<1>0\g<3>"), + ) + new_cmakelist_content = reduce( + lambda content, pattern: re.sub(*pattern, content), patterns, + minor.string) + with open(cmakelist_path, "w") as fp: + fp.write(new_cmakelist_content) + return _intify_version(major), _intify_version(minor) + 1, 0 + + +def bump_patch(cmakelist_path: str) -> Tuple[int, int, int]: + """Semantically bump the patch version in CMakeLists.txt given CMAKELIST_PATH + in-place as side-effect, return the result semVer.""" + major, minor, patch = parse_semver(cmakelist_path=cmakelist_path, + return_match_groups=True) + new_cmakelist_content = re.sub(patch.re, + rf"\g<1>{_intify_version(patch) + 1}\g<3>", + patch.string) + with open(cmakelist_path, "w") as fp: + fp.write(new_cmakelist_content) + return (*map(_intify_version, [major, minor]), _intify_version(patch) + 1) + + +def create_branch_and_push_ref(repo: git.Repo, feature_branch: str): + """Create a new FEATURE_BRANCH in REPO and push the ref to its remote origin.""" + origin = repo.remotes.origin + repo.create_head(feature_branch).checkout() + repo.git.push("--set-upstream", origin, repo.head.ref) + + +def commit_files_and_push(repo: git.Repo, message: str, author: git.Actor, + committer: git.Actor, files: List[str]): + """Add local FILES to git, commit them to REPO's remote origin with MESSAGE, + AUTHOR and COMMITTER info.""" + origin = repo.remotes.origin + repo.index.add(files) + repo.index.commit(message=message, author=author, committer=committer) + origin.push() + + +def make_pull_request(auth_info: HTTPBasicAuth, + title: str, + feature_branch: str, + content: str, + base_branch: str = "master") -> int: + """Make a pull request from FEATURE_BRANCH to BASE_BRANCH with TITLE and CONTENT, + AuthN is done via AUTH_INFO. Return the number of the pull request or + throw HTTPError. This is not idempotent.""" + url = make_api_url("pulls") + headers = {"Accept": "application/vnd.github.v3+json"} + payload = { + "title": title, + "head": feature_branch, + "base": base_branch, + "body": content, + "maintainer_can_modify": True, + "draft": False, + } + response = requests.post(url=url, + headers=headers, + json=payload, + auth=auth_info) + response.raise_for_status() + return response.json()["number"] + + +def merge_pull_request(auth_info: HTTPBasicAuth, pull_number: int) -> str: + """Squash and merge a pull request identified by PULL_NUMBER, AuthN is done + via AUTH_INFO. Return the merge commit SHA hash or throw HTTPError. + This is not idempotent.""" + url = make_api_url(f"pulls/{pull_number}/merge") + headers = {"Accept": "application/vnd.github.v3+json"} + payload = { + "merge_method": "squash", + } + response = requests.put(url=url, + headers=headers, + json=payload, + auth=auth_info) + response.raise_for_status() + return response.json()["sha"] + + +def update_branch_ref(auth_info: HTTPBasicAuth, + commit_hash: str, + target_branch: str = "stable", + force: bool = True): + """Update the ref of TARGET_BRANCH to point to COMMIT_HASH, + AuthN is done via AUTH_INFO. This is idempotent.""" + url = make_api_url(f"git/refs/heads/{target_branch}") + headers = {"Accept": "application/vnd.github.v3+json"} + payload = {"sha": commit_hash, "force": force} + response = requests.patch(url=url, + headers=headers, + json=payload, + auth=auth_info) + response.raise_for_status() + + +def create_release(auth_info: HTTPBasicAuth, tag_name: str, name: str, + commit_hash: str, content: str): + """Create a new release with NAME and CONTENT points at COMMIT_HASH and + tagged as TAG_NAME, AuthN is done via AUTH_INFO. This is not idempotent.""" + url = make_api_url("releases") + headers = {"Accept": "application/vnd.github.v3+json"} + payload = { + "tag_name": tag_name, + # using commit hash than "master" here prevents race conditions from happening + "target_commitish": commit_hash, + "name": name, + "body": content, + "draft": False, + "prerelease": False, + } + response = requests.post(url=url, + headers=headers, + json=payload, + auth=auth_info) + response.raise_for_status() + + +def get_cmd_args(): + parser = argparse.ArgumentParser( + description="Taichi release automation cli") + parser.add_argument("release_type", + choices=["major", "minor", "patch"], + help="The type of the semantic release") + parser.add_argument( + "-t", + "--token", + type=str, + required=True, + help="OAuth token to send authenticated requests to Github API") + parser.add_argument("-a", + "--author", + type=str, + default="Taichi Gardener", + help="The name of the author of the release") + parser.add_argument("-e", + "--email", + type=str, + default="taichigardener@gmail.com", + help="The email of the author of the release") + return parser.parse_args() + + +def main(args): + # 0. Initialize release metadata + logger.info("=> 0. Initialize release metadata") + repo = git.Repo(".") + author = committer = git.Actor(args.name, args.email) + + # TODO: CONVERT TO USE OAUTH TOKEN + authentication = HTTPBasicAuth("PERSON", "PERSONAL TOKEN") + # TODO: ALSO NEED TO SETUP GIT CLIENT + + # Parse out the semantic versions + major_, minor_, patch_ = parse_semver(cmakelist_path="./CMakeLists.txt", + return_match_groups=False) + + # 1. Dispatch on release types, bump the version and regenerate docs + logger.info( + f"=> 1. Bump the {args.release_type} version and regenerate docs") + if args.release_type == "major": + major, minor, patch = bump_major(cmakelist_path="./CMakeLists.txt") + elif args.release_type == "minor": + major, minor, patch = bump_minor(cmakelist_path="./CMakeLists.txt") + else: + major, minor, patch = bump_patch(cmakelist_path="./CMakeLists.txt") + regenerate_docs() + logger.info( + f"=>\tThe version is bumped from {major_}.{minor_}.{patch_} to {major}.{minor}.{patch}" + ) + + # 2. Create a branch for the release + logger.info("=> 2. Create a branch for the release") + release_branch = f"release-{major}.{minor}.{patch}" + create_branch_and_push_ref(repo=repo, feature_branch=release_branch) + logger.info(f"=>\tBranch {release_branch} has been created") + + # 3. Add, commit and push changes + logger.info("=> 3. Add, commit and push changes") + files = ["docs", "CMakeLists.txt"] + commit_files_and_push(repo=repo, + message=f"[release] v{major}.{minor}.{patch}", + author=author, + committer=committer, + files=files) + + # 4. Generate the changelog + logger.info("=> 4. Generate the changelog") + changelog = generate_changelog() + + # 5. Make the release PR + logger.info("=> 5. Make the release PR") + pull_request = make_pull_request( + title=f"[release] v{major}.{minor}.{patch}", + feature_branch=release_branch, + content=changelog, + auth_info=authentication, + base_branch="master") + logger.info(f"=>\tA release PR #{pull_request} has been created") + + # TODO: NEED TO WAIT THE TESTS TO PASS TO MERGE + + # TODO: EITHER WAIT FOR A FEW SECONDS OR KEEP CHECKING MERGEABLITY WITH RETRIES AND TIMEOUTS + # 6. Merge the release PR + logger.info("=> 6. Merge the release PR") + merge_commit = merge_pull_request(auth_info=authentication, + pull_number=pull_request) + logger.info( + f"=>\tRelease PR #{pull_request} has been merged as commit {merge_commit}" + ) + + # 7. Update stable branch to point to the latest release commit + logger.info( + "=> 7. Update stable branch to point to the latest release commit") + update_branch_ref(auth_info=authentication, + commit_hash=merge_commit, + target_branch="stable") + logger.info(f"=>\tThe stable branch has been updated to {merge_commit}") + + # 8. Create the official release + logger.info("=> 8. Create the official release") + create_release(auth_info=authentication, + tag_name=f"v{major}.{minor}.{patch}", + name=f"v{major}.{minor}.{patch}", + commit_hash=merge_commit, + content=changelog) + logger.info( + f"=>\tA Github release v{major}.{minor}.{patch} has been created") + + +if __name__ == "__main__": + logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', + level=logging.INFO, + datefmt='%Y-%m-%d %H:%M:%S') + arguments = get_cmd_args() + main(args=arguments) From 3285796b97c7210c989489f6e9e3aa3f28876d6f Mon Sep 17 00:00:00 2001 From: "Chengchen(Rex) Wang" <14366016+rexwangcc@users.noreply.github.com> Date: Sun, 13 Sep 2020 23:12:55 -0400 Subject: [PATCH 2/5] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 彭于斌 <1931127624@qq.com> --- misc/ci_release_pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/ci_release_pipeline.py b/misc/ci_release_pipeline.py index af355ab25d128..74d526c7d3a70 100644 --- a/misc/ci_release_pipeline.py +++ b/misc/ci_release_pipeline.py @@ -249,7 +249,7 @@ def main(args): # 0. Initialize release metadata logger.info("=> 0. Initialize release metadata") repo = git.Repo(".") - author = committer = git.Actor(args.name, args.email) + author = committer = git.Actor(args. author, args.email) # TODO: CONVERT TO USE OAUTH TOKEN authentication = HTTPBasicAuth("PERSON", "PERSONAL TOKEN") @@ -313,6 +313,7 @@ def main(args): f"=>\tRelease PR #{pull_request} has been merged as commit {merge_commit}" ) + # TODO: NEED TO WAIT THE BUILDBOTS TO PASS BEFORE RELEASE # 7. Update stable branch to point to the latest release commit logger.info( "=> 7. Update stable branch to point to the latest release commit") From df923014c53b916e650d60cca918cadefa72e890 Mon Sep 17 00:00:00 2001 From: rexwangcc Date: Mon, 14 Sep 2020 22:47:10 -0400 Subject: [PATCH 3/5] A typo. --- misc/ci_release_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/ci_release_pipeline.py b/misc/ci_release_pipeline.py index 74d526c7d3a70..3d411300c4d72 100644 --- a/misc/ci_release_pipeline.py +++ b/misc/ci_release_pipeline.py @@ -249,7 +249,7 @@ def main(args): # 0. Initialize release metadata logger.info("=> 0. Initialize release metadata") repo = git.Repo(".") - author = committer = git.Actor(args. author, args.email) + author = committer = git.Actor(args.author, args.email) # TODO: CONVERT TO USE OAUTH TOKEN authentication = HTTPBasicAuth("PERSON", "PERSONAL TOKEN") From 41522c81c2137a43da8589dcd38cfd95c6e86db1 Mon Sep 17 00:00:00 2001 From: rexwangcc Date: Tue, 15 Sep 2020 01:01:33 -0400 Subject: [PATCH 4/5] Outsource the version bump job from cmake to conf.py. --- docs/conf.py | 59 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 04ede75fd622b..46ee4b61bb027 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,8 +13,59 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # import os -import sys +# import sys # sys.path.insert(0, os.path.abspath('.')) +import re +from functools import partial +from pathlib import Path +from typing import Callable, Match, Tuple, Union + + +# -- Version forge ----------------------------------------------------------- +# Note this section has some duplicate code which +# is directly copied from /misc. +def _semver_matcher() -> Callable: + """Return a func performs regex matching for semantic versions. + The regex uses word_boundry and is sensitive to the changes + to the contents of the CMakeLists.txt file. The match should + have 3 matched groups: + 1. `SET(TI_VERSION_MAJOR ` + 2. the corresponding version number + 3. `)` + """ + return lambda s, w: re.search(rf"(\bSET\(TI_VERSION_{w} )(\d+)(\b\))", s) + + +def _intify_version(v: Match) -> int: + """Convert matched group v to integer.""" + return int(v.group(2)) + + +def parse_semver( + cmakelist_path: str, + return_match_groups: bool = False +) -> Union[Tuple[int, int, int], Tuple[Match[str], Match[str], Match[str]]]: + """Parse and return the major, minor and patch version numbers + (or matched groups) from CMakeLists.txt given CMAKELIST_PATH. + """ + with open(cmakelist_path, "r") as fp: + cmakelist = fp.read() + matcher = partial(_semver_matcher(), cmakelist) + major, minor, patch = map(matcher, ["MAJOR", "MINOR", "PATCH"]) + if return_match_groups: + return major, minor, patch + return tuple(map(_intify_version, (major, minor, patch))) + +# CMakeLists.txt is the only source of the truth when forging +# the version, this script always reads from it, parses the version +# and dump the version string to `version` file. +cmake_file = Path(__file__).resolve().parents[1].joinpath('CMakeLists.txt') +major, minor, patch = parse_semver(cmakelist_path=str(cmake_file)) +version_file = Path(__file__).resolve().parent.joinpath("version") +taichi_version = f"{major}.{minor}.{patch}" +with version_file.open('w') as f: + f.write(f"{taichi_version}\n") + print('Building doc version', taichi_version) # -- Project information ----------------------------------------------------- @@ -22,12 +73,6 @@ copyright = '2020, Taichi Developers' author = 'Taichi Developers' -version_fn = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'version') -with open(version_fn) as f: - taichi_version = f.readline().strip() - print('Building doc version', taichi_version) - # The short X.Y version version = taichi_version # The full version, including alpha/beta/rc tags From d27e0e5563c5063383b835abb392225c110612e4 Mon Sep 17 00:00:00 2001 From: rexwangcc Date: Sun, 20 Sep 2020 18:06:19 -0400 Subject: [PATCH 5/5] Lint. --- docs/conf.py | 1 + misc/ci_release_pipeline.py | 15 ++------------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 46ee4b61bb027..036dcf1f21559 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,6 +56,7 @@ def parse_semver( return major, minor, patch return tuple(map(_intify_version, (major, minor, patch))) + # CMakeLists.txt is the only source of the truth when forging # the version, this script always reads from it, parses the version # and dump the version string to `version` file. diff --git a/misc/ci_release_pipeline.py b/misc/ci_release_pipeline.py index 3d411300c4d72..067560f3533f3 100644 --- a/misc/ci_release_pipeline.py +++ b/misc/ci_release_pipeline.py @@ -11,7 +11,6 @@ import requests from requests.auth import HTTPBasicAuth - API_PREFIX = 'https://api.github.com/repos/taichi-dev/taichi' logger = logging.getLogger(__name__) @@ -32,15 +31,6 @@ def generate_changelog() -> str: return changelog -def regenerate_docs(): - # redirect both stdout and stderr to DEVNULL to make it less noisy - subprocess.run(["cmake ."], - shell=True, - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - - def _semver_matcher() -> Callable: """Return a func performs regex matching for semantic versions. The regex uses word_boundry and is sensitive to the changes @@ -259,16 +249,15 @@ def main(args): major_, minor_, patch_ = parse_semver(cmakelist_path="./CMakeLists.txt", return_match_groups=False) - # 1. Dispatch on release types, bump the version and regenerate docs + # 1. Dispatch on release types, bump the version logger.info( - f"=> 1. Bump the {args.release_type} version and regenerate docs") + f"=> 1. Bump the {args.release_type} version") if args.release_type == "major": major, minor, patch = bump_major(cmakelist_path="./CMakeLists.txt") elif args.release_type == "minor": major, minor, patch = bump_minor(cmakelist_path="./CMakeLists.txt") else: major, minor, patch = bump_patch(cmakelist_path="./CMakeLists.txt") - regenerate_docs() logger.info( f"=>\tThe version is bumped from {major_}.{minor_}.{patch_} to {major}.{minor}.{patch}" )