From e9c468f6f4d98d98ec1ace9f14f3719cdb8e64ad Mon Sep 17 00:00:00 2001 From: Barosl Lee Date: Sat, 13 Dec 2014 04:51:53 +0900 Subject: [PATCH] Add batched merge (rollup) feature Currently it works as follows: 1. Mark the commits to be merged together as rollup. (e.g. r+ rollup) These commits will have an implicit priority of -1 to postpone the individual merge. 2. If one of the marked commits reaches the top of the queue, all the marked commits will be merged together and tested. 3. While merging commits, those commits that fail to merge are ignored. 4. You can prioritize the rollup by setting p=. Typical usage: 1. We have three commits to be tested together. 2. Set r+ rollup to the first commit. 3. Set r+ rollup to the second commit. 4. Set r+ rollup p=10 to the last commit, which will trigger the rollup process soon. Fixes #34. --- bors.py | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 7 deletions(-) diff --git a/bors.py b/bors.py index 9677a65..bf5a88d 100755 --- a/bors.py +++ b/bors.py @@ -227,6 +227,7 @@ def __init__(self, cfg, gh, j): self.user = cfg["gh_user"].encode("utf8") self.test_ref = cfg["test_ref"].encode("utf8") self.master_ref = cfg["master_ref"].encode("utf8") + self.batch_ref = 'batch' self.reviewers = [ r.encode("utf8") for r in cfg["reviewers"] ] self.num=j["number"] self.dst_owner=cfg["owner"].encode("utf8") @@ -314,8 +315,17 @@ def approval_list(self): for (_,_,c) in self.head_comments for m in [re.match(r"^r=(\w+)", c)] if m ]) + def batched(self): + for date, user, comment in self.head_comments: + if re.search(r'\b(?:rollup|batch)\b', comment): + return True + return False + def priority(self): p = 0 + + if self.batched(): p = -1 + for (d, u, c) in self.head_comments: m = re.search(r"\bp=(-?\d+)\b", c) if m != None: @@ -437,6 +447,20 @@ def reset_test_ref_to_master(self): self.dst().git().refs().heads(self.test_ref).patch(sha=master_sha, force=True) + def parse_merge_sha(self): + parsed_merge_sha = None + + for s in self.dst().statuses(self.sha).get(): + if s['creator']['login'].encode('utf-8') == self.user and s['state'].encode('utf-8') == 'pending': + mat = re.match(r'running tests for.*?candidate ([a-z0-9]+)', s['description'].encode('utf-8')) + if mat: parsed_merge_sha = mat.group(1) + break + + if self.merge_sha: + assert self.merge_sha == parsed_merge_sha + else: + self.merge_sha = parsed_merge_sha + def merge_pull_head_to_test_ref(self): s = "merging %s into %s" % (self.short(), self.test_ref) try: @@ -455,7 +479,7 @@ def merge_pull_head_to_test_ref(self): self.merge_sha) self.log.info(s) self.add_comment(self.sha, s) - self.set_pending("running tests", u) + self.set_pending("running tests for candidate {}".format(self.merge_sha), u) except github.ApiError: s = s + " failed" @@ -463,6 +487,77 @@ def merge_pull_head_to_test_ref(self): self.add_comment(self.sha, s) self.set_error(s) + def merge_batched_pull_reqs_to_test_ref(self, pulls): + batch_msg = 'merging {} batched pull requests into {}'.format( + len([x for x in pulls if x.current_state() == STATE_APPROVED]), + self.batch_ref, + ) + self.log.info(batch_msg) + self.add_comment(self.sha, batch_msg) + + info = self.dst().git().refs().heads(self.master_ref).get() + master_sha = info['object']['sha'].encode('utf-8') + try: + self.dst().git().refs().heads(self.batch_ref).get() + self.dst().git().refs().heads(self.batch_ref).patch(sha=master_sha, force=True) + except github.ApiError: + self.dst().git().refs().post(sha=master_sha, ref='refs/heads/' + self.batch_ref) + + successes = [] + failures = [] + + batch_sha = '' + + for pull in pulls: + if pull.current_state() == STATE_APPROVED: + self.log.info('merging {} into {}'.format(pull.short(), self.batch_ref)) + + msg = 'Merge pull request #{} from {}/{}\n\n{}\n\nReviewed-by: {}'.format( + pull.num, + pull.src_owner, pull.ref, + pull.title, + ', '.join(pull.approval_list()) + ) + pull_repr = '- {}/{} = {}: {}'.format(pull.src_owner, pull.ref, pull.sha, pull.title) + + try: + info = self.dst().merges().post(base=self.batch_ref, head=pull.sha, commit_message=msg) + batch_sha = info['sha'].encode('utf-8') + except github.ApiError: + failures.append(pull_repr) + else: + successes.append(pull_repr) + + if batch_sha: + try: + self.dst().git().refs().heads(self.test_ref).get() + self.dst().git().refs().heads(self.test_ref).patch(sha=batch_sha) + except github.ApiError as e: + self.dst().git().refs().post(sha=batch_sha, ref='refs/heads/' + self.test_ref) + + url = 'https://github.com/{}/{}/commit/{}'.format(self.dst_owner, self.dst_repo, batch_sha) + short_msg = 'running tests for rollup candidate {} ({} successes, {} failures)'.format(batch_sha, len(successes), len(failures)) + msg = 'Testing rollup candidate = {:.8}'.format(batch_sha) + if successes: msg += '\n\n**Successful merges:**\n\n{}'.format('\n'.join(successes)) + if failures: msg += '\n\n**Failed merges:**\n\n{}'.format('\n'.join(failures)) + + self.log.info(short_msg) + self.add_comment(self.sha, msg) + self.set_pending(short_msg, url) + else: + batch_msg += ' failed' + + self.log.info(batch_msg) + self.add_comment(self.sha, batch_msg) + self.set_error(batch_msg) + + def merge_or_batch(self, pulls): + self.reset_test_ref_to_master() + if self.batched(): + self.merge_batched_pull_reqs_to_test_ref(pulls) + else: + self.merge_pull_head_to_test_ref() + def advance_master_ref_to_test(self): assert self.merge_sha != None s = ("fast-forwarding %s to %s = %.8s" % @@ -487,7 +582,7 @@ def advance_master_ref_to_test(self): - def try_advance(self): + def try_advance(self, pulls): s = self.current_state() self.log.info("considering %s", self.desc()) @@ -504,17 +599,16 @@ def try_advance(self): self.src_repo, self.sha)))) - self.reset_test_ref_to_master() - self.merge_pull_head_to_test_ref() + self.merge_or_batch(pulls) elif s == STATE_PENDING: + self.parse_merge_sha() if self.merge_sha == None: c = ("No active merge of candidate %.8s found, likely manual push to %s" % (self.sha, self.master_ref)) self.log.info(c) self.add_comment(self.sha, c) - self.reset_test_ref_to_master() - self.merge_pull_head_to_test_ref() + self.merge_or_batch(pulls) return self.log.info("%s - found pending state, checking tests", self.short()) assert self.merge_sha != None @@ -548,6 +642,7 @@ def try_advance(self): elif s == STATE_TESTED: self.log.info("%s - tests successful, attempting landing", self.short()) + self.parse_merge_sha() self.advance_master_ref_to_test() @@ -717,7 +812,7 @@ def main(): else: p = pulls[-1] logging.info("working with most-ripe pull %s", p.short()) - p.try_advance() + p.try_advance(list(reversed(pulls)))