From a2740af834b463726dbb86ee891ea7de31cec179 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 9 Nov 2023 11:25:38 +0100 Subject: [PATCH 01/46] Do not reset all labels on commit --- process_pr.py | 150 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 111 insertions(+), 39 deletions(-) diff --git a/process_pr.py b/process_pr.py index 9a3867ceecb9..6689a10667d6 100644 --- a/process_pr.py +++ b/process_pr.py @@ -28,11 +28,11 @@ from datetime import datetime from os.path import join, exists, dirname from os import environ -from github_utils import edit_pr, api_rate_limits +from github_utils import edit_pr, api_rate_limits, github_time from github_utils import set_comment_emoji, get_comment_emojis, delete_comment_emoji, set_gh_user from socket import setdefaulttimeout from _py2with3compatibility import run_cmd -from json import dumps, load +from json import dumps, load, loads try: from categories import CMSSW_LABELS @@ -125,6 +125,7 @@ def format(s, **kwds): r"^\s*(?:(?:@|)cmsbuild\s*[,]*\s+|)(?:please\s*[,]*\s+|)ignore\s+tests-rejected\s+(?:with|)([a-z -]+)$", re.I, ) +REGEX_COMMITS_SEEN = re.compile(r"") TEST_WAIT_GAP = 720 ALL_CHECK_FUNCTIONS = None EXTRA_RELVALS_TESTS = ["threading", "gpu", "high-stats", "nano"] @@ -204,18 +205,35 @@ def get_commenter_categories(commenter, comment_date): def get_last_commit(pr): - last_commit = None + commits_ = get_pr_commits_reversed(pr) + if commits_: + return commits_[-1] + else: + return None + + +def get_changed_files_in_commit(repo, commit_obj): + commit = repo.get_commit(commit_obj.commit.sha) + return [x.filename for x in commit.files] + + +def get_pr_commits_reversed(pr): + """ + + :param pr: + :return: PaginatedList[Commit] | List[Commit] + """ try: # This requires at least PyGithub 1.23.0. Making it optional for the moment. - last_commit = pr.get_commits().reversed[0] - except: + return pr.get_commits().reversed + except: # noqa # This seems to fail for more than 250 commits. Not sure if the # problem is github itself or the bindings. try: - last_commit = pr.get_commits()[pr.commits - 1] + return reversed(list(pr.get_commits())) except IndexError: print("Index error: May be PR with no commits") - return last_commit + return [] def get_package_categories(package): @@ -666,6 +684,10 @@ def get_status_state(context, statuses): return "" +def dumps_compact(value): + return dumps(value, separators=(",", ":")) + + def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=False): global L2_DATA if (not force) and ignore_issue(repo_config, repo, issue): @@ -740,6 +762,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # For future pre_checks # if prId>=somePRNumber: default_pre_checks+=["some","new","checks"] pre_checks_url = {} + + events = {} + commit_cache = {} + all_commits = [] + if issue.pull_request: pr = repo.get_pull(prId) if pr.changed_files == 0: @@ -870,8 +897,12 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers.update(set(watchingGroups[watcher])) watchers = set([gh_user_char + u for u in watchers]) print("Watchers " + ", ".join(watchers)) - last_commit_obj = get_last_commit(pr) - if last_commit_obj is None: + all_commits = get_pr_commits_reversed(pr) + for commit in all_commits: + events[commit.commit.committer.date] = get_changed_files_in_commit(commit) + if all_commits: + last_commit_obj = all_commits[0] + else: return last_commit = last_commit_obj.commit commit_statuses = last_commit_obj.get_combined_status().statuses @@ -1115,6 +1146,40 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if m.group(2): code_check_apply_patch = True + new_signatures = {} + # Check L2 signoff for users in this PR signing categories + if [x for x in commenter_categories if x in signing_categories]: + ctype = "" + selected_cats = [] + if re.match("^([+]1|approve[d]?|sign|signed)$", first_line, re.I): + ctype = "+1" + selected_cats = commenter_categories + elif re.match("^([-]1|reject|rejected)$", first_line, re.I): + ctype = "-1" + selected_cats = commenter_categories + elif re.match("^[+-][a-z][a-z0-9-]+$", first_line, re.I): + category_name = first_line[1:].lower() + if category_name in commenter_categories: + ctype = first_line[0] + "1" + selected_cats = [category_name] + if ctype == "+1": + for sign in selected_cats: + new_signatures[sign] = "approved" + if (test_comment is None) and ( + (repository in auto_test_repo) or ("*" in auto_test_repo) + ): + test_comment = comment + if sign == "orp": + mustClose = False + elif ctype == "-1": + for sign in selected_cats: + new_signatures[sign] = "rejected" + if sign == "orp": + mustClose = False + + if new_signatures: + events[github_time(comment.created_at)] = {"type": "sign", "value": new_signatures} + # Ignore all other messages which are before last commit. if issue.pull_request and (comment.created_at < last_commit_date): continue @@ -1255,38 +1320,45 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if not dryRun: set_comment_emoji(comment.id, repository) - # Check L2 signoff for users in this PR signing categories - if [x for x in commenter_categories if x in signing_categories]: - ctype = "" - selected_cats = [] - if re.match("^([+]1|approve[d]?|sign|signed)$", first_line, re.I): - ctype = "+1" - selected_cats = commenter_categories - elif re.match("^([-]1|reject|rejected)$", first_line, re.I): - ctype = "-1" - selected_cats = commenter_categories - elif re.match("^[+-][a-z][a-z0-9-]+$", first_line, re.I): - category_name = first_line[1:].lower() - if category_name in commenter_categories: - ctype = first_line[0] + "1" - selected_cats = [category_name] - if ctype == "+1": - for sign in selected_cats: - signatures[sign] = "approved" - if (test_comment is None) and ( - (repository in auto_test_repo) or ("*" in auto_test_repo) - ): - test_comment = comment - if sign == "orp": - mustClose = False - elif ctype == "-1": - for sign in selected_cats: - signatures[sign] = "rejected" - if sign == "orp": - mustClose = False - # end of parsing comments section + # Get the commit cache from the `already_seen` commit + print("Recalculating signatures") + if already_seen: + print("Loading commit cache") + seen_commits_match = REGEX_COMMITS_SEEN.search(already_seen.body) + if seen_commits_match: + commit_cache = loads(seen_commits_match[1]) + + for commit in all_commits: + if commit.sha not in commit_cache: + commit_cache[commit.sha] = { + "time": github_time(commit.committer.date), + "files": get_changed_files_in_commit(repo, commit), + } + + cache_entry = commit_cache[commit.sha] + events[cache_entry["time"]] = {"type": "commit", "value": cache_entry["files"]} + + if not dryRun: + new_body = REGEX_COMMITS_SEEN.sub("", already_seen.body) + new_body += "" + already_seen.edit(new_body) + + events = dict(sorted(events.items())) + + for event in events: + if event["type"] == "comment": + for cat, sign in event["value"].items(): + signatures[cat] = sign + elif event["type"] == "commit": + chg_categories = [ + x for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) + ] + signatures["orp"] = "pending" + for cat in chg_categories: + signatures[cat] = "pending" + if push_test_issue: auto_close_push_test_issue = True try: From c54ca815d481c993c426fead97459f40ed17da20 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 9 Nov 2023 14:37:05 +0100 Subject: [PATCH 02/46] Add pyproject.toml with Black and pylint configs --- pyproject.toml | 547 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 547 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000000..2e12ecf7cfcc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,547 @@ +[tool.black] +line-length = 99 +target-version = ['py36', 'py37', 'py38', 'py39', 'py310', 'py311'] + +[tool.pylint.main] +# Analyse import fallback blocks. This can be used to support both Python 2 and 3 +# compatible code, which means that the block might have code that exists only in +# one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks = true + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint in +# a server-like mode. +# clear-cache-post-run = + +# Always return a 0 (non-error) status code, even if lint errors are found. This +# is primarily useful in continuous integration scripts. +# exit-zero = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +# extension-pkg-allow-list = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +# extension-pkg-whitelist = + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +# fail-on = + +# Specify a score threshold under which the program will exit with error. +fail-under = 10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +# from-stdin = + +# Files or directories to be skipped. They should be base names, not paths. +ignore = ["CVS"] + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, it +# can't be used as an escape character. +# ignore-paths = + +# Files or directories matching the regular expression patterns are skipped. The +# regex matches against base names, not paths. The default value ignores Emacs +# file locks +ignore-patterns = ["^\\.#"] + +# List of module names for which member attributes should not be checked (useful +# for modules/projects where namespaces are manipulated during runtime and thus +# existing member attributes cannot be deduced by static analysis). It supports +# qualified module names, as well as Unix pattern matching. +# ignored-modules = + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +# init-hook = + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs = 1 + +# Control the amount of potential inferred values when inferring a single object. +# This can help the performance when dealing with large functions or complex, +# nested conditions. +limit-inference-results = 100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +# load-plugins = + +# Pickle collected data for later comparisons. +persistent = false + +# Minimum Python version to use for version dependent checks. Will default to the +# version used to run pylint. +py-version = "3.6" + +# Discover python modules and packages in the file system subtree. +# recursive = + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +# source-roots = + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode = true + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +# unsafe-load-any-extension = + +[tool.pylint.basic] +# Naming style matching correct argument names. +argument-naming-style = "any" + +# Regular expression matching correct argument names. Overrides argument-naming- +# style. If left empty, argument names will be checked with the set naming style. +# argument-rgx = + +# Naming style matching correct attribute names. +attr-naming-style = "any" + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +# attr-rgx = + +# Bad variable names which should always be refused, separated by a comma. +bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"] + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +# bad-names-rgxs = + +# Naming style matching correct class attribute names. +class-attribute-naming-style = "any" + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +# class-attribute-rgx = + +# Naming style matching correct class constant names. +class-const-naming-style = "UPPER_CASE" + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +# class-const-rgx = + +# Naming style matching correct class names. +class-naming-style = "PascalCase" + +# Regular expression matching correct class names. Overrides class-naming-style. +# If left empty, class names will be checked with the set naming style. +# class-rgx = + +# Naming style matching correct constant names. +const-naming-style = "UPPER_CASE" + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming style. +# const-rgx = + +# Minimum line length for functions/classes that require docstrings, shorter ones +# are exempt. +docstring-min-length = -1 + +# Naming style matching correct function names. +function-naming-style = "any" + +# Regular expression matching correct function names. Overrides function-naming- +# style. If left empty, function names will be checked with the set naming style. +# function-rgx = + +# Good variable names which should always be accepted, separated by a comma. +good-names = ["i", "j", "k", "ex", "Run", "_"] + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +# good-names-rgxs = + +# Include a hint for the correct naming format with invalid-name. +# include-naming-hint = + +# Naming style matching correct inline iteration names. +inlinevar-naming-style = "any" + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +# inlinevar-rgx = + +# Naming style matching correct method names. +method-naming-style = "any" + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +# method-rgx = + +# Naming style matching correct module names. +module-naming-style = "any" + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +# module-rgx = + +# Colon-delimited sets of names that determine each other's naming style when the +# name regexes allow several styles. +# name-group = + +# Regular expression which should only match function or class names that do not +# require a docstring. +no-docstring-rgx = "^_" + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. These +# decorators are taken in consideration only for invalid-name. +property-classes = ["abc.abstractproperty"] + +# Regular expression matching correct type alias names. If left empty, type alias +# names will be checked with the set naming style. +# typealias-rgx = + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +# typevar-rgx = + +# Naming style matching correct variable names. +variable-naming-style = "any" + +# Regular expression matching correct variable names. Overrides variable-naming- +# style. If left empty, variable names will be checked with the set naming style. +# variable-rgx = + +[tool.pylint.classes] +# Warn about protected attribute access inside special methods +# check-protected-access-in-special-methods = + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods = ["__init__", "__new__", "setUp", "asyncSetUp", "__post_init__"] + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected = ["_asdict", "_fields", "_replace", "_source", "_make", "os._exit"] + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg = ["cls"] + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg = ["mcs"] + +[tool.pylint.design] +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +# exclude-too-few-public-methods = + +# List of qualified class names to ignore when counting class parents (see R0901) +# ignored-parents = + +# Maximum number of arguments for function / method. +max-args = 5 + +# Maximum number of attributes for a class (see R0902). +max-attributes = 7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr = 5 + +# Maximum number of branch for function / method body. +max-branches = 12 + +# Maximum number of locals for function / method body. +max-locals = 15 + +# Maximum number of parents for a class (see R0901). +max-parents = 7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods = 20 + +# Maximum number of return / yield for function / method body. +max-returns = 6 + +# Maximum number of statements in function / method body. +max-statements = 50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods = 2 + +[tool.pylint.exceptions] +# Exceptions that will emit a warning when caught. +overgeneral-exceptions = ["builtins.BaseException", "builtins.Exception"] + +[tool.pylint.format] +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +# expected-line-ending-format = + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines = "^\\s*(# )??$" + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren = 4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string = " " + +# Maximum number of characters on a single line. +max-line-length = 99 + +# Maximum number of lines in a module. +max-module-lines = 1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +# single-line-class-stmt = + +# Allow the body of an if to be on the same line as the test if there is no else. +# single-line-if-stmt = + +[tool.pylint.imports] +# List of modules that can be imported at any level, not just the top level one. +# allow-any-import-level = + +# Allow explicit reexports by alias from a package __init__. +# allow-reexport-from-package = + +# Allow wildcard imports from modules that define __all__. +# allow-wildcard-with-all = + +# Deprecated modules which should not be used, separated by a comma. +# deprecated-modules = + +# Output a graph (.gv or any supported image format) of external dependencies to +# the given file (report RP0402 must not be disabled). +# ext-import-graph = + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be disabled). +# import-graph = + +# Output a graph (.gv or any supported image format) of internal dependencies to +# the given file (report RP0402 must not be disabled). +# int-import-graph = + +# Force import order to recognize a module as part of the standard compatibility +# libraries. +# known-standard-library = + +# Force import order to recognize a module as part of a third party library. +known-third-party = ["enchant"] + +# Couples of modules and preferred modules, separated by a comma. +# preferred-modules = + +[tool.pylint.logging] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style = "old" + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules = ["logging"] + +[tool.pylint."messages control"] +# Only show warnings with the listed confidence levels. Leave empty to show all. +# Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] + +# Disable the message, report, category or checker with the given id(s). You can +# either give multiple identifiers separated by comma (,) or put this option +# multiple times (only on the command line, not in the configuration file where +# it should appear only once). You can also use "--disable=all" to disable +# everything first and then re-enable specific checks. For example, if you want +# to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable = ["raw-checker-failed", "bad-inline-option", "locally-disabled", "file-ignored", "suppressed-message", "useless-suppression", "deprecated-pragma", "use-symbolic-message-instead", "use-implicit-booleaness-not-comparison-to-string", "use-implicit-booleaness-not-comparison-to-zero", "missing-function-docstring", "unspecified-encoding", "line-too-long", "too-many-return-statements", "too-many-branches", "too-many-statements", "invalid-name", "missing-module-docstring", "too-many-lines", "too-many-arguments", "too-many-nested-blocks", "ungrouped-imports", "too-many-locals"] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where it +# should appear only once). See also the "--disable" option for examples. +# enable = + +[tool.pylint.method_args] +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods = ["requests.api.delete", "requests.api.get", "requests.api.head", "requests.api.options", "requests.api.patch", "requests.api.post", "requests.api.put", "requests.api.request"] + +[tool.pylint.miscellaneous] +# List of note tags to take in consideration, separated by a comma. +notes = ["FIXME", "XXX", "TODO"] + +# Regular expression of note tags to take in consideration. +# notes-rgx = + +[tool.pylint.refactoring] +# Maximum number of nested blocks for function / method body +max-nested-blocks = 5 + +# Complete name of functions that never returns. When checking for inconsistent- +# return-statements if a never returning function is called then it will be +# considered as an explicit return statement and no message will be printed. +never-returning-functions = ["sys.exit", "argparse.parse_error"] + +[tool.pylint.reports] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each category, +# as well as 'statement' which is the total number of statements analyzed. This +# score is used by the global evaluation report (RP0004). +evaluation = "max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))" + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +# msg-template = + +# Set the output format. Available formats are: text, parseable, colorized, json2 +# (improved json format), json (old json format) and msvs (visual studio). You +# can also give a reporter class, e.g. mypackage.mymodule.MyReporterClass. +# output-format = + +# Tells whether to display a full report or only the messages. +# reports = + +# Activate the evaluation score. +score = true + +[tool.pylint.similarities] +# Comments are removed from the similarity computation +ignore-comments = true + +# Docstrings are removed from the similarity computation +ignore-docstrings = true + +# Imports are removed from the similarity computation +ignore-imports = true + +# Signatures are removed from the similarity computation +ignore-signatures = true + +# Minimum lines number of a similarity. +min-similarity-lines = 4 + +[tool.pylint.spelling] +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions = 4 + +# Spelling dictionary name. Available dictionaries: en_AG (hunspell), en_AU +# (hunspell), en_BS (hunspell), en_BW (hunspell), en_BZ (hunspell), en_CA +# (hunspell), en_DK (hunspell), en_GB (hunspell), en_GH (hunspell), en_HK +# (hunspell), en_IE (hunspell), en_IN (hunspell), en_JM (hunspell), en_MW +# (hunspell), en_NA (hunspell), en_NG (hunspell), en_NZ (hunspell), en_PH +# (hunspell), en_SG (hunspell), en_TT (hunspell), en_US (hunspell), en_ZA +# (hunspell), en_ZM (hunspell), en_ZW (hunspell). +# spelling-dict = + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:" + +# List of comma separated words that should not be checked. +# spelling-ignore-words = + +# A path to a file that contains the private dictionary; one word per line. +# spelling-private-dict-file = + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +# spelling-store-unknown-words = + +[tool.pylint.typecheck] +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators = ["contextlib.contextmanager"] + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +# generated-members = + +# Tells whether missing members accessed in mixin class should be ignored. A +# class is considered mixin if its name matches the mixin-class-rgx option. +# Tells whether to warn about missing members when the owner of the attribute is +# inferred to be None. +ignore-none = true + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference can +# return multiple potential results while evaluating a Python object, but some +# branches might not be evaluated, which results in partial inference. In that +# case, it might be useful to still emit no-member and other checks for the rest +# of the inferred objects. +ignore-on-opaque-inference = true + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes = ["optparse.Values", "thread._local", "_thread._local", "argparse.Namespace"] + +# Show a hint with possible names when a member name was not found. The aspect of +# finding the hint is based on edit distance. +missing-member-hint = true + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance = 1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices = 1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx = ".*[Mm]ixin" + +# List of decorators that change the signature of a decorated function. +# signature-mutators = + +[tool.pylint.variables] +# List of additional names supposed to be defined in builtins. Remember that you +# should avoid defining new builtins when possible. +# additional-builtins = + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables = true + +# List of names allowed to shadow builtins +# allowed-redefined-builtins = + +# List of strings which can identify a callback function by name. A callback name +# must start or end with one of those strings. +callbacks = ["cb_", "_cb"] + +# A regular expression matching the name of dummy variables (i.e. expected to not +# be used). +dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" + +# Argument names that match this expression will be ignored. +ignored-argument-names = "_.*|^ignored_|^unused_" + +# Tells whether we should check for unused import in __init__ files. +# init-import = + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] + + From da8903e632ad10eb52f1c0ce4ae538b5eabf6ce8 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 9 Nov 2023 15:32:59 +0100 Subject: [PATCH 03/46] Changes from review --- cms_static.py | 1 + process_pr.py | 76 +++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/cms_static.py b/cms_static.py index 63036c44467c..dea5228595dd 100644 --- a/cms_static.py +++ b/cms_static.py @@ -11,6 +11,7 @@ CMSBUILD_GH_USER = "cmsbuild" CMSBOT_IGNORE_MSG = "\s*" CMSBOT_NO_NOTIFY_MSG = "\s*" +CMSBOT_TECHNICAL_MSG = "cms-bot internal usage" VALID_CMS_SW_REPOS_FOR_TESTS = [ "cmssw", "cmsdist", diff --git a/process_pr.py b/process_pr.py index 6689a10667d6..7a9ce7823be2 100644 --- a/process_pr.py +++ b/process_pr.py @@ -20,6 +20,7 @@ CMSBOT_IGNORE_MSG, VALID_CMS_SW_REPOS_FOR_TESTS, CREATE_REPO, + CMSBOT_TECHNICAL_MSG, ) from cms_static import BACKPORT_STR, GH_CMSSW_ORGANIZATION, CMSBOT_NO_NOTIFY_MSG from githublabels import TYPE_COMMANDS, TEST_IGNORE_REASON @@ -28,7 +29,7 @@ from datetime import datetime from os.path import join, exists, dirname from os import environ -from github_utils import edit_pr, api_rate_limits, github_time +from github_utils import edit_pr, api_rate_limits from github_utils import set_comment_emoji, get_comment_emojis, delete_comment_emoji, set_gh_user from socket import setdefaulttimeout from _py2with3compatibility import run_cmd @@ -899,7 +900,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Watchers " + ", ".join(watchers)) all_commits = get_pr_commits_reversed(pr) for commit in all_commits: - events[commit.commit.committer.date] = get_changed_files_in_commit(commit) + events[commit.commit.committer.date] = get_changed_files_in_commit(repo, commit) if all_commits: last_commit_obj = all_commits[0] else: @@ -975,6 +976,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) print("Pre check status:", pre_checks_state) already_seen = None + technical_comment = None pull_request_updated = False comparison_done = False comparison_notrun = False @@ -1034,6 +1036,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pull_request_updated = True continue + if (commenter == cmsbuild_user) and re.match(CMSBOT_TECHNICAL_MSG, first_line): + technical_comment = comment + assign_type, new_cats = get_assign_categories(first_line) if new_cats: if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user): @@ -1178,7 +1183,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F mustClose = False if new_signatures: - events[github_time(comment.created_at)] = {"type": "sign", "value": new_signatures} + events[comment.created_at] = {"type": "sign", "value": new_signatures} # Ignore all other messages which are before last commit. if issue.pull_request and (comment.created_at < last_commit_date): @@ -1322,33 +1327,62 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # end of parsing comments section - # Get the commit cache from the `already_seen` commit + # Get the commit cache from `already_seen` commit or technical commit print("Recalculating signatures") - if already_seen: + cache_comment = None + new_comment = False + + if technical_comment: + cache_comment = technical_comment + else: + if already_seen: + cache_comment = already_seen + else: + new_comment = True + # if not dryRun: + # cache_comment = issue.create_comment(CMSBOT_TECHNICAL_MSG + "\n") + + if cache_comment: print("Loading commit cache") - seen_commits_match = REGEX_COMMITS_SEEN.search(already_seen.body) + seen_commits_match = REGEX_COMMITS_SEEN.search(cache_comment.body) if seen_commits_match: commit_cache = loads(seen_commits_match[1]) - for commit in all_commits: - if commit.sha not in commit_cache: - commit_cache[commit.sha] = { - "time": github_time(commit.committer.date), - "files": get_changed_files_in_commit(repo, commit), - } - - cache_entry = commit_cache[commit.sha] - events[cache_entry["time"]] = {"type": "commit", "value": cache_entry["files"]} + old_body = cache_comment.body if cache_comment else CMSBOT_TECHNICAL_MSG + new_body = REGEX_COMMITS_SEEN.sub( + "", old_body + ) - if not dryRun: - new_body = REGEX_COMMITS_SEEN.sub("", already_seen.body) - new_body += "" - already_seen.edit(new_body) + if not dryRun: + if new_comment: + issue.create_comment(new_body) + else: + cache_comment.edit(new_body) + else: + if new_comment: + print("DRY RUN: Creating technical comment with text") + print(new_body) + else: + print("DRY RUN: Updating existing comment with text") + print(new_body) + + for commit in all_commits: + if commit.sha not in commit_cache: + commit_cache[commit.sha] = { + "time": int(commit.commit.committer.date.timestamp()), + "files": get_changed_files_in_commit(repo, commit), + } + + cache_entry = commit_cache[commit.sha] + events[datetime.fromtimestamp(cache_entry["time"])] = { + "type": "commit", + "value": cache_entry["files"], + } events = dict(sorted(events.items())) - for event in events: - if event["type"] == "comment": + for event in events.values(): + if event["type"] == "sign": for cat, sign in event["value"].items(): signatures[cat] = sign elif event["type"] == "commit": From 3c208117a06e93c76d704feb8d9a23cff6a0816b Mon Sep 17 00:00:00 2001 From: iarspider Date: Fri, 10 Nov 2023 16:24:02 +0100 Subject: [PATCH 04/46] Changes from review (1/2) --- process_pr.py | 75 ++++--- pyproject.toml | 547 ------------------------------------------------- 2 files changed, 50 insertions(+), 572 deletions(-) delete mode 100644 pyproject.toml diff --git a/process_pr.py b/process_pr.py index 7a9ce7823be2..88a5fe046e6a 100644 --- a/process_pr.py +++ b/process_pr.py @@ -126,7 +126,8 @@ def format(s, **kwds): r"^\s*(?:(?:@|)cmsbuild\s*[,]*\s+|)(?:please\s*[,]*\s+|)ignore\s+tests-rejected\s+(?:with|)([a-z -]+)$", re.I, ) -REGEX_COMMITS_SEEN = re.compile(r"") +REGEX_COMMITS_CACHE = re.compile(r"") +REGEX_IGNORE_COMMIT_COUNT = "\+commit-count" TEST_WAIT_GAP = 720 ALL_CHECK_FUNCTIONS = None EXTRA_RELVALS_TESTS = ["threading", "gpu", "high-stats", "nano"] @@ -177,6 +178,7 @@ def format(s, **kwds): + "|_input|)": [RELVAL_OPTS, "EXTRA_MATRIX_COMMAND_ARGS", True], } +MAX_INITIAL_COMMITS_IN_PR = 20 L2_DATA = {} @@ -767,6 +769,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F events = {} commit_cache = {} all_commits = [] + override_too_many_commits = False + warned_too_many_commits = False + + l2s = ", ".join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS]) if issue.pull_request: pr = repo.get_pull(prId) @@ -899,12 +905,12 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers = set([gh_user_char + u for u in watchers]) print("Watchers " + ", ".join(watchers)) all_commits = get_pr_commits_reversed(pr) - for commit in all_commits: - events[commit.commit.committer.date] = get_changed_files_in_commit(repo, commit) + if all_commits: last_commit_obj = all_commits[0] else: return + last_commit = last_commit_obj.commit commit_statuses = last_commit_obj.get_combined_status().statuses bot_status = get_status(bot_status_name, commit_statuses) @@ -1036,9 +1042,19 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pull_request_updated = True continue - if (commenter == cmsbuild_user) and re.match(CMSBOT_TECHNICAL_MSG, first_line): + if ( + (commenter == cmsbuild_user) + and re.match(CMSBOT_TECHNICAL_MSG, first_line) + and not technical_comment + ): technical_comment = comment + if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: + warned_too_many_commits = True + + if commenter in l2s and re.match("^\s*" + REGEX_IGNORE_COMMIT_COUNT + "\s*$", first_line): + override_too_many_commits = True + assign_type, new_cats = get_assign_categories(first_line) if new_cats: if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user): @@ -1326,46 +1342,42 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F set_comment_emoji(comment.id, repository) # end of parsing comments section + if ( + (not warned_too_many_commits) + and not override_too_many_commits + and len(all_commits) >= MAX_INITIAL_COMMITS_IN_PR + ): + issue.create_comment( + f"This PR contains too many commits ({len(all_commits)} > {MAX_INITIAL_COMMITS_IN_PR}). " + f"Make sure you chose the right target branch. " + "\n{l2s}, you can override this check with `+commit-count`." + ) + + if len(all_commits) < MAX_INITIAL_COMMITS_IN_PR or override_too_many_commits: + for commit in all_commits: + events[commit.commit.committer.date] = get_changed_files_in_commit(repo, commit) # Get the commit cache from `already_seen` commit or technical commit print("Recalculating signatures") cache_comment = None - new_comment = False if technical_comment: cache_comment = technical_comment else: if already_seen: cache_comment = already_seen - else: - new_comment = True - # if not dryRun: - # cache_comment = issue.create_comment(CMSBOT_TECHNICAL_MSG + "\n") if cache_comment: print("Loading commit cache") - seen_commits_match = REGEX_COMMITS_SEEN.search(cache_comment.body) + seen_commits_match = REGEX_COMMITS_CACHE.search(cache_comment.body) if seen_commits_match: commit_cache = loads(seen_commits_match[1]) old_body = cache_comment.body if cache_comment else CMSBOT_TECHNICAL_MSG - new_body = REGEX_COMMITS_SEEN.sub( + new_body = REGEX_COMMITS_CACHE.sub( "", old_body ) - if not dryRun: - if new_comment: - issue.create_comment(new_body) - else: - cache_comment.edit(new_body) - else: - if new_comment: - print("DRY RUN: Creating technical comment with text") - print(new_body) - else: - print("DRY RUN: Updating existing comment with text") - print(new_body) - for commit in all_commits: if commit.sha not in commit_cache: commit_cache[commit.sha] = { @@ -1379,6 +1391,20 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F "value": cache_entry["files"], } + if not dryRun: + if cache_comment: + if old_body != new_body: + cache_comment.edit(new_body) + else: + issue.create_comment(new_body) + else: + if cache_comment: + print("DRY RUN: Updating existing comment with text") + print(new_body) + else: + print("DRY RUN: Creating technical comment with text") + print(new_body) + events = dict(sorted(events.items())) for event in events.values(): @@ -1777,7 +1803,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F uname = "" if issue.user.name: uname = issue.user.name.encode("ascii", "ignore").decode() - l2s = ", ".join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS]) issueMessage = format( "%(msgPrefix)s %(gh_user_char)s%(user)s" " %(name)s.\n\n" diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 2e12ecf7cfcc..000000000000 --- a/pyproject.toml +++ /dev/null @@ -1,547 +0,0 @@ -[tool.black] -line-length = 99 -target-version = ['py36', 'py37', 'py38', 'py39', 'py310', 'py311'] - -[tool.pylint.main] -# Analyse import fallback blocks. This can be used to support both Python 2 and 3 -# compatible code, which means that the block might have code that exists only in -# one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks = true - -# Clear in-memory caches upon conclusion of linting. Useful if running pylint in -# a server-like mode. -# clear-cache-post-run = - -# Always return a 0 (non-error) status code, even if lint errors are found. This -# is primarily useful in continuous integration scripts. -# exit-zero = - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -# extension-pkg-allow-list = - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. (This is an alternative name to extension-pkg-allow-list -# for backward compatibility.) -# extension-pkg-whitelist = - -# Return non-zero exit code if any of these messages/categories are detected, -# even if score is above --fail-under value. Syntax same as enable. Messages -# specified are enabled, while categories only check already-enabled messages. -# fail-on = - -# Specify a score threshold under which the program will exit with error. -fail-under = 10 - -# Interpret the stdin as a python script, whose filename needs to be passed as -# the module_or_package argument. -# from-stdin = - -# Files or directories to be skipped. They should be base names, not paths. -ignore = ["CVS"] - -# Add files or directories matching the regular expressions patterns to the -# ignore-list. The regex matches against paths and can be in Posix or Windows -# format. Because '\\' represents the directory delimiter on Windows systems, it -# can't be used as an escape character. -# ignore-paths = - -# Files or directories matching the regular expression patterns are skipped. The -# regex matches against base names, not paths. The default value ignores Emacs -# file locks -ignore-patterns = ["^\\.#"] - -# List of module names for which member attributes should not be checked (useful -# for modules/projects where namespaces are manipulated during runtime and thus -# existing member attributes cannot be deduced by static analysis). It supports -# qualified module names, as well as Unix pattern matching. -# ignored-modules = - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -# init-hook = - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use, and will cap the count on Windows to -# avoid hangs. -jobs = 1 - -# Control the amount of potential inferred values when inferring a single object. -# This can help the performance when dealing with large functions or complex, -# nested conditions. -limit-inference-results = 100 - -# List of plugins (as comma separated values of python module names) to load, -# usually to register additional checkers. -# load-plugins = - -# Pickle collected data for later comparisons. -persistent = false - -# Minimum Python version to use for version dependent checks. Will default to the -# version used to run pylint. -py-version = "3.6" - -# Discover python modules and packages in the file system subtree. -# recursive = - -# Add paths to the list of the source roots. Supports globbing patterns. The -# source root is an absolute path or a path relative to the current working -# directory used to determine a package namespace for modules located under the -# source root. -# source-roots = - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode = true - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -# unsafe-load-any-extension = - -[tool.pylint.basic] -# Naming style matching correct argument names. -argument-naming-style = "any" - -# Regular expression matching correct argument names. Overrides argument-naming- -# style. If left empty, argument names will be checked with the set naming style. -# argument-rgx = - -# Naming style matching correct attribute names. -attr-naming-style = "any" - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. If left empty, attribute names will be checked with the set naming -# style. -# attr-rgx = - -# Bad variable names which should always be refused, separated by a comma. -bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"] - -# Bad variable names regexes, separated by a comma. If names match any regex, -# they will always be refused -# bad-names-rgxs = - -# Naming style matching correct class attribute names. -class-attribute-naming-style = "any" - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. If left empty, class attribute names will be checked -# with the set naming style. -# class-attribute-rgx = - -# Naming style matching correct class constant names. -class-const-naming-style = "UPPER_CASE" - -# Regular expression matching correct class constant names. Overrides class- -# const-naming-style. If left empty, class constant names will be checked with -# the set naming style. -# class-const-rgx = - -# Naming style matching correct class names. -class-naming-style = "PascalCase" - -# Regular expression matching correct class names. Overrides class-naming-style. -# If left empty, class names will be checked with the set naming style. -# class-rgx = - -# Naming style matching correct constant names. -const-naming-style = "UPPER_CASE" - -# Regular expression matching correct constant names. Overrides const-naming- -# style. If left empty, constant names will be checked with the set naming style. -# const-rgx = - -# Minimum line length for functions/classes that require docstrings, shorter ones -# are exempt. -docstring-min-length = -1 - -# Naming style matching correct function names. -function-naming-style = "any" - -# Regular expression matching correct function names. Overrides function-naming- -# style. If left empty, function names will be checked with the set naming style. -# function-rgx = - -# Good variable names which should always be accepted, separated by a comma. -good-names = ["i", "j", "k", "ex", "Run", "_"] - -# Good variable names regexes, separated by a comma. If names match any regex, -# they will always be accepted -# good-names-rgxs = - -# Include a hint for the correct naming format with invalid-name. -# include-naming-hint = - -# Naming style matching correct inline iteration names. -inlinevar-naming-style = "any" - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. If left empty, inline iteration names will be checked -# with the set naming style. -# inlinevar-rgx = - -# Naming style matching correct method names. -method-naming-style = "any" - -# Regular expression matching correct method names. Overrides method-naming- -# style. If left empty, method names will be checked with the set naming style. -# method-rgx = - -# Naming style matching correct module names. -module-naming-style = "any" - -# Regular expression matching correct module names. Overrides module-naming- -# style. If left empty, module names will be checked with the set naming style. -# module-rgx = - -# Colon-delimited sets of names that determine each other's naming style when the -# name regexes allow several styles. -# name-group = - -# Regular expression which should only match function or class names that do not -# require a docstring. -no-docstring-rgx = "^_" - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. These -# decorators are taken in consideration only for invalid-name. -property-classes = ["abc.abstractproperty"] - -# Regular expression matching correct type alias names. If left empty, type alias -# names will be checked with the set naming style. -# typealias-rgx = - -# Regular expression matching correct type variable names. If left empty, type -# variable names will be checked with the set naming style. -# typevar-rgx = - -# Naming style matching correct variable names. -variable-naming-style = "any" - -# Regular expression matching correct variable names. Overrides variable-naming- -# style. If left empty, variable names will be checked with the set naming style. -# variable-rgx = - -[tool.pylint.classes] -# Warn about protected attribute access inside special methods -# check-protected-access-in-special-methods = - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods = ["__init__", "__new__", "setUp", "asyncSetUp", "__post_init__"] - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected = ["_asdict", "_fields", "_replace", "_source", "_make", "os._exit"] - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg = ["cls"] - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg = ["mcs"] - -[tool.pylint.design] -# List of regular expressions of class ancestor names to ignore when counting -# public methods (see R0903) -# exclude-too-few-public-methods = - -# List of qualified class names to ignore when counting class parents (see R0901) -# ignored-parents = - -# Maximum number of arguments for function / method. -max-args = 5 - -# Maximum number of attributes for a class (see R0902). -max-attributes = 7 - -# Maximum number of boolean expressions in an if statement (see R0916). -max-bool-expr = 5 - -# Maximum number of branch for function / method body. -max-branches = 12 - -# Maximum number of locals for function / method body. -max-locals = 15 - -# Maximum number of parents for a class (see R0901). -max-parents = 7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods = 20 - -# Maximum number of return / yield for function / method body. -max-returns = 6 - -# Maximum number of statements in function / method body. -max-statements = 50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods = 2 - -[tool.pylint.exceptions] -# Exceptions that will emit a warning when caught. -overgeneral-exceptions = ["builtins.BaseException", "builtins.Exception"] - -[tool.pylint.format] -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -# expected-line-ending-format = - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines = "^\\s*(# )??$" - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren = 4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string = " " - -# Maximum number of characters on a single line. -max-line-length = 99 - -# Maximum number of lines in a module. -max-module-lines = 1000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -# single-line-class-stmt = - -# Allow the body of an if to be on the same line as the test if there is no else. -# single-line-if-stmt = - -[tool.pylint.imports] -# List of modules that can be imported at any level, not just the top level one. -# allow-any-import-level = - -# Allow explicit reexports by alias from a package __init__. -# allow-reexport-from-package = - -# Allow wildcard imports from modules that define __all__. -# allow-wildcard-with-all = - -# Deprecated modules which should not be used, separated by a comma. -# deprecated-modules = - -# Output a graph (.gv or any supported image format) of external dependencies to -# the given file (report RP0402 must not be disabled). -# ext-import-graph = - -# Output a graph (.gv or any supported image format) of all (i.e. internal and -# external) dependencies to the given file (report RP0402 must not be disabled). -# import-graph = - -# Output a graph (.gv or any supported image format) of internal dependencies to -# the given file (report RP0402 must not be disabled). -# int-import-graph = - -# Force import order to recognize a module as part of the standard compatibility -# libraries. -# known-standard-library = - -# Force import order to recognize a module as part of a third party library. -known-third-party = ["enchant"] - -# Couples of modules and preferred modules, separated by a comma. -# preferred-modules = - -[tool.pylint.logging] -# The type of string formatting that logging methods do. `old` means using % -# formatting, `new` is for `{}` formatting. -logging-format-style = "old" - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules = ["logging"] - -[tool.pylint."messages control"] -# Only show warnings with the listed confidence levels. Leave empty to show all. -# Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] - -# Disable the message, report, category or checker with the given id(s). You can -# either give multiple identifiers separated by comma (,) or put this option -# multiple times (only on the command line, not in the configuration file where -# it should appear only once). You can also use "--disable=all" to disable -# everything first and then re-enable specific checks. For example, if you want -# to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable = ["raw-checker-failed", "bad-inline-option", "locally-disabled", "file-ignored", "suppressed-message", "useless-suppression", "deprecated-pragma", "use-symbolic-message-instead", "use-implicit-booleaness-not-comparison-to-string", "use-implicit-booleaness-not-comparison-to-zero", "missing-function-docstring", "unspecified-encoding", "line-too-long", "too-many-return-statements", "too-many-branches", "too-many-statements", "invalid-name", "missing-module-docstring", "too-many-lines", "too-many-arguments", "too-many-nested-blocks", "ungrouped-imports", "too-many-locals"] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where it -# should appear only once). See also the "--disable" option for examples. -# enable = - -[tool.pylint.method_args] -# List of qualified names (i.e., library.method) which require a timeout -# parameter e.g. 'requests.api.get,requests.api.post' -timeout-methods = ["requests.api.delete", "requests.api.get", "requests.api.head", "requests.api.options", "requests.api.patch", "requests.api.post", "requests.api.put", "requests.api.request"] - -[tool.pylint.miscellaneous] -# List of note tags to take in consideration, separated by a comma. -notes = ["FIXME", "XXX", "TODO"] - -# Regular expression of note tags to take in consideration. -# notes-rgx = - -[tool.pylint.refactoring] -# Maximum number of nested blocks for function / method body -max-nested-blocks = 5 - -# Complete name of functions that never returns. When checking for inconsistent- -# return-statements if a never returning function is called then it will be -# considered as an explicit return statement and no message will be printed. -never-returning-functions = ["sys.exit", "argparse.parse_error"] - -[tool.pylint.reports] -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'fatal', 'error', 'warning', 'refactor', -# 'convention', and 'info' which contain the number of messages in each category, -# as well as 'statement' which is the total number of statements analyzed. This -# score is used by the global evaluation report (RP0004). -evaluation = "max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))" - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -# msg-template = - -# Set the output format. Available formats are: text, parseable, colorized, json2 -# (improved json format), json (old json format) and msvs (visual studio). You -# can also give a reporter class, e.g. mypackage.mymodule.MyReporterClass. -# output-format = - -# Tells whether to display a full report or only the messages. -# reports = - -# Activate the evaluation score. -score = true - -[tool.pylint.similarities] -# Comments are removed from the similarity computation -ignore-comments = true - -# Docstrings are removed from the similarity computation -ignore-docstrings = true - -# Imports are removed from the similarity computation -ignore-imports = true - -# Signatures are removed from the similarity computation -ignore-signatures = true - -# Minimum lines number of a similarity. -min-similarity-lines = 4 - -[tool.pylint.spelling] -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions = 4 - -# Spelling dictionary name. Available dictionaries: en_AG (hunspell), en_AU -# (hunspell), en_BS (hunspell), en_BW (hunspell), en_BZ (hunspell), en_CA -# (hunspell), en_DK (hunspell), en_GB (hunspell), en_GH (hunspell), en_HK -# (hunspell), en_IE (hunspell), en_IN (hunspell), en_JM (hunspell), en_MW -# (hunspell), en_NA (hunspell), en_NG (hunspell), en_NZ (hunspell), en_PH -# (hunspell), en_SG (hunspell), en_TT (hunspell), en_US (hunspell), en_ZA -# (hunspell), en_ZM (hunspell), en_ZW (hunspell). -# spelling-dict = - -# List of comma separated words that should be considered directives if they -# appear at the beginning of a comment and should not be checked. -spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:" - -# List of comma separated words that should not be checked. -# spelling-ignore-words = - -# A path to a file that contains the private dictionary; one word per line. -# spelling-private-dict-file = - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -# spelling-store-unknown-words = - -[tool.pylint.typecheck] -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators = ["contextlib.contextmanager"] - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -# generated-members = - -# Tells whether missing members accessed in mixin class should be ignored. A -# class is considered mixin if its name matches the mixin-class-rgx option. -# Tells whether to warn about missing members when the owner of the attribute is -# inferred to be None. -ignore-none = true - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference can -# return multiple potential results while evaluating a Python object, but some -# branches might not be evaluated, which results in partial inference. In that -# case, it might be useful to still emit no-member and other checks for the rest -# of the inferred objects. -ignore-on-opaque-inference = true - -# List of symbolic message names to ignore for Mixin members. -ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes = ["optparse.Values", "thread._local", "_thread._local", "argparse.Namespace"] - -# Show a hint with possible names when a member name was not found. The aspect of -# finding the hint is based on edit distance. -missing-member-hint = true - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance = 1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices = 1 - -# Regex pattern to define which classes are considered mixins. -mixin-class-rgx = ".*[Mm]ixin" - -# List of decorators that change the signature of a decorated function. -# signature-mutators = - -[tool.pylint.variables] -# List of additional names supposed to be defined in builtins. Remember that you -# should avoid defining new builtins when possible. -# additional-builtins = - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables = true - -# List of names allowed to shadow builtins -# allowed-redefined-builtins = - -# List of strings which can identify a callback function by name. A callback name -# must start or end with one of those strings. -callbacks = ["cb_", "_cb"] - -# A regular expression matching the name of dummy variables (i.e. expected to not -# be used). -dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" - -# Argument names that match this expression will be ignored. -ignored-argument-names = "_.*|^ignored_|^unused_" - -# Tells whether we should check for unused import in __init__ files. -# init-import = - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] - - From ec2bd461fa3f9282176c765d89fd2175c60a0915 Mon Sep 17 00:00:00 2001 From: iarspider Date: Sat, 11 Nov 2023 13:26:52 +0100 Subject: [PATCH 05/46] Changes from review (2/2) --- process_pr.py | 529 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 341 insertions(+), 188 deletions(-) diff --git a/process_pr.py b/process_pr.py index 88a5fe046e6a..84cba66af5b4 100644 --- a/process_pr.py +++ b/process_pr.py @@ -8,7 +8,11 @@ CMSDIST_REPOS, ) from categories import CMSSW_CATEGORIES -from releases import RELEASE_BRANCH_MILESTONE, RELEASE_BRANCH_PRODUCTION, CMSSW_DEVEL_BRANCH +from releases import ( + RELEASE_BRANCH_MILESTONE, + RELEASE_BRANCH_PRODUCTION, + CMSSW_DEVEL_BRANCH, +) from cms_static import ( VALID_CMSDIST_BRANCHES, NEW_ISSUE_PREFIX, @@ -30,7 +34,12 @@ from os.path import join, exists, dirname from os import environ from github_utils import edit_pr, api_rate_limits -from github_utils import set_comment_emoji, get_comment_emojis, delete_comment_emoji, set_gh_user +from github_utils import ( + set_comment_emoji, + get_comment_emojis, + delete_comment_emoji, + set_gh_user, +) from socket import setdefaulttimeout from _py2with3compatibility import run_cmd from json import dumps, load, loads @@ -100,11 +109,17 @@ def format(s, **kwds): CMSSW_PACKAGE_PATTERN = "[A-Z][a-zA-Z0-9]+(/[a-zA-Z0-9]+|)" ARCH_PATTERN = "[a-z0-9]+_[a-z0-9]+_[a-z0-9]+" CMSSW_RELEASE_QUEUE_PATTERN = format( - "(%(cmssw)s|%(arch)s|%(cmssw)s/%(arch)s)", cmssw=CMSSW_QUEUE_PATTERN, arch=ARCH_PATTERN + "(%(cmssw)s|%(arch)s|%(cmssw)s/%(arch)s)", + cmssw=CMSSW_QUEUE_PATTERN, + arch=ARCH_PATTERN, ) RELVAL_OPTS = "[-][a-zA-Z0-9_.,\s/'-]+" -CLOSE_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I) -REOPEN_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I) +CLOSE_REQUEST = re.compile( + "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I +) +REOPEN_REQUEST = re.compile( + "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I +) CMS_PR_PATTERN = format( "(#[1-9][0-9]*|(%(cmsorgs)s)/+[a-zA-Z0-9_-]+#[1-9][0-9]*|https://+github.com/+(%(cmsorgs)s)/+[a-zA-Z0-9_-]+/+pull/+[1-9][0-9]*)", cmsorgs="|".join(EXTERNAL_REPOS), @@ -139,7 +154,10 @@ def format(s, **kwds): MULTILINE_COMMENTS_MAP = { "(workflow|relval)(s|)(" + EXTRA_RELVALS_TESTS_OPTS - + "|)": [format("%(workflow)s(\s*,\s*%(workflow)s|)*", workflow=WF_PATTERN), "MATRIX_EXTRAS"], + + "|)": [ + format("%(workflow)s(\s*,\s*%(workflow)s|)*", workflow=WF_PATTERN), + "MATRIX_EXTRAS", + ], "(workflow|relval)(s|)_profiling": [ format("%(workflow)s(\s*,\s*%(workflow)s|)*", workflow=WF_PATTERN), "PROFILING_WORKFLOWS", @@ -152,10 +170,16 @@ def format(s, **kwds): "disable_poison": ["true|false", "DISABLE_POISON"], "use_ib_tag": ["true|false", "USE_IB_TAG"], "baseline": ["self|default", "USE_BASELINE"], - "skip_test(s|)": [format("(%(tests)s)(\s*,\s*(%(tests)s))*", tests=SKIP_TESTS), "SKIP_TESTS"], + "skip_test(s|)": [ + format("(%(tests)s)(\s*,\s*(%(tests)s))*", tests=SKIP_TESTS), + "SKIP_TESTS", + ], "dry_run": ["true|false", "DRY_RUN"], "jenkins_(slave|node)": [JENKINS_NODES, "RUN_ON_SLAVE"], - "(arch(itecture(s|))|release|release/arch)": [CMSSW_RELEASE_QUEUE_PATTERN, "RELEASE_FORMAT"], + "(arch(itecture(s|))|release|release/arch)": [ + CMSSW_RELEASE_QUEUE_PATTERN, + "RELEASE_FORMAT", + ], ENABLE_TEST_PTRN: [ format("(%(tests)s)(\s*,\s*(%(tests)s))*", tests=EXTRA_TESTS), "ENABLE_BOT_TESTS", @@ -271,7 +295,13 @@ def read_repo_file(repo_config, repo_file, default=None): def create_properties_file_tests( - repository, pr_number, parameters, dryRun, abort=False, req_type="tests", repo_config=None + repository, + pr_number, + parameters, + dryRun, + abort=False, + req_type="tests", + repo_config=None, ): if abort: req_type = "abort" @@ -335,16 +365,24 @@ def find_last_comment(issue, user, match): if (user != comment.user.login) or (not comment.body): continue if not re.match( - match, comment.body.encode("ascii", "ignore").decode().strip("\n\t\r "), re.MULTILINE + match, + comment.body.encode("ascii", "ignore").decode().strip("\n\t\r "), + re.MULTILINE, ): continue last_comment = comment - print("Matched comment from ", comment.user.login + " with comment id ", comment.id) + print( + "Matched comment from ", + comment.user.login + " with comment id ", + comment.id, + ) return last_comment def modify_comment(comment, match, replace, dryRun): - comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" + comment_msg = ( + comment.body.encode("ascii", "ignore").decode() if comment.body else "" + ) if match: new_comment_msg = re.sub(match, replace, comment_msg) else: @@ -434,7 +472,9 @@ def check_extra_labels(first_line, extra_labels): def check_type_labels(first_line, extra_labels): ex_labels = {} rem_labels = {} - for type_cmd in [x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip()]: + for type_cmd in [ + x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip() + ]: valid_lab = False rem_lab = type_cmd[0] == "-" if type_cmd[0] in ["-", "+"]: @@ -663,7 +703,9 @@ def cmssw_file2Package(repo_config, filename): def get_jenkins_job(issue): test_line = "" - for line in [l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n")]: + for line in [ + l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n") + ]: if line.startswith("Build logs are available at:"): test_line = line if test_line: @@ -712,7 +754,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pass if not cmsbuild_user: cmsbuild_user = repo_config.CMSBUILD_USER - print("Working on ", repo.full_name, " for PR/Issue ", prId, "with admin user", cmsbuild_user) + print( + "Working on ", + repo.full_name, + " for PR/Issue ", + prId, + "with admin user", + cmsbuild_user, + ) print("Notify User: ", gh_user_char) set_gh_user(cmsbuild_user) cmssw_repo = repo_name == GH_CMSSW_REPO @@ -723,7 +772,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F create_test_property = False repo_cache = {repository: repo} packages = set([]) - chg_files = [] package_categories = {} extra_labels = {"mtype": []} add_external_category = False @@ -736,16 +784,21 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers = [] # Process Pull Request pkg_categories = set([]) - REGEX_TYPE_CMDS = "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" - REGEX_EX_CMDS = "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" % ( - repo.full_name + REGEX_TYPE_CMDS = ( + "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" + ) + REGEX_EX_CMDS = ( + "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" + % (repo.full_name) ) known_ignore_tests = "%s" % MULTILINE_COMMENTS_MAP["ignore_test(s|)"][0] REGEX_EX_IGNORE_CHKS = "^ignore\s+((%s)(\s*,\s*(%s))*|none)$" % ( known_ignore_tests, known_ignore_tests, ) - REGEX_EX_ENABLE_TESTS = "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] + REGEX_EX_ENABLE_TESTS = ( + "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] + ) L2_DATA = init_l2_data(cms_repo) last_commit_date = None last_commit_obj = None @@ -769,7 +822,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F events = {} commit_cache = {} all_commits = [] - override_too_many_commits = False + ok_too_many_commits = False warned_too_many_commits = False l2s = ", ".join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS]) @@ -796,114 +849,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # A pull request is by default closed if the branch is a closed one. if is_closed_branch(pr.base.ref): mustClose = True - # Process the changes for the given pull request so that we can determine the - # signatures it requires. - if cmssw_repo or not external_repo: - if cmssw_repo: - if pr.base.ref == "master": - signing_categories.add("code-checks") - updateMilestone(repo, issue, pr, dryRun) - chg_files = get_changed_files(repo, pr) - packages = sorted( - [x for x in set([cmssw_file2Package(repo_config, f) for f in chg_files])] - ) - for pkg_file in chg_files: - for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): - for regex in pkgs_regexp: - if regex.match(pkg_file): - extra_labels["mtype"].append(ex_lab) - print( - "Non-Blocking label:%s:%s:%s" % (ex_lab, regex.pattern, pkg_file) - ) - break - if not extra_labels["mtype"]: - del extra_labels["mtype"] - print("Extra non-blocking labels:", extra_labels) - print("First Package: ", packages[0]) - create_test_property = True - else: - add_external_category = True - packages = set(["externals/" + repository]) - ex_pkg = external_to_package(repository) - if ex_pkg: - packages.add(ex_pkg) - if (repo_org != GH_CMSSW_ORGANIZATION) or (repo_name in VALID_CMS_SW_REPOS_FOR_TESTS): - create_test_property = True - if (repo_name == GH_CMSDIST_REPO) and ( - not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) - ): - print("Skipping PR as it does not belong to valid CMSDIST branch") - return - - print("Following packages affected:") - print("\n".join(packages)) - for package in packages: - package_categories[package] = set([]) - for category in get_package_categories(package): - package_categories[package].add(category) - pkg_categories.add(category) - signing_categories.update(pkg_categories) - - # For PR, we always require tests. - signing_categories.add("tests") - if add_external_category: - signing_categories.add("externals") - if cms_repo: - print("This pull request requires ORP approval") - signing_categories.add("orp") - print("Following categories affected:") - print("\n".join(signing_categories)) - - if cmssw_repo: - # If there is a new package, add also a dummy "new" category. - all_packages = [ - package - for category_packages in list(CMSSW_CATEGORIES.values()) - for package in category_packages - ] - has_category = all([package in all_packages for package in packages]) - if not has_category: - new_package_message = "\nThe following packages do not have a category, yet:\n\n" - new_package_message += ( - "\n".join([package for package in packages if not package in all_packages]) - + "\n" - ) - new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" - print(new_package_message) - signing_categories.add("new-package") - - # Add watchers.yaml information to the WATCHERS dict. - WATCHERS = read_repo_file(repo_config, "watchers.yaml", {}) - # Given the files modified by the PR, check if there are additional developers watching one or more. - author = pr.user.login - watchers = set( - [ - user - for chg_file in chg_files - for user, watched_regexp in list(WATCHERS.items()) - for regexp in watched_regexp - if re.match("^" + regexp + ".*", chg_file) and user != author - ] - ) - # Handle category watchers - catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {}) - non_block_cats = [] if not "mtype" in extra_labels else extra_labels["mtype"] - for user, cats in list(catWatchers.items()): - for cat in cats: - if (cat in signing_categories) or (cat in non_block_cats): - print("Added ", user, " to watch due to cat", cat) - watchers.add(user) - - # Handle watchers - watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) - for watcher in [x for x in watchers]: - if not watcher in watchingGroups: - continue - watchers.remove(watcher) - watchers.update(set(watchingGroups[watcher])) - watchers = set([gh_user_char + u for u in watchers]) - print("Watchers " + ", ".join(watchers)) all_commits = get_pr_commits_reversed(pr) if all_commits: @@ -921,7 +867,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F cms_status_prefix = "cms/%s" % prId bot_status = get_status(bot_status_name, commit_statuses) code_checks_status = [ - s for s in commit_statuses if s.context == "%s/code-checks" % cms_status_prefix + s + for s in commit_statuses + if s.context == "%s/code-checks" % cms_status_prefix ] print("PR Statuses:", commit_statuses) print(len(commit_statuses)) @@ -932,7 +880,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F " at ", last_commit_date, ) - print("Latest commit message: ", last_commit.message.encode("ascii", "ignore").decode()) + print( + "Latest commit message: ", + last_commit.message.encode("ascii", "ignore").decode(), + ) print("Latest commit sha: ", last_commit.sha) print("PR update time", pr.updated_at) print("Time UTC:", datetime.utcnow()) @@ -944,7 +895,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if (not dryRun) and add_labels: - labels = [x.name.encode("ascii", "ignore").decode() for x in issue.labels] + labels = [ + x.name.encode("ascii", "ignore").decode() for x in issue.labels + ] if not "future-commit" in labels: labels.append("future-commit") issue.edit(labels=labels) @@ -967,7 +920,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if repository == CMSSW_REPO_NAME and re.match(CREATE_REPO, issue.title): - with open("query-new-data-repo-issues-" + str(issue.number) + ".properties", "w") as f: + with open( + "query-new-data-repo-issues-" + str(issue.number) + ".properties", "w" + ) as f: f.write("ISSUE_NUMBER=" + str(issue.number) + "\n") # Process the issue comments @@ -1020,12 +975,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F commenter_categories = get_commenter_categories( commenter, int(comment.created_at.strftime("%s")) ) - valid_commenter = (commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org]) or ( - len(commenter_categories) > 0 - ) + valid_commenter = ( + commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org] + ) or (len(commenter_categories) > 0) if (not valid_commenter) and (requestor != commenter): continue - comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" + comment_msg = ( + comment.body.encode("ascii", "ignore").decode() if comment.body else "" + ) # The first line is an invariant. comment_lines = [l.strip() for l in comment_msg.split("\n") if l.strip()] first_line = comment_lines[0:1] @@ -1049,15 +1006,21 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): technical_comment = comment - if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: + if ( + commenter == cmsbuild_user + ) and "This PR contains too many commits" in first_line: warned_too_many_commits = True - if commenter in l2s and re.match("^\s*" + REGEX_IGNORE_COMMIT_COUNT + "\s*$", first_line): - override_too_many_commits = True + if commenter in l2s and re.match( + "^\s*" + REGEX_IGNORE_COMMIT_COUNT + "\s*$", first_line + ): + ok_too_many_commits = True assign_type, new_cats = get_assign_categories(first_line) if new_cats: - if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user): + if (assign_type == "new categories assigned:") and ( + commenter == cmsbuild_user + ): for ex_cat in new_cats: if ex_cat in assign_cats: assign_cats[ex_cat] = 1 @@ -1079,7 +1042,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Some of the special users can say "hold" prevent automatic merging of # fully signed PRs. if re.match("^hold$", first_line, re.I): - if commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): + if commenter_categories or ( + commenter in releaseManagers + PR_HOLD_MANAGERS + ): hold[commenter] = 1 continue if re.match(REGEX_EX_CMDS, first_line, re.I): @@ -1100,7 +1065,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F continue if re.match(REGEX_EX_ENABLE_TESTS, first_line, re.I): if valid_commenter: - enable_tests, ignore = check_enable_bot_tests(first_line.split(" ", 1)[-1]) + enable_tests, ignore = check_enable_bot_tests( + first_line.split(" ", 1)[-1] + ) if not dryRun: set_comment_emoji(comment.id, repository, emoji="+1") continue @@ -1115,11 +1082,15 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if re.match("^unhold$", first_line, re.I): if "orp" in commenter_categories: hold = {} - elif commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): + elif commenter_categories or ( + commenter in releaseManagers + PR_HOLD_MANAGERS + ): if commenter in hold: del hold[commenter] continue - if (commenter == cmsbuild_user) and (re.match("^" + HOLD_MSG + ".+", first_line)): + if (commenter == cmsbuild_user) and ( + re.match("^" + HOLD_MSG + ".+", first_line) + ): for u in first_line.split(HOLD_MSG, 2)[1].split(","): u = u.strip().lstrip("@") if u in hold: @@ -1138,14 +1109,18 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F (not issue.pull_request) and (commenter in CMSSW_ISSUES_TRACKERS) ): mustClose = False - if (issue.state == "closed") and (comment.created_at >= issue.closed_at): + if (issue.state == "closed") and ( + comment.created_at >= issue.closed_at + ): reOpen = True print("==>Reopen request received from %s" % commenter) continue if valid_commenter: - valid_multiline_comment, test_params, test_params_m = multiline_check_function( - first_line, comment_lines, repository - ) + ( + valid_multiline_comment, + test_params, + test_params_m, + ) = multiline_check_function(first_line, comment_lines, repository) if test_params_m: test_params_msg = str(comment.id) + ":" + test_params_m test_params_comment = comment @@ -1154,7 +1129,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params = dict(test_params) if "ENABLE_BOT_TESTS" in global_test_params: enable_tests = global_test_params["ENABLE_BOT_TESTS"] - test_params_msg = str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) + test_params_msg = ( + str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) + ) continue if cmssw_repo: @@ -1209,7 +1186,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[first_line] = "pending" if first_line not in pre_checks + extra_pre_checks: extra_pre_checks.append(first_line) - if code_checks_status and (code_checks_status[0].updated_at >= comment.created_at): + if code_checks_status and ( + code_checks_status[0].updated_at >= comment.created_at + ): continue if first_line in pre_checks: if pre_checks_state["code-checks"] in ["pending", ""]: @@ -1245,7 +1224,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F IGNORING_TESTS_MSG, first_line ): signatures["tests"] = "pending" - elif re.match("Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line): + elif re.match( + "Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line + ): pull_request_updated = False elif re.match(TRIGERING_TESTS_MSG, first_line) or re.match( TRIGERING_TESTS_MSG1, first_line @@ -1276,7 +1257,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if "+1" in first_line: signatures["tests"] = "approved" comp_warnings = ( - len([1 for l in comment_lines if "Compilation Warnings: Yes" in l]) > 0 + len( + [ + 1 + for l in comment_lines + if "Compilation Warnings: Yes" in l + ] + ) + > 0 ) pre_checks_url["tests"] = comment.html_url elif "-1" in first_line: @@ -1291,9 +1279,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if issue.pull_request or push_test_issue: # Check if the release manager asked for merging this. - if ((commenter in releaseManagers) or ("orp" in commenter_categories)) and re.match( - "^\s*(merge)\s*$", first_line, re.I - ): + if ( + (commenter in releaseManagers) or ("orp" in commenter_categories) + ) and re.match("^\s*(merge)\s*$", first_line, re.I): mustMerge = True mustClose = False if ("orp" in commenter_categories) and ("orp" in signatures): @@ -1302,7 +1290,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Check if the someone asked to trigger the tests if valid_commenter: - ok, v2, v3, v4 = check_test_cmd(first_line, repository, global_test_params) + ok, v2, v3, v4 = check_test_cmd( + first_line, repository, global_test_params + ) if ok: test_comment = comment abort_test = None @@ -1324,7 +1314,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Comment message:", first_line) signatures["tests"] = "pending" continue - elif REGEX_TEST_ABORT.match(first_line) and (signatures["tests"] == "pending"): + elif REGEX_TEST_ABORT.match(first_line) and ( + signatures["tests"] == "pending" + ): abort_test = comment test_comment = None signatures["tests"] = "pending" @@ -1342,9 +1334,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F set_comment_emoji(comment.id, repository) # end of parsing comments section + if ( (not warned_too_many_commits) - and not override_too_many_commits + and not ok_too_many_commits and len(all_commits) >= MAX_INITIAL_COMMITS_IN_PR ): issue.create_comment( @@ -1353,10 +1346,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F "\n{l2s}, you can override this check with `+commit-count`." ) - if len(all_commits) < MAX_INITIAL_COMMITS_IN_PR or override_too_many_commits: - for commit in all_commits: - events[commit.commit.committer.date] = get_changed_files_in_commit(repo, commit) - # Get the commit cache from `already_seen` commit or technical commit print("Recalculating signatures") cache_comment = None @@ -1378,18 +1367,23 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F "", old_body ) - for commit in all_commits: - if commit.sha not in commit_cache: - commit_cache[commit.sha] = { - "time": int(commit.commit.committer.date.timestamp()), - "files": get_changed_files_in_commit(repo, commit), + chg_files = set() + + if len(all_commits) < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: + for commit in all_commits: + if commit.sha not in commit_cache: + commit_cache[commit.sha] = { + "time": int(commit.commit.committer.date.timestamp()), + "files": get_changed_files_in_commit(repo, commit), + } + + cache_entry = commit_cache[commit.sha] + events[datetime.fromtimestamp(cache_entry["time"])] = { + "type": "commit", + "value": cache_entry["files"], } - cache_entry = commit_cache[commit.sha] - events[datetime.fromtimestamp(cache_entry["time"])] = { - "type": "commit", - "value": cache_entry["files"], - } + chg_files.update(cache_entry["files"]) if not dryRun: if cache_comment: @@ -1407,13 +1401,133 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F events = dict(sorted(events.items())) + if issue.pull_request: + pr = repo.get_pull(prId) + + # Process the changes for the given pull request so that we can determine the + # signatures it requires. + if cmssw_repo or not external_repo: + if cmssw_repo: + if pr.base.ref == "master": + signing_categories.add("code-checks") + updateMilestone(repo, issue, pr, dryRun) + + packages = sorted( + list(set(cmssw_file2Package(repo_config, f) for f in chg_files)) + ) + for pkg_file in chg_files: + for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): + for regex in pkgs_regexp: + if regex.match(pkg_file): + extra_labels["mtype"].append(ex_lab) + print( + "Non-Blocking label:%s:%s:%s" + % (ex_lab, regex.pattern, pkg_file) + ) + break + if not extra_labels["mtype"]: + del extra_labels["mtype"] + print("Extra non-blocking labels:", extra_labels) + print("First Package: ", packages[0]) + create_test_property = True + else: + add_external_category = True + packages = {"externals/" + repository} + ex_pkg = external_to_package(repository) + if ex_pkg: + packages.add(ex_pkg) + if (repo_org != GH_CMSSW_ORGANIZATION) or ( + repo_name in VALID_CMS_SW_REPOS_FOR_TESTS + ): + create_test_property = True + if (repo_name == GH_CMSDIST_REPO) and ( + not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) + ): + print("Skipping PR as it does not belong to valid CMSDIST branch") + return + + print("Following packages affected:") + print("\n".join(packages)) + for package in packages: + package_categories[package] = set([]) + for category in get_package_categories(package): + package_categories[package].add(category) + pkg_categories.add(category) + signing_categories.update(pkg_categories) + + # For PR, we always require tests. + signing_categories.add("tests") + if add_external_category: + signing_categories.add("externals") + if cms_repo: + print("This pull request requires ORP approval") + signing_categories.add("orp") + + print("Following categories affected:") + print("\n".join(signing_categories)) + + if cmssw_repo: + # If there is a new package, add also a dummy "new" category. + all_packages = [ + package + for category_packages in list(CMSSW_CATEGORIES.values()) + for package in category_packages + ] + has_category = all([package in all_packages for package in packages]) + if not has_category: + new_package_message = ( + "\nThe following packages do not have a category, yet:\n\n" + ) + new_package_message += ( + "\n".join( + [package for package in packages if not package in all_packages] + ) + + "\n" + ) + new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" + print(new_package_message) + signing_categories.add("new-package") + + # Add watchers.yaml information to the WATCHERS dict. + WATCHERS = read_repo_file(repo_config, "watchers.yaml", {}) + # Given the files modified by the PR, check if there are additional developers watching one or more. + author = pr.user.login + watchers = set( + [ + user + for chg_file in chg_files + for user, watched_regexp in list(WATCHERS.items()) + for regexp in watched_regexp + if re.match("^" + regexp + ".*", chg_file) and user != author + ] + ) + # Handle category watchers + catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {}) + non_block_cats = [] if not "mtype" in extra_labels else extra_labels["mtype"] + for user, cats in list(catWatchers.items()): + for cat in cats: + if (cat in signing_categories) or (cat in non_block_cats): + print("Added ", user, " to watch due to cat", cat) + watchers.add(user) + + # Handle watchers + watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) + for watcher in [x for x in watchers]: + if not watcher in watchingGroups: + continue + watchers.remove(watcher) + watchers.update(set(watchingGroups[watcher])) + watchers = set([gh_user_char + u for u in watchers]) + print("Watchers " + ", ".join(watchers)) + for event in events.values(): if event["type"] == "sign": for cat, sign in event["value"].items(): signatures[cat] = sign elif event["type"] == "commit": chg_categories = [ - x for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) + x + for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) ] signatures["orp"] = "pending" for cat in chg_categories: @@ -1516,11 +1630,19 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print(desc) if not dryRun: last_commit_obj.create_status( - "success", description=desc, target_url=turl, context=bot_status_name + "success", + description=desc, + target_url=turl, + context=bot_status_name, ) set_comment_emoji(test_comment.id, repository) if bot_status: - print(bot_status.target_url, turl, signatures["tests"], bot_status.description) + print( + bot_status.target_url, + turl, + signatures["tests"], + bot_status.description, + ) if ( bot_status and bot_status.target_url == turl @@ -1529,7 +1651,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): signatures["tests"] = "started" if ( - get_status_state("%s/unknown/release" % cms_status_prefix, commit_statuses) + get_status_state( + "%s/unknown/release" % cms_status_prefix, commit_statuses + ) == "error" ): signatures["tests"] = "pending" @@ -1552,7 +1676,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F for s in [ i for i in commit_statuses - if ((i.context == scontext) or (i.context.startswith(scontext + "/"))) + if ( + (i.context == scontext) + or (i.context.startswith(scontext + "/")) + ) ]: if (not result_url) and ("/jenkins-artifacts/" in s.target_url): xdata = s.target_url.split("/") @@ -1673,10 +1800,15 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if already_seen: if dryRun: - print("Update PR seen message to include backport PR number", backport_pr_num) + print( + "Update PR seen message to include backport PR number", + backport_pr_num, + ) else: new_msg = "" - for l in already_seen.body.encode("ascii", "ignore").decode().split("\n"): + for l in ( + already_seen.body.encode("ascii", "ignore").decode().split("\n") + ): if BACKPORT_STR in l: continue new_msg += l + "\n" @@ -1825,7 +1957,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # get release managers SUPER_USERS = read_repo_file(repo_config, "super-users.yaml", []) - releaseManagersList = ", ".join([gh_user_char + x for x in set(releaseManagers + SUPER_USERS)]) + releaseManagersList = ", ".join( + [gh_user_char + x for x in set(releaseManagers + SUPER_USERS)] + ) if cmssw_prs: global_test_params["PULL_REQUESTS"] = cmssw_prs @@ -1861,11 +1995,20 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params["CONTEXT_PREFIX"] = cms_status_prefix if trigger_test: create_properties_file_tests( - repository, prId, global_test_params, dryRun, abort=False, repo_config=repo_config + repository, + prId, + global_test_params, + dryRun, + abort=False, + repo_config=repo_config, ) if not dryRun: set_comment_emoji(test_comment.id, repository) - elif abort_test and bot_status and (not bot_status.description.startswith("Aborted")): + elif ( + abort_test + and bot_status + and (not bot_status.description.startswith("Aborted")) + ): if not has_user_emoji(abort_test, repository, "+1", cmsbuild_user): create_properties_file_tests( repository, prId, global_test_params, dryRun, abort=True @@ -1947,7 +2090,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F gh_user_char + name for name, l2_categories in list(CMSSW_L2.items()) for signature in signing_categories - if signature in l2_categories and signature in unsigned and signature not in ["orp"] + if signature in l2_categories + and signature in unsigned + and signature not in ["orp"] ] missing_notifications = set(missing_notifications) @@ -1987,7 +2132,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pkg_msg = [] for pkg in packages: if pkg in package_categories: - pkg_msg.append("- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg]))) + pkg_msg.append( + "- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg])) + ) else: pkg_msg.append("- %s (**new**)" % pkg) messageNewPR = format( @@ -2075,7 +2222,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pre_check not in signatures: signatures[pre_check] = "pending" print( - "PRE CHECK: %s,%s,%s" % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) + "PRE CHECK: %s,%s,%s" + % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) ) if signatures[pre_check] != "pending": if pre_checks_state[pre_check] in ["pending", ""]: @@ -2091,7 +2239,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) continue if (not dryRunOrig) and (pre_checks_state[pre_check] == ""): - params = {"PULL_REQUEST": "%s" % (prId), "CONTEXT_PREFIX": cms_status_prefix} + params = { + "PULL_REQUEST": "%s" % (prId), + "CONTEXT_PREFIX": cms_status_prefix, + } if pre_check == "code-checks": params["CMSSW_TOOL_CONF"] = code_checks_tools params["APPLY_PATCH"] = str(code_check_apply_patch).lower() @@ -2123,7 +2274,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F mustMerge = True else: print("This pull request will not be automatically merged.") - if mustMerge == True: + if mustMerge: print("This pull request must be merged.") if not dryRun and (pr.state == "open"): pr.merge() @@ -2144,7 +2295,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if not dryRun: emoji = "-1" if "ERRORS: " in test_params_msg else "+1" if e and (e["content"] != emoji): - delete_comment_emoji(str(e["id"]), test_params_comment.id, repository) + delete_comment_emoji( + str(e["id"]), test_params_comment.id, repository + ) state = "success" if emoji == "+1" else "error" last_commit_obj.create_status( state, From 5942437132c9735561382953845a3999a6cb6529 Mon Sep 17 00:00:00 2001 From: iarspider Date: Sat, 11 Nov 2023 15:06:19 +0100 Subject: [PATCH 06/46] Code-style --- process_pr.py | 182 +++++++++++++------------------------------------- 1 file changed, 48 insertions(+), 134 deletions(-) diff --git a/process_pr.py b/process_pr.py index 84cba66af5b4..29a62ec01cc3 100644 --- a/process_pr.py +++ b/process_pr.py @@ -114,12 +114,8 @@ def format(s, **kwds): arch=ARCH_PATTERN, ) RELVAL_OPTS = "[-][a-zA-Z0-9_.,\s/'-]+" -CLOSE_REQUEST = re.compile( - "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I -) -REOPEN_REQUEST = re.compile( - "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I -) +CLOSE_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I) +REOPEN_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I) CMS_PR_PATTERN = format( "(#[1-9][0-9]*|(%(cmsorgs)s)/+[a-zA-Z0-9_-]+#[1-9][0-9]*|https://+github.com/+(%(cmsorgs)s)/+[a-zA-Z0-9_-]+/+pull/+[1-9][0-9]*)", cmsorgs="|".join(EXTERNAL_REPOS), @@ -380,9 +376,7 @@ def find_last_comment(issue, user, match): def modify_comment(comment, match, replace, dryRun): - comment_msg = ( - comment.body.encode("ascii", "ignore").decode() if comment.body else "" - ) + comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" if match: new_comment_msg = re.sub(match, replace, comment_msg) else: @@ -472,9 +466,7 @@ def check_extra_labels(first_line, extra_labels): def check_type_labels(first_line, extra_labels): ex_labels = {} rem_labels = {} - for type_cmd in [ - x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip() - ]: + for type_cmd in [x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip()]: valid_lab = False rem_lab = type_cmd[0] == "-" if type_cmd[0] in ["-", "+"]: @@ -703,9 +695,7 @@ def cmssw_file2Package(repo_config, filename): def get_jenkins_job(issue): test_line = "" - for line in [ - l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n") - ]: + for line in [l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n")]: if line.startswith("Build logs are available at:"): test_line = line if test_line: @@ -784,21 +774,16 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers = [] # Process Pull Request pkg_categories = set([]) - REGEX_TYPE_CMDS = ( - "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" - ) - REGEX_EX_CMDS = ( - "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" - % (repo.full_name) + REGEX_TYPE_CMDS = "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" + REGEX_EX_CMDS = "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" % ( + repo.full_name ) known_ignore_tests = "%s" % MULTILINE_COMMENTS_MAP["ignore_test(s|)"][0] REGEX_EX_IGNORE_CHKS = "^ignore\s+((%s)(\s*,\s*(%s))*|none)$" % ( known_ignore_tests, known_ignore_tests, ) - REGEX_EX_ENABLE_TESTS = ( - "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] - ) + REGEX_EX_ENABLE_TESTS = "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] L2_DATA = init_l2_data(cms_repo) last_commit_date = None last_commit_obj = None @@ -867,9 +852,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F cms_status_prefix = "cms/%s" % prId bot_status = get_status(bot_status_name, commit_statuses) code_checks_status = [ - s - for s in commit_statuses - if s.context == "%s/code-checks" % cms_status_prefix + s for s in commit_statuses if s.context == "%s/code-checks" % cms_status_prefix ] print("PR Statuses:", commit_statuses) print(len(commit_statuses)) @@ -895,9 +878,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if (not dryRun) and add_labels: - labels = [ - x.name.encode("ascii", "ignore").decode() for x in issue.labels - ] + labels = [x.name.encode("ascii", "ignore").decode() for x in issue.labels] if not "future-commit" in labels: labels.append("future-commit") issue.edit(labels=labels) @@ -920,9 +901,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if repository == CMSSW_REPO_NAME and re.match(CREATE_REPO, issue.title): - with open( - "query-new-data-repo-issues-" + str(issue.number) + ".properties", "w" - ) as f: + with open("query-new-data-repo-issues-" + str(issue.number) + ".properties", "w") as f: f.write("ISSUE_NUMBER=" + str(issue.number) + "\n") # Process the issue comments @@ -975,14 +954,12 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F commenter_categories = get_commenter_categories( commenter, int(comment.created_at.strftime("%s")) ) - valid_commenter = ( - commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org] - ) or (len(commenter_categories) > 0) + valid_commenter = (commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org]) or ( + len(commenter_categories) > 0 + ) if (not valid_commenter) and (requestor != commenter): continue - comment_msg = ( - comment.body.encode("ascii", "ignore").decode() if comment.body else "" - ) + comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" # The first line is an invariant. comment_lines = [l.strip() for l in comment_msg.split("\n") if l.strip()] first_line = comment_lines[0:1] @@ -1006,21 +983,15 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): technical_comment = comment - if ( - commenter == cmsbuild_user - ) and "This PR contains too many commits" in first_line: + if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: warned_too_many_commits = True - if commenter in l2s and re.match( - "^\s*" + REGEX_IGNORE_COMMIT_COUNT + "\s*$", first_line - ): + if commenter in l2s and re.match("^\s*" + REGEX_IGNORE_COMMIT_COUNT + "\s*$", first_line): ok_too_many_commits = True assign_type, new_cats = get_assign_categories(first_line) if new_cats: - if (assign_type == "new categories assigned:") and ( - commenter == cmsbuild_user - ): + if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user): for ex_cat in new_cats: if ex_cat in assign_cats: assign_cats[ex_cat] = 1 @@ -1042,9 +1013,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Some of the special users can say "hold" prevent automatic merging of # fully signed PRs. if re.match("^hold$", first_line, re.I): - if commenter_categories or ( - commenter in releaseManagers + PR_HOLD_MANAGERS - ): + if commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): hold[commenter] = 1 continue if re.match(REGEX_EX_CMDS, first_line, re.I): @@ -1065,9 +1034,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F continue if re.match(REGEX_EX_ENABLE_TESTS, first_line, re.I): if valid_commenter: - enable_tests, ignore = check_enable_bot_tests( - first_line.split(" ", 1)[-1] - ) + enable_tests, ignore = check_enable_bot_tests(first_line.split(" ", 1)[-1]) if not dryRun: set_comment_emoji(comment.id, repository, emoji="+1") continue @@ -1082,15 +1049,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if re.match("^unhold$", first_line, re.I): if "orp" in commenter_categories: hold = {} - elif commenter_categories or ( - commenter in releaseManagers + PR_HOLD_MANAGERS - ): + elif commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): if commenter in hold: del hold[commenter] continue - if (commenter == cmsbuild_user) and ( - re.match("^" + HOLD_MSG + ".+", first_line) - ): + if (commenter == cmsbuild_user) and (re.match("^" + HOLD_MSG + ".+", first_line)): for u in first_line.split(HOLD_MSG, 2)[1].split(","): u = u.strip().lstrip("@") if u in hold: @@ -1109,9 +1072,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F (not issue.pull_request) and (commenter in CMSSW_ISSUES_TRACKERS) ): mustClose = False - if (issue.state == "closed") and ( - comment.created_at >= issue.closed_at - ): + if (issue.state == "closed") and (comment.created_at >= issue.closed_at): reOpen = True print("==>Reopen request received from %s" % commenter) continue @@ -1129,9 +1090,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params = dict(test_params) if "ENABLE_BOT_TESTS" in global_test_params: enable_tests = global_test_params["ENABLE_BOT_TESTS"] - test_params_msg = ( - str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) - ) + test_params_msg = str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) continue if cmssw_repo: @@ -1186,9 +1145,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[first_line] = "pending" if first_line not in pre_checks + extra_pre_checks: extra_pre_checks.append(first_line) - if code_checks_status and ( - code_checks_status[0].updated_at >= comment.created_at - ): + if code_checks_status and (code_checks_status[0].updated_at >= comment.created_at): continue if first_line in pre_checks: if pre_checks_state["code-checks"] in ["pending", ""]: @@ -1224,9 +1181,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F IGNORING_TESTS_MSG, first_line ): signatures["tests"] = "pending" - elif re.match( - "Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line - ): + elif re.match("Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line): pull_request_updated = False elif re.match(TRIGERING_TESTS_MSG, first_line) or re.match( TRIGERING_TESTS_MSG1, first_line @@ -1257,14 +1212,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if "+1" in first_line: signatures["tests"] = "approved" comp_warnings = ( - len( - [ - 1 - for l in comment_lines - if "Compilation Warnings: Yes" in l - ] - ) - > 0 + len([1 for l in comment_lines if "Compilation Warnings: Yes" in l]) > 0 ) pre_checks_url["tests"] = comment.html_url elif "-1" in first_line: @@ -1279,9 +1227,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if issue.pull_request or push_test_issue: # Check if the release manager asked for merging this. - if ( - (commenter in releaseManagers) or ("orp" in commenter_categories) - ) and re.match("^\s*(merge)\s*$", first_line, re.I): + if ((commenter in releaseManagers) or ("orp" in commenter_categories)) and re.match( + "^\s*(merge)\s*$", first_line, re.I + ): mustMerge = True mustClose = False if ("orp" in commenter_categories) and ("orp" in signatures): @@ -1290,9 +1238,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Check if the someone asked to trigger the tests if valid_commenter: - ok, v2, v3, v4 = check_test_cmd( - first_line, repository, global_test_params - ) + ok, v2, v3, v4 = check_test_cmd(first_line, repository, global_test_params) if ok: test_comment = comment abort_test = None @@ -1314,9 +1260,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Comment message:", first_line) signatures["tests"] = "pending" continue - elif REGEX_TEST_ABORT.match(first_line) and ( - signatures["tests"] == "pending" - ): + elif REGEX_TEST_ABORT.match(first_line) and (signatures["tests"] == "pending"): abort_test = comment test_comment = None signatures["tests"] = "pending" @@ -1412,17 +1356,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signing_categories.add("code-checks") updateMilestone(repo, issue, pr, dryRun) - packages = sorted( - list(set(cmssw_file2Package(repo_config, f) for f in chg_files)) - ) + packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) for pkg_file in chg_files: for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): for regex in pkgs_regexp: if regex.match(pkg_file): extra_labels["mtype"].append(ex_lab) print( - "Non-Blocking label:%s:%s:%s" - % (ex_lab, regex.pattern, pkg_file) + "Non-Blocking label:%s:%s:%s" % (ex_lab, regex.pattern, pkg_file) ) break if not extra_labels["mtype"]: @@ -1436,9 +1377,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ex_pkg = external_to_package(repository) if ex_pkg: packages.add(ex_pkg) - if (repo_org != GH_CMSSW_ORGANIZATION) or ( - repo_name in VALID_CMS_SW_REPOS_FOR_TESTS - ): + if (repo_org != GH_CMSSW_ORGANIZATION) or (repo_name in VALID_CMS_SW_REPOS_FOR_TESTS): create_test_property = True if (repo_name == GH_CMSDIST_REPO) and ( not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) @@ -1475,13 +1414,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ] has_category = all([package in all_packages for package in packages]) if not has_category: - new_package_message = ( - "\nThe following packages do not have a category, yet:\n\n" - ) + new_package_message = "\nThe following packages do not have a category, yet:\n\n" new_package_message += ( - "\n".join( - [package for package in packages if not package in all_packages] - ) + "\n".join([package for package in packages if not package in all_packages]) + "\n" ) new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" @@ -1526,8 +1461,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[cat] = sign elif event["type"] == "commit": chg_categories = [ - x - for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) + x for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) ] signatures["orp"] = "pending" for cat in chg_categories: @@ -1651,9 +1585,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): signatures["tests"] = "started" if ( - get_status_state( - "%s/unknown/release" % cms_status_prefix, commit_statuses - ) + get_status_state("%s/unknown/release" % cms_status_prefix, commit_statuses) == "error" ): signatures["tests"] = "pending" @@ -1676,10 +1608,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F for s in [ i for i in commit_statuses - if ( - (i.context == scontext) - or (i.context.startswith(scontext + "/")) - ) + if ((i.context == scontext) or (i.context.startswith(scontext + "/"))) ]: if (not result_url) and ("/jenkins-artifacts/" in s.target_url): xdata = s.target_url.split("/") @@ -1806,9 +1735,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) else: new_msg = "" - for l in ( - already_seen.body.encode("ascii", "ignore").decode().split("\n") - ): + for l in already_seen.body.encode("ascii", "ignore").decode().split("\n"): if BACKPORT_STR in l: continue new_msg += l + "\n" @@ -1957,9 +1884,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # get release managers SUPER_USERS = read_repo_file(repo_config, "super-users.yaml", []) - releaseManagersList = ", ".join( - [gh_user_char + x for x in set(releaseManagers + SUPER_USERS)] - ) + releaseManagersList = ", ".join([gh_user_char + x for x in set(releaseManagers + SUPER_USERS)]) if cmssw_prs: global_test_params["PULL_REQUESTS"] = cmssw_prs @@ -2004,11 +1929,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) if not dryRun: set_comment_emoji(test_comment.id, repository) - elif ( - abort_test - and bot_status - and (not bot_status.description.startswith("Aborted")) - ): + elif abort_test and bot_status and (not bot_status.description.startswith("Aborted")): if not has_user_emoji(abort_test, repository, "+1", cmsbuild_user): create_properties_file_tests( repository, prId, global_test_params, dryRun, abort=True @@ -2090,9 +2011,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F gh_user_char + name for name, l2_categories in list(CMSSW_L2.items()) for signature in signing_categories - if signature in l2_categories - and signature in unsigned - and signature not in ["orp"] + if signature in l2_categories and signature in unsigned and signature not in ["orp"] ] missing_notifications = set(missing_notifications) @@ -2132,9 +2051,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pkg_msg = [] for pkg in packages: if pkg in package_categories: - pkg_msg.append( - "- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg])) - ) + pkg_msg.append("- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg]))) else: pkg_msg.append("- %s (**new**)" % pkg) messageNewPR = format( @@ -2222,8 +2139,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pre_check not in signatures: signatures[pre_check] = "pending" print( - "PRE CHECK: %s,%s,%s" - % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) + "PRE CHECK: %s,%s,%s" % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) ) if signatures[pre_check] != "pending": if pre_checks_state[pre_check] in ["pending", ""]: @@ -2295,9 +2211,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if not dryRun: emoji = "-1" if "ERRORS: " in test_params_msg else "+1" if e and (e["content"] != emoji): - delete_comment_emoji( - str(e["id"]), test_params_comment.id, repository - ) + delete_comment_emoji(str(e["id"]), test_params_comment.id, repository) state = "success" if emoji == "+1" else "error" last_commit_obj.create_status( state, From 161a818cb7bf60bf733b9d4d43ac30815371b94d Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 13 Nov 2023 11:37:50 +0100 Subject: [PATCH 07/46] Fixes after tests --- process_pr.py | 256 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 172 insertions(+), 84 deletions(-) diff --git a/process_pr.py b/process_pr.py index 29a62ec01cc3..3968ded41f2e 100644 --- a/process_pr.py +++ b/process_pr.py @@ -114,8 +114,12 @@ def format(s, **kwds): arch=ARCH_PATTERN, ) RELVAL_OPTS = "[-][a-zA-Z0-9_.,\s/'-]+" -CLOSE_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I) -REOPEN_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I) +CLOSE_REQUEST = re.compile( + "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I +) +REOPEN_REQUEST = re.compile( + "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I +) CMS_PR_PATTERN = format( "(#[1-9][0-9]*|(%(cmsorgs)s)/+[a-zA-Z0-9_-]+#[1-9][0-9]*|https://+github.com/+(%(cmsorgs)s)/+[a-zA-Z0-9_-]+/+pull/+[1-9][0-9]*)", cmsorgs="|".join(EXTERNAL_REPOS), @@ -376,7 +380,9 @@ def find_last_comment(issue, user, match): def modify_comment(comment, match, replace, dryRun): - comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" + comment_msg = ( + comment.body.encode("ascii", "ignore").decode() if comment.body else "" + ) if match: new_comment_msg = re.sub(match, replace, comment_msg) else: @@ -466,7 +472,9 @@ def check_extra_labels(first_line, extra_labels): def check_type_labels(first_line, extra_labels): ex_labels = {} rem_labels = {} - for type_cmd in [x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip()]: + for type_cmd in [ + x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip() + ]: valid_lab = False rem_lab = type_cmd[0] == "-" if type_cmd[0] in ["-", "+"]: @@ -695,7 +703,9 @@ def cmssw_file2Package(repo_config, filename): def get_jenkins_job(issue): test_line = "" - for line in [l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n")]: + for line in [ + l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n") + ]: if line.startswith("Build logs are available at:"): test_line = line if test_line: @@ -774,16 +784,21 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers = [] # Process Pull Request pkg_categories = set([]) - REGEX_TYPE_CMDS = "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" - REGEX_EX_CMDS = "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" % ( - repo.full_name + REGEX_TYPE_CMDS = ( + "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" + ) + REGEX_EX_CMDS = ( + "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" + % (repo.full_name) ) known_ignore_tests = "%s" % MULTILINE_COMMENTS_MAP["ignore_test(s|)"][0] REGEX_EX_IGNORE_CHKS = "^ignore\s+((%s)(\s*,\s*(%s))*|none)$" % ( known_ignore_tests, known_ignore_tests, ) - REGEX_EX_ENABLE_TESTS = "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] + REGEX_EX_ENABLE_TESTS = ( + "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] + ) L2_DATA = init_l2_data(cms_repo) last_commit_date = None last_commit_obj = None @@ -852,10 +867,12 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F cms_status_prefix = "cms/%s" % prId bot_status = get_status(bot_status_name, commit_statuses) code_checks_status = [ - s for s in commit_statuses if s.context == "%s/code-checks" % cms_status_prefix + s + for s in commit_statuses + if s.context == "%s/code-checks" % cms_status_prefix ] print("PR Statuses:", commit_statuses) - print(len(commit_statuses)) + print("#PR Statuses:", len(commit_statuses)) last_commit_date = last_commit.committer.date print( "Latest commit by ", @@ -878,7 +895,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if (not dryRun) and add_labels: - labels = [x.name.encode("ascii", "ignore").decode() for x in issue.labels] + labels = [ + x.name.encode("ascii", "ignore").decode() for x in issue.labels + ] if not "future-commit" in labels: labels.append("future-commit") issue.edit(labels=labels) @@ -901,7 +920,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if repository == CMSSW_REPO_NAME and re.match(CREATE_REPO, issue.title): - with open("query-new-data-repo-issues-" + str(issue.number) + ".properties", "w") as f: + with open( + "query-new-data-repo-issues-" + str(issue.number) + ".properties", "w" + ) as f: f.write("ISSUE_NUMBER=" + str(issue.number) + "\n") # Process the issue comments @@ -954,12 +975,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F commenter_categories = get_commenter_categories( commenter, int(comment.created_at.strftime("%s")) ) - valid_commenter = (commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org]) or ( - len(commenter_categories) > 0 - ) + valid_commenter = ( + commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org] + ) or (len(commenter_categories) > 0) if (not valid_commenter) and (requestor != commenter): continue - comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" + comment_msg = ( + comment.body.encode("ascii", "ignore").decode() if comment.body else "" + ) # The first line is an invariant. comment_lines = [l.strip() for l in comment_msg.split("\n") if l.strip()] first_line = comment_lines[0:1] @@ -983,22 +1006,28 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): technical_comment = comment - if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: + if ( + commenter == cmsbuild_user + ) and "This PR contains too many commits" in first_line: warned_too_many_commits = True - if commenter in l2s and re.match("^\s*" + REGEX_IGNORE_COMMIT_COUNT + "\s*$", first_line): + if commenter in l2s and re.match( + r"^\s*" + REGEX_IGNORE_COMMIT_COUNT + r"\s*$", first_line + ): ok_too_many_commits = True assign_type, new_cats = get_assign_categories(first_line) if new_cats: - if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user): + if (assign_type == "new categories assigned:") and ( + commenter == cmsbuild_user + ): for ex_cat in new_cats: if ex_cat in assign_cats: assign_cats[ex_cat] = 1 if commenter_categories or (commenter in CMSSW_ISSUES_TRACKERS): if assign_type == "assign": for ex_cat in new_cats: - if not ex_cat in signing_categories: + if ex_cat not in signing_categories: assign_cats[ex_cat] = 0 signing_categories.add(ex_cat) signatures[ex_cat] = "pending" @@ -1013,7 +1042,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Some of the special users can say "hold" prevent automatic merging of # fully signed PRs. if re.match("^hold$", first_line, re.I): - if commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): + if commenter_categories or ( + commenter in releaseManagers + PR_HOLD_MANAGERS + ): hold[commenter] = 1 continue if re.match(REGEX_EX_CMDS, first_line, re.I): @@ -1034,7 +1065,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F continue if re.match(REGEX_EX_ENABLE_TESTS, first_line, re.I): if valid_commenter: - enable_tests, ignore = check_enable_bot_tests(first_line.split(" ", 1)[-1]) + enable_tests, ignore = check_enable_bot_tests( + first_line.split(" ", 1)[-1] + ) if not dryRun: set_comment_emoji(comment.id, repository, emoji="+1") continue @@ -1049,11 +1082,15 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if re.match("^unhold$", first_line, re.I): if "orp" in commenter_categories: hold = {} - elif commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): + elif commenter_categories or ( + commenter in releaseManagers + PR_HOLD_MANAGERS + ): if commenter in hold: del hold[commenter] continue - if (commenter == cmsbuild_user) and (re.match("^" + HOLD_MSG + ".+", first_line)): + if (commenter == cmsbuild_user) and ( + re.match("^" + HOLD_MSG + ".+", first_line) + ): for u in first_line.split(HOLD_MSG, 2)[1].split(","): u = u.strip().lstrip("@") if u in hold: @@ -1072,7 +1109,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F (not issue.pull_request) and (commenter in CMSSW_ISSUES_TRACKERS) ): mustClose = False - if (issue.state == "closed") and (comment.created_at >= issue.closed_at): + if (issue.state == "closed") and ( + comment.created_at >= issue.closed_at + ): reOpen = True print("==>Reopen request received from %s" % commenter) continue @@ -1090,7 +1129,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params = dict(test_params) if "ENABLE_BOT_TESTS" in global_test_params: enable_tests = global_test_params["ENABLE_BOT_TESTS"] - test_params_msg = str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) + test_params_msg = ( + str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) + ) continue if cmssw_repo: @@ -1103,11 +1144,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if m.group(2): code_check_apply_patch = True - new_signatures = {} - # Check L2 signoff for users in this PR signing categories - if [x for x in commenter_categories if x in signing_categories]: - ctype = "" selected_cats = [] + if re.match("^([+]1|approve[d]?|sign|signed)$", first_line, re.I): ctype = "+1" selected_cats = commenter_categories @@ -1119,23 +1157,16 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if category_name in commenter_categories: ctype = first_line[0] + "1" selected_cats = [category_name] - if ctype == "+1": - for sign in selected_cats: - new_signatures[sign] = "approved" - if (test_comment is None) and ( - (repository in auto_test_repo) or ("*" in auto_test_repo) - ): - test_comment = comment - if sign == "orp": - mustClose = False - elif ctype == "-1": - for sign in selected_cats: - new_signatures[sign] = "rejected" - if sign == "orp": - mustClose = False - - if new_signatures: - events[comment.created_at] = {"type": "sign", "value": new_signatures} + + if selected_cats: + events[comment.created_at] = { + "type": "sign", + "value": { + "ctype": ctype, + "selected_cats": selected_cats, + "comment": comment, + }, + } # Ignore all other messages which are before last commit. if issue.pull_request and (comment.created_at < last_commit_date): @@ -1145,7 +1176,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[first_line] = "pending" if first_line not in pre_checks + extra_pre_checks: extra_pre_checks.append(first_line) - if code_checks_status and (code_checks_status[0].updated_at >= comment.created_at): + if code_checks_status and ( + code_checks_status[0].updated_at >= comment.created_at + ): continue if first_line in pre_checks: if pre_checks_state["code-checks"] in ["pending", ""]: @@ -1181,7 +1214,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F IGNORING_TESTS_MSG, first_line ): signatures["tests"] = "pending" - elif re.match("Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line): + elif re.match( + "Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line + ): pull_request_updated = False elif re.match(TRIGERING_TESTS_MSG, first_line) or re.match( TRIGERING_TESTS_MSG1, first_line @@ -1211,8 +1246,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F comp_warnings = False if "+1" in first_line: signatures["tests"] = "approved" - comp_warnings = ( - len([1 for l in comment_lines if "Compilation Warnings: Yes" in l]) > 0 + comp_warnings = any( + "Compilation Warnings: Yes" in l for l in comment_lines ) pre_checks_url["tests"] = comment.html_url elif "-1" in first_line: @@ -1227,9 +1262,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if issue.pull_request or push_test_issue: # Check if the release manager asked for merging this. - if ((commenter in releaseManagers) or ("orp" in commenter_categories)) and re.match( - "^\s*(merge)\s*$", first_line, re.I - ): + if ( + (commenter in releaseManagers) or ("orp" in commenter_categories) + ) and re.match("^\s*(merge)\s*$", first_line, re.I): mustMerge = True mustClose = False if ("orp" in commenter_categories) and ("orp" in signatures): @@ -1238,7 +1273,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Check if the someone asked to trigger the tests if valid_commenter: - ok, v2, v3, v4 = check_test_cmd(first_line, repository, global_test_params) + ok, v2, v3, v4 = check_test_cmd( + first_line, repository, global_test_params + ) if ok: test_comment = comment abort_test = None @@ -1260,7 +1297,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Comment message:", first_line) signatures["tests"] = "pending" continue - elif REGEX_TEST_ABORT.match(first_line) and (signatures["tests"] == "pending"): + elif REGEX_TEST_ABORT.match(first_line) and ( + signatures["tests"] == "pending" + ): abort_test = comment test_comment = None signatures["tests"] = "pending" @@ -1282,10 +1321,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if ( (not warned_too_many_commits) and not ok_too_many_commits - and len(all_commits) >= MAX_INITIAL_COMMITS_IN_PR + and pr.commits >= MAX_INITIAL_COMMITS_IN_PR ): issue.create_comment( - f"This PR contains too many commits ({len(all_commits)} > {MAX_INITIAL_COMMITS_IN_PR}). " + f"This PR contains too many commits ({pr.commits} > {MAX_INITIAL_COMMITS_IN_PR}). " f"Make sure you chose the right target branch. " "\n{l2s}, you can override this check with `+commit-count`." ) @@ -1306,14 +1345,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if seen_commits_match: commit_cache = loads(seen_commits_match[1]) - old_body = cache_comment.body if cache_comment else CMSBOT_TECHNICAL_MSG - new_body = REGEX_COMMITS_CACHE.sub( - "", old_body - ) - chg_files = set() - if len(all_commits) < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: + if pr.commits < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: for commit in all_commits: if commit.sha not in commit_cache: commit_cache[commit.sha] = { @@ -1329,6 +1363,13 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F chg_files.update(cache_entry["files"]) + old_body = cache_comment.body if cache_comment else CMSBOT_TECHNICAL_MSG + new_body = ( + REGEX_COMMITS_CACHE.sub("", old_body) + + "" + ) if not dryRun: if cache_comment: if old_body != new_body: @@ -1344,6 +1385,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print(new_body) events = dict(sorted(events.items())) + print("Events:", events) if issue.pull_request: pr = repo.get_pull(prId) @@ -1356,14 +1398,17 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signing_categories.add("code-checks") updateMilestone(repo, issue, pr, dryRun) - packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) + packages = sorted( + list(set(cmssw_file2Package(repo_config, f) for f in chg_files)) + ) for pkg_file in chg_files: for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): for regex in pkgs_regexp: if regex.match(pkg_file): extra_labels["mtype"].append(ex_lab) print( - "Non-Blocking label:%s:%s:%s" % (ex_lab, regex.pattern, pkg_file) + "Non-Blocking label:%s:%s:%s" + % (ex_lab, regex.pattern, pkg_file) ) break if not extra_labels["mtype"]: @@ -1377,7 +1422,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ex_pkg = external_to_package(repository) if ex_pkg: packages.add(ex_pkg) - if (repo_org != GH_CMSSW_ORGANIZATION) or (repo_name in VALID_CMS_SW_REPOS_FOR_TESTS): + if (repo_org != GH_CMSSW_ORGANIZATION) or ( + repo_name in VALID_CMS_SW_REPOS_FOR_TESTS + ): create_test_property = True if (repo_name == GH_CMSDIST_REPO) and ( not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) @@ -1414,9 +1461,13 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ] has_category = all([package in all_packages for package in packages]) if not has_category: - new_package_message = "\nThe following packages do not have a category, yet:\n\n" + new_package_message = ( + "\nThe following packages do not have a category, yet:\n\n" + ) new_package_message += ( - "\n".join([package for package in packages if not package in all_packages]) + "\n".join( + [package for package in packages if not package in all_packages] + ) + "\n" ) new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" @@ -1448,7 +1499,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Handle watchers watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) for watcher in [x for x in watchers]: - if not watcher in watchingGroups: + if watcher not in watchingGroups: continue watchers.remove(watcher) watchers.update(set(watchingGroups[watcher])) @@ -1457,11 +1508,28 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F for event in events.values(): if event["type"] == "sign": - for cat, sign in event["value"].items(): - signatures[cat] = sign + selected_cats = event["value"]["selected_cats"] + ctype = event["value"]["ctype"] + if any(x in signing_categories for x in selected_cats): + if ctype == "+1": + for sign in selected_cats: + signatures[sign] = "approved" + if (test_comment is None) and ( + (repository in auto_test_repo) or ("*" in auto_test_repo) + ): + test_comment = event["value"]["comment"] + if sign == "orp": + mustClose = False + elif ctype == "-1": + for sign in selected_cats: + signatures[sign] = "rejected" + if sign == "orp": + mustClose = False + elif event["type"] == "commit": chg_categories = [ - x for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) + x + for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) ] signatures["orp"] = "pending" for cat in chg_categories: @@ -1585,7 +1653,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): signatures["tests"] = "started" if ( - get_status_state("%s/unknown/release" % cms_status_prefix, commit_statuses) + get_status_state( + "%s/unknown/release" % cms_status_prefix, commit_statuses + ) == "error" ): signatures["tests"] = "pending" @@ -1608,7 +1678,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F for s in [ i for i in commit_statuses - if ((i.context == scontext) or (i.context.startswith(scontext + "/"))) + if ( + (i.context == scontext) + or (i.context.startswith(scontext + "/")) + ) ]: if (not result_url) and ("/jenkins-artifacts/" in s.target_url): xdata = s.target_url.split("/") @@ -1735,7 +1808,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) else: new_msg = "" - for l in already_seen.body.encode("ascii", "ignore").decode().split("\n"): + for l in ( + already_seen.body.encode("ascii", "ignore").decode().split("\n") + ): if BACKPORT_STR in l: continue new_msg += l + "\n" @@ -1829,7 +1904,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) print("Blockers:", blockers) - print("Changed Labels:", labels - old_labels, old_labels - labels) + print("Changed Labels: added", labels - old_labels, "removed", old_labels - labels) if old_labels == labels: print("Labels unchanged.") elif not dryRunOrig: @@ -1884,7 +1959,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # get release managers SUPER_USERS = read_repo_file(repo_config, "super-users.yaml", []) - releaseManagersList = ", ".join([gh_user_char + x for x in set(releaseManagers + SUPER_USERS)]) + releaseManagersList = ", ".join( + [gh_user_char + x for x in set(releaseManagers + SUPER_USERS)] + ) if cmssw_prs: global_test_params["PULL_REQUESTS"] = cmssw_prs @@ -1929,7 +2006,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) if not dryRun: set_comment_emoji(test_comment.id, repository) - elif abort_test and bot_status and (not bot_status.description.startswith("Aborted")): + elif ( + abort_test + and bot_status + and (not bot_status.description.startswith("Aborted")) + ): if not has_user_emoji(abort_test, repository, "+1", cmsbuild_user): create_properties_file_tests( repository, prId, global_test_params, dryRun, abort=True @@ -2011,7 +2092,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F gh_user_char + name for name, l2_categories in list(CMSSW_L2.items()) for signature in signing_categories - if signature in l2_categories and signature in unsigned and signature not in ["orp"] + if signature in l2_categories + and signature in unsigned + and signature not in ["orp"] ] missing_notifications = set(missing_notifications) @@ -2051,7 +2134,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pkg_msg = [] for pkg in packages: if pkg in package_categories: - pkg_msg.append("- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg]))) + pkg_msg.append( + "- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg])) + ) else: pkg_msg.append("- %s (**new**)" % pkg) messageNewPR = format( @@ -2139,7 +2224,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pre_check not in signatures: signatures[pre_check] = "pending" print( - "PRE CHECK: %s,%s,%s" % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) + "PRE CHECK: %s,%s,%s" + % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) ) if signatures[pre_check] != "pending": if pre_checks_state[pre_check] in ["pending", ""]: @@ -2211,7 +2297,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if not dryRun: emoji = "-1" if "ERRORS: " in test_params_msg else "+1" if e and (e["content"] != emoji): - delete_comment_emoji(str(e["id"]), test_params_comment.id, repository) + delete_comment_emoji( + str(e["id"]), test_params_comment.id, repository + ) state = "success" if emoji == "+1" else "error" last_commit_obj.create_status( state, From 015fa635b42bbb2c859ab5b2a62aef197adb9197 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 13 Nov 2023 11:40:26 +0100 Subject: [PATCH 08/46] Fix style --- process_pr.py | 173 ++++++++++++++------------------------------------ 1 file changed, 47 insertions(+), 126 deletions(-) diff --git a/process_pr.py b/process_pr.py index 3968ded41f2e..994d92eaee70 100644 --- a/process_pr.py +++ b/process_pr.py @@ -114,12 +114,8 @@ def format(s, **kwds): arch=ARCH_PATTERN, ) RELVAL_OPTS = "[-][a-zA-Z0-9_.,\s/'-]+" -CLOSE_REQUEST = re.compile( - "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I -) -REOPEN_REQUEST = re.compile( - "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I -) +CLOSE_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I) +REOPEN_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I) CMS_PR_PATTERN = format( "(#[1-9][0-9]*|(%(cmsorgs)s)/+[a-zA-Z0-9_-]+#[1-9][0-9]*|https://+github.com/+(%(cmsorgs)s)/+[a-zA-Z0-9_-]+/+pull/+[1-9][0-9]*)", cmsorgs="|".join(EXTERNAL_REPOS), @@ -380,9 +376,7 @@ def find_last_comment(issue, user, match): def modify_comment(comment, match, replace, dryRun): - comment_msg = ( - comment.body.encode("ascii", "ignore").decode() if comment.body else "" - ) + comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" if match: new_comment_msg = re.sub(match, replace, comment_msg) else: @@ -472,9 +466,7 @@ def check_extra_labels(first_line, extra_labels): def check_type_labels(first_line, extra_labels): ex_labels = {} rem_labels = {} - for type_cmd in [ - x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip() - ]: + for type_cmd in [x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip()]: valid_lab = False rem_lab = type_cmd[0] == "-" if type_cmd[0] in ["-", "+"]: @@ -703,9 +695,7 @@ def cmssw_file2Package(repo_config, filename): def get_jenkins_job(issue): test_line = "" - for line in [ - l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n") - ]: + for line in [l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n")]: if line.startswith("Build logs are available at:"): test_line = line if test_line: @@ -784,21 +774,16 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers = [] # Process Pull Request pkg_categories = set([]) - REGEX_TYPE_CMDS = ( - "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" - ) - REGEX_EX_CMDS = ( - "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" - % (repo.full_name) + REGEX_TYPE_CMDS = "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" + REGEX_EX_CMDS = "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" % ( + repo.full_name ) known_ignore_tests = "%s" % MULTILINE_COMMENTS_MAP["ignore_test(s|)"][0] REGEX_EX_IGNORE_CHKS = "^ignore\s+((%s)(\s*,\s*(%s))*|none)$" % ( known_ignore_tests, known_ignore_tests, ) - REGEX_EX_ENABLE_TESTS = ( - "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] - ) + REGEX_EX_ENABLE_TESTS = "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] L2_DATA = init_l2_data(cms_repo) last_commit_date = None last_commit_obj = None @@ -867,9 +852,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F cms_status_prefix = "cms/%s" % prId bot_status = get_status(bot_status_name, commit_statuses) code_checks_status = [ - s - for s in commit_statuses - if s.context == "%s/code-checks" % cms_status_prefix + s for s in commit_statuses if s.context == "%s/code-checks" % cms_status_prefix ] print("PR Statuses:", commit_statuses) print("#PR Statuses:", len(commit_statuses)) @@ -895,9 +878,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if (not dryRun) and add_labels: - labels = [ - x.name.encode("ascii", "ignore").decode() for x in issue.labels - ] + labels = [x.name.encode("ascii", "ignore").decode() for x in issue.labels] if not "future-commit" in labels: labels.append("future-commit") issue.edit(labels=labels) @@ -920,9 +901,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if repository == CMSSW_REPO_NAME and re.match(CREATE_REPO, issue.title): - with open( - "query-new-data-repo-issues-" + str(issue.number) + ".properties", "w" - ) as f: + with open("query-new-data-repo-issues-" + str(issue.number) + ".properties", "w") as f: f.write("ISSUE_NUMBER=" + str(issue.number) + "\n") # Process the issue comments @@ -975,14 +954,12 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F commenter_categories = get_commenter_categories( commenter, int(comment.created_at.strftime("%s")) ) - valid_commenter = ( - commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org] - ) or (len(commenter_categories) > 0) + valid_commenter = (commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org]) or ( + len(commenter_categories) > 0 + ) if (not valid_commenter) and (requestor != commenter): continue - comment_msg = ( - comment.body.encode("ascii", "ignore").decode() if comment.body else "" - ) + comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" # The first line is an invariant. comment_lines = [l.strip() for l in comment_msg.split("\n") if l.strip()] first_line = comment_lines[0:1] @@ -1006,9 +983,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): technical_comment = comment - if ( - commenter == cmsbuild_user - ) and "This PR contains too many commits" in first_line: + if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: warned_too_many_commits = True if commenter in l2s and re.match( @@ -1018,9 +993,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F assign_type, new_cats = get_assign_categories(first_line) if new_cats: - if (assign_type == "new categories assigned:") and ( - commenter == cmsbuild_user - ): + if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user): for ex_cat in new_cats: if ex_cat in assign_cats: assign_cats[ex_cat] = 1 @@ -1042,9 +1015,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Some of the special users can say "hold" prevent automatic merging of # fully signed PRs. if re.match("^hold$", first_line, re.I): - if commenter_categories or ( - commenter in releaseManagers + PR_HOLD_MANAGERS - ): + if commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): hold[commenter] = 1 continue if re.match(REGEX_EX_CMDS, first_line, re.I): @@ -1065,9 +1036,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F continue if re.match(REGEX_EX_ENABLE_TESTS, first_line, re.I): if valid_commenter: - enable_tests, ignore = check_enable_bot_tests( - first_line.split(" ", 1)[-1] - ) + enable_tests, ignore = check_enable_bot_tests(first_line.split(" ", 1)[-1]) if not dryRun: set_comment_emoji(comment.id, repository, emoji="+1") continue @@ -1082,15 +1051,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if re.match("^unhold$", first_line, re.I): if "orp" in commenter_categories: hold = {} - elif commenter_categories or ( - commenter in releaseManagers + PR_HOLD_MANAGERS - ): + elif commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): if commenter in hold: del hold[commenter] continue - if (commenter == cmsbuild_user) and ( - re.match("^" + HOLD_MSG + ".+", first_line) - ): + if (commenter == cmsbuild_user) and (re.match("^" + HOLD_MSG + ".+", first_line)): for u in first_line.split(HOLD_MSG, 2)[1].split(","): u = u.strip().lstrip("@") if u in hold: @@ -1109,9 +1074,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F (not issue.pull_request) and (commenter in CMSSW_ISSUES_TRACKERS) ): mustClose = False - if (issue.state == "closed") and ( - comment.created_at >= issue.closed_at - ): + if (issue.state == "closed") and (comment.created_at >= issue.closed_at): reOpen = True print("==>Reopen request received from %s" % commenter) continue @@ -1129,9 +1092,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params = dict(test_params) if "ENABLE_BOT_TESTS" in global_test_params: enable_tests = global_test_params["ENABLE_BOT_TESTS"] - test_params_msg = ( - str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) - ) + test_params_msg = str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) continue if cmssw_repo: @@ -1176,9 +1137,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[first_line] = "pending" if first_line not in pre_checks + extra_pre_checks: extra_pre_checks.append(first_line) - if code_checks_status and ( - code_checks_status[0].updated_at >= comment.created_at - ): + if code_checks_status and (code_checks_status[0].updated_at >= comment.created_at): continue if first_line in pre_checks: if pre_checks_state["code-checks"] in ["pending", ""]: @@ -1214,9 +1173,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F IGNORING_TESTS_MSG, first_line ): signatures["tests"] = "pending" - elif re.match( - "Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line - ): + elif re.match("Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line): pull_request_updated = False elif re.match(TRIGERING_TESTS_MSG, first_line) or re.match( TRIGERING_TESTS_MSG1, first_line @@ -1246,9 +1203,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F comp_warnings = False if "+1" in first_line: signatures["tests"] = "approved" - comp_warnings = any( - "Compilation Warnings: Yes" in l for l in comment_lines - ) + comp_warnings = any("Compilation Warnings: Yes" in l for l in comment_lines) pre_checks_url["tests"] = comment.html_url elif "-1" in first_line: signatures["tests"] = "rejected" @@ -1262,9 +1217,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if issue.pull_request or push_test_issue: # Check if the release manager asked for merging this. - if ( - (commenter in releaseManagers) or ("orp" in commenter_categories) - ) and re.match("^\s*(merge)\s*$", first_line, re.I): + if ((commenter in releaseManagers) or ("orp" in commenter_categories)) and re.match( + "^\s*(merge)\s*$", first_line, re.I + ): mustMerge = True mustClose = False if ("orp" in commenter_categories) and ("orp" in signatures): @@ -1273,9 +1228,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Check if the someone asked to trigger the tests if valid_commenter: - ok, v2, v3, v4 = check_test_cmd( - first_line, repository, global_test_params - ) + ok, v2, v3, v4 = check_test_cmd(first_line, repository, global_test_params) if ok: test_comment = comment abort_test = None @@ -1297,9 +1250,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Comment message:", first_line) signatures["tests"] = "pending" continue - elif REGEX_TEST_ABORT.match(first_line) and ( - signatures["tests"] == "pending" - ): + elif REGEX_TEST_ABORT.match(first_line) and (signatures["tests"] == "pending"): abort_test = comment test_comment = None signatures["tests"] = "pending" @@ -1398,17 +1349,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signing_categories.add("code-checks") updateMilestone(repo, issue, pr, dryRun) - packages = sorted( - list(set(cmssw_file2Package(repo_config, f) for f in chg_files)) - ) + packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) for pkg_file in chg_files: for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): for regex in pkgs_regexp: if regex.match(pkg_file): extra_labels["mtype"].append(ex_lab) print( - "Non-Blocking label:%s:%s:%s" - % (ex_lab, regex.pattern, pkg_file) + "Non-Blocking label:%s:%s:%s" % (ex_lab, regex.pattern, pkg_file) ) break if not extra_labels["mtype"]: @@ -1422,9 +1370,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ex_pkg = external_to_package(repository) if ex_pkg: packages.add(ex_pkg) - if (repo_org != GH_CMSSW_ORGANIZATION) or ( - repo_name in VALID_CMS_SW_REPOS_FOR_TESTS - ): + if (repo_org != GH_CMSSW_ORGANIZATION) or (repo_name in VALID_CMS_SW_REPOS_FOR_TESTS): create_test_property = True if (repo_name == GH_CMSDIST_REPO) and ( not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) @@ -1461,13 +1407,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ] has_category = all([package in all_packages for package in packages]) if not has_category: - new_package_message = ( - "\nThe following packages do not have a category, yet:\n\n" - ) + new_package_message = "\nThe following packages do not have a category, yet:\n\n" new_package_message += ( - "\n".join( - [package for package in packages if not package in all_packages] - ) + "\n".join([package for package in packages if not package in all_packages]) + "\n" ) new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" @@ -1528,8 +1470,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F elif event["type"] == "commit": chg_categories = [ - x - for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) + x for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) ] signatures["orp"] = "pending" for cat in chg_categories: @@ -1653,9 +1594,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): signatures["tests"] = "started" if ( - get_status_state( - "%s/unknown/release" % cms_status_prefix, commit_statuses - ) + get_status_state("%s/unknown/release" % cms_status_prefix, commit_statuses) == "error" ): signatures["tests"] = "pending" @@ -1678,10 +1617,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F for s in [ i for i in commit_statuses - if ( - (i.context == scontext) - or (i.context.startswith(scontext + "/")) - ) + if ((i.context == scontext) or (i.context.startswith(scontext + "/"))) ]: if (not result_url) and ("/jenkins-artifacts/" in s.target_url): xdata = s.target_url.split("/") @@ -1808,9 +1744,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) else: new_msg = "" - for l in ( - already_seen.body.encode("ascii", "ignore").decode().split("\n") - ): + for l in already_seen.body.encode("ascii", "ignore").decode().split("\n"): if BACKPORT_STR in l: continue new_msg += l + "\n" @@ -1959,9 +1893,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # get release managers SUPER_USERS = read_repo_file(repo_config, "super-users.yaml", []) - releaseManagersList = ", ".join( - [gh_user_char + x for x in set(releaseManagers + SUPER_USERS)] - ) + releaseManagersList = ", ".join([gh_user_char + x for x in set(releaseManagers + SUPER_USERS)]) if cmssw_prs: global_test_params["PULL_REQUESTS"] = cmssw_prs @@ -2006,11 +1938,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) if not dryRun: set_comment_emoji(test_comment.id, repository) - elif ( - abort_test - and bot_status - and (not bot_status.description.startswith("Aborted")) - ): + elif abort_test and bot_status and (not bot_status.description.startswith("Aborted")): if not has_user_emoji(abort_test, repository, "+1", cmsbuild_user): create_properties_file_tests( repository, prId, global_test_params, dryRun, abort=True @@ -2092,9 +2020,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F gh_user_char + name for name, l2_categories in list(CMSSW_L2.items()) for signature in signing_categories - if signature in l2_categories - and signature in unsigned - and signature not in ["orp"] + if signature in l2_categories and signature in unsigned and signature not in ["orp"] ] missing_notifications = set(missing_notifications) @@ -2134,9 +2060,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pkg_msg = [] for pkg in packages: if pkg in package_categories: - pkg_msg.append( - "- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg])) - ) + pkg_msg.append("- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg]))) else: pkg_msg.append("- %s (**new**)" % pkg) messageNewPR = format( @@ -2224,8 +2148,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pre_check not in signatures: signatures[pre_check] = "pending" print( - "PRE CHECK: %s,%s,%s" - % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) + "PRE CHECK: %s,%s,%s" % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) ) if signatures[pre_check] != "pending": if pre_checks_state[pre_check] in ["pending", ""]: @@ -2297,9 +2220,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if not dryRun: emoji = "-1" if "ERRORS: " in test_params_msg else "+1" if e and (e["content"] != emoji): - delete_comment_emoji( - str(e["id"]), test_params_comment.id, repository - ) + delete_comment_emoji(str(e["id"]), test_params_comment.id, repository) state = "success" if emoji == "+1" else "error" last_commit_obj.create_status( state, From 18faffb8d1328e5fd9a132f8f6b25559b4dc622f Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Tue, 14 Nov 2023 10:50:51 +0100 Subject: [PATCH 09/46] Fix initial comment generation (watchers + categories) --- process_pr.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/process_pr.py b/process_pr.py index 994d92eaee70..4021ff260283 100644 --- a/process_pr.py +++ b/process_pr.py @@ -905,7 +905,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F f.write("ISSUE_NUMBER=" + str(issue.number) + "\n") # Process the issue comments - signatures = dict([(x, "pending") for x in signing_categories]) extra_pre_checks = [] pre_checks = [] if issue.pull_request: @@ -1276,8 +1275,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): issue.create_comment( f"This PR contains too many commits ({pr.commits} > {MAX_INITIAL_COMMITS_IN_PR}). " - f"Make sure you chose the right target branch. " - "\n{l2s}, you can override this check with `+commit-count`." + "Make sure you chose the right target branch.\n" + f"{l2s}, you can override this check with `+commit-count`." ) # Get the commit cache from `already_seen` commit or technical commit @@ -1398,6 +1397,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Following categories affected:") print("\n".join(signing_categories)) + signatures = dict([(x, "pending") for x in signing_categories]) + if cmssw_repo: # If there is a new package, add also a dummy "new" category. all_packages = [ @@ -1480,7 +1481,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F auto_close_push_test_issue = True try: auto_close_push_test_issue = repo_config.AUTO_CLOSE_PUSH_TESTS_ISSUE - except: + except AttributeError: pass if ( auto_close_push_test_issue @@ -2138,10 +2139,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Pull request is already fully signed. Not sending message.") else: print("Already notified L2 about " + str(pr.number)) - if commentMsg and not dryRun: + if commentMsg and dryRun: print("The following comment will be made:") try: - print(commentMsg.decode("ascii", "replace")) + print(commentMsg.encode("ascii", "replace").decode()) except: pass for pre_check in pre_checks + extra_pre_checks: From 75f7817d5178f6ca2446216552cf88c1a52ba914 Mon Sep 17 00:00:00 2001 From: iarspider Date: Wed, 15 Nov 2023 15:18:44 +0100 Subject: [PATCH 10/46] Update process_pr.py --- process_pr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process_pr.py b/process_pr.py index 4021ff260283..e84f0a87b6fc 100644 --- a/process_pr.py +++ b/process_pr.py @@ -198,7 +198,7 @@ def format(s, **kwds): + "|_input|)": [RELVAL_OPTS, "EXTRA_MATRIX_COMMAND_ARGS", True], } -MAX_INITIAL_COMMITS_IN_PR = 20 +MAX_INITIAL_COMMITS_IN_PR = 100 L2_DATA = {} From 82f51edf502eef9df3310570af56135fcada93ca Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 16 Nov 2023 14:28:00 +0100 Subject: [PATCH 11/46] Move get_last_commit and get_pr_commits_reversed to github_utils --- check-future-commits-prs.py | 2 +- github_utils.py | 136 +++++++++++++++++++----- process_pr.py | 204 ++++++++++++++++++++++-------------- 3 files changed, 241 insertions(+), 101 deletions(-) diff --git a/check-future-commits-prs.py b/check-future-commits-prs.py index c4ff2af4197f..fb4644112afe 100755 --- a/check-future-commits-prs.py +++ b/check-future-commits-prs.py @@ -36,7 +36,7 @@ if exists(join(repo_dir, "repo_config.py")): sys.path.insert(0, repo_dir) import repo_config -from process_pr import get_last_commit +from github_utils import get_last_commit gh = Github(login_or_token=open(expanduser(repo_config.GH_TOKEN)).read().strip()) api_rate_limits(gh) diff --git a/github_utils.py b/github_utils.py index 3cd942b341eb..d27ee9cded7f 100644 --- a/github_utils.py +++ b/github_utils.py @@ -45,7 +45,12 @@ def get_page_range(): def _check_rate_limits( - rate_limit, rate_limit_max, rate_limiting_resettime, msg=True, when_slow=False, prefix="" + rate_limit, + rate_limit_max, + rate_limiting_resettime, + msg=True, + when_slow=False, + prefix="", ): global GH_TOKENS, GH_TOKEN_INDEX from calendar import timegm @@ -98,7 +103,12 @@ def _check_rate_limits( def check_rate_limits(msg=True, when_slow=False, prefix=""): _check_rate_limits( - GH_RATE_LIMIT[0], GH_RATE_LIMIT[1], GH_RATE_LIMIT[2], msg, when_slow, prefix=prefix + GH_RATE_LIMIT[0], + GH_RATE_LIMIT[1], + GH_RATE_LIMIT[2], + msg, + when_slow, + prefix=prefix, ) @@ -125,7 +135,9 @@ def api_rate_limits(gh, msg=True, when_slow=False, prefix=""): def get_ported_PRs(repo, src_branch, des_branch): done_prs_id = {} - prRe = re.compile("Automatically ported from " + src_branch + " #(\d+)\s+.*", re.MULTILINE) + prRe = re.compile( + "Automatically ported from " + src_branch + " #(\d+)\s+.*", re.MULTILINE + ) for pr in repo.get_pulls(base=des_branch): body = pr.body.encode("ascii", "ignore") if sys.version_info[0] == 3: @@ -253,9 +265,14 @@ def port_pr(repo, pr_num, des_branch, dryRun=False): print(newHead) print(newBody) if not dryRun: - newPR = repo.create_pull(title=pr.title, body=newBody, base=des_branch, head=newHead) + newPR = repo.create_pull( + title=pr.title, body=newBody, base=des_branch, head=newHead + ) else: - print("DryRun: should have created Pull Request for %s using %s" % (des_branch, newHead)) + print( + "DryRun: should have created Pull Request for %s using %s" + % (des_branch, newHead) + ) print("Every thing looks good") git_cmd = format( "cd %(clone_dir)s; git branch -d %(new_branch)s", @@ -431,7 +448,9 @@ def get_failed_pending_members(org): def get_delete_pending_members(org, invitation_id): - return github_api("/orgs/%s/invitations/%s" % (org, invitation_id), method="DELETE", raw=True) + return github_api( + "/orgs/%s/invitations/%s" % (org, invitation_id), method="DELETE", raw=True + ) def get_organization_members(org, role="all", filter="all"): @@ -456,7 +475,9 @@ def add_organization_member(org, member, role="member"): def invite_organization_member(org, member, role="direct_member"): return github_api( - "/orgs/%s/invitations" % org, params={"role": role, "invitee_id": member}, method="POST" + "/orgs/%s/invitations" % org, + params={"role": role, "invitee_id": member}, + method="POST", ) @@ -471,7 +492,9 @@ def edit_pr(repo, pr_num, title=None, body=None, state=None, base=None): params["base"] = base if state: params["state"] = state - return github_api(uri="/repos/%s/pulls/%s" % (repo, pr_num), params=params, method="PATCH") + return github_api( + uri="/repos/%s/pulls/%s" % (repo, pr_num), params=params, method="PATCH" + ) def create_issue_comment(repo, issue_num, body): @@ -483,7 +506,9 @@ def create_issue_comment(repo, issue_num, body): def get_issue_labels(repo, issue_num): get_gh_token(repo) - return github_api(uri="/repos/%s/issues/%s/labels" % (repo, issue_num), method="GET") + return github_api( + uri="/repos/%s/issues/%s/labels" % (repo, issue_num), method="GET" + ) def add_issue_labels(repo, issue_num, labels=[]): @@ -507,7 +532,9 @@ def set_issue_labels(repo, issue_num, labels=[]): def remove_issue_labels_all(repo, issue_num): get_gh_token(repo) return github_api( - uri="/repos/%s/issues/%s/labels" % (repo, issue_num), method="DELETE", status=[204] + uri="/repos/%s/issues/%s/labels" % (repo, issue_num), + method="DELETE", + status=[204], ) @@ -544,7 +571,11 @@ def github_api( headers = {} url = "https://api.github.com%s" % uri data = "" - if per_page and ("per_page" not in params) and (not method in ["POST", "PATCH", "PUT"]): + if ( + per_page + and ("per_page" not in params) + and (not method in ["POST", "PATCH", "PUT"]) + ): params["per_page"] = per_page if method == "GET": if params: @@ -607,7 +638,14 @@ def github_api( if max_pages > 0 and page > max_pages: break data += github_api( - uri, params, method, headers, page, raw=raw, per_page=per_page, all_pages=False + uri, + params, + method, + headers, + page, + raw=raw, + per_page=per_page, + all_pages=False, ) return data @@ -681,7 +719,9 @@ def set_gh_user(user): def get_combined_statuses(commit, repository): get_gh_token(repository) - return github_api("/repos/%s/commits/%s/status" % (repository, commit), method="GET") + return github_api( + "/repos/%s/commits/%s/status" % (repository, commit), method="GET" + ) def get_pr_commits(pr, repository, per_page=None, last_page=False): @@ -716,7 +756,8 @@ def set_comment_emoji(comment_id, repository, emoji="+1", reset_other=True): get_gh_token(repository) params = {"content": emoji} return github_api( - "/repos/%s/issues/comments/%s/reactions" % (repository, comment_id), params=params + "/repos/%s/issues/comments/%s/reactions" % (repository, comment_id), + params=params, ) @@ -735,7 +776,9 @@ def get_repository_issues( def get_issue_comments(repository, issue_num): get_gh_token(repository) - return github_api("/repos/%s/issues/%s/comments" % (repository, issue_num), method="GET") + return github_api( + "/repos/%s/issues/%s/comments" % (repository, issue_num), method="GET" + ) def get_issue(repository, issue_num): @@ -762,14 +805,16 @@ def get_release_by_tag(repository, tag): def get_comment_emojis(comment_id, repository): get_gh_token(repository) return github_api( - "/repos/%s/issues/comments/%s/reactions" % (repository, comment_id), method="GET" + "/repos/%s/issues/comments/%s/reactions" % (repository, comment_id), + method="GET", ) def delete_comment_emoji(emoji_id, comment_id, repository): get_gh_token(repository) return github_api( - "/repos/%s/issues/comments/%s/reactions/%s" % (repository, comment_id, emoji_id), + "/repos/%s/issues/comments/%s/reactions/%s" + % (repository, comment_id, emoji_id), method="DELETE", raw=True, ) @@ -790,7 +835,12 @@ def mark_commit_status( reset=False, ): get_gh_token(repository) - params = {"state": state, "target_url": url, "description": description, "context": context} + params = { + "state": state, + "target_url": url, + "description": description, + "context": context, + } github_api("/repos/%s/statuses/%s" % (repository, commit), params=params) if reset: statuses = get_combined_statuses(commit, repository) @@ -804,7 +854,9 @@ def mark_commit_status( for s in statuses["statuses"]: if s["context"].startswith(context + "/"): params["context"] = s["context"] - github_api("/repos/%s/statuses/%s" % (repository, commit), params=params) + github_api( + "/repos/%s/statuses/%s" % (repository, commit), params=params + ) return @@ -816,7 +868,9 @@ def get_branch(repository, branch_name): def get_git_tag(repository, tag_name): get_gh_token(repository) - data = github_api("/repos/%s/git/ref/tags/%s" % (repository, tag_name), method="GET") + data = github_api( + "/repos/%s/git/ref/tags/%s" % (repository, tag_name), method="GET" + ) return data @@ -853,7 +907,9 @@ def get_org_packages(org, package_type="container", visibility=None, token_file= def get_org_package(org, package, package_type="container", token_file=None): get_gh_token(token_file=token_file) return github_api( - "/orgs/%s/packages/%s/%s" % (org, package_type, package), method="GET", all_pages=True + "/orgs/%s/packages/%s/%s" % (org, package_type, package), + method="GET", + all_pages=True, ) @@ -866,10 +922,13 @@ def get_org_package_versions(org, package, package_type="container", token_file= ) -def get_org_package_version(org, package, version_id, package_type="container", token_file=None): +def get_org_package_version( + org, package, version_id, package_type="container", token_file=None +): get_gh_token(token_file=token_file) return github_api( - "/orgs/%s/packages/%s/%s/versions/%s" % (org, package_type, package, version_id), + "/orgs/%s/packages/%s/%s/versions/%s" + % (org, package_type, package, version_id), method="GET", ) @@ -892,7 +951,9 @@ def get_commits(repository, branch, until, per_page=1): def find_tags(repository, name): get_gh_token(repository) - data = github_api("/repos/%s/git/matching-refs/tags/%s" % (repository, name), method="GET") + data = github_api( + "/repos/%s/git/matching-refs/tags/%s" % (repository, name), method="GET" + ) return data @@ -901,3 +962,30 @@ def get_pr(repository, pr_id): data = github_api("/repos/%s/pulls/%s" % (repository, pr_id), method="GET") return data + + +def get_last_commit(pr): + commits_ = get_pr_commits_reversed(pr) + if commits_: + return commits_[-1] + else: + return None + + +def get_pr_commits_reversed(pr): + """ + + :param pr: + :return: PaginatedList[Commit] | List[Commit] + """ + try: + # This requires at least PyGithub 1.23.0. Making it optional for the moment. + return pr.get_commits().reversed + except: # noqa + # This seems to fail for more than 250 commits. Not sure if the + # problem is github itself or the bindings. + try: + return reversed(list(pr.get_commits())) + except IndexError: + print("Index error: May be PR with no commits") + return [] diff --git a/process_pr.py b/process_pr.py index e84f0a87b6fc..a0df00902983 100644 --- a/process_pr.py +++ b/process_pr.py @@ -33,7 +33,7 @@ from datetime import datetime from os.path import join, exists, dirname from os import environ -from github_utils import edit_pr, api_rate_limits +from github_utils import edit_pr, api_rate_limits, get_pr_commits_reversed from github_utils import ( set_comment_emoji, get_comment_emojis, @@ -114,8 +114,12 @@ def format(s, **kwds): arch=ARCH_PATTERN, ) RELVAL_OPTS = "[-][a-zA-Z0-9_.,\s/'-]+" -CLOSE_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I) -REOPEN_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I) +CLOSE_REQUEST = re.compile( + "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I +) +REOPEN_REQUEST = re.compile( + "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I +) CMS_PR_PATTERN = format( "(#[1-9][0-9]*|(%(cmsorgs)s)/+[a-zA-Z0-9_-]+#[1-9][0-9]*|https://+github.com/+(%(cmsorgs)s)/+[a-zA-Z0-9_-]+/+pull/+[1-9][0-9]*)", cmsorgs="|".join(EXTERNAL_REPOS), @@ -198,7 +202,7 @@ def format(s, **kwds): + "|_input|)": [RELVAL_OPTS, "EXTRA_MATRIX_COMMAND_ARGS", True], } -MAX_INITIAL_COMMITS_IN_PR = 100 +MAX_INITIAL_COMMITS_IN_PR = 200 L2_DATA = {} @@ -227,38 +231,11 @@ def get_commenter_categories(commenter, comment_date): return [] -def get_last_commit(pr): - commits_ = get_pr_commits_reversed(pr) - if commits_: - return commits_[-1] - else: - return None - - def get_changed_files_in_commit(repo, commit_obj): commit = repo.get_commit(commit_obj.commit.sha) return [x.filename for x in commit.files] -def get_pr_commits_reversed(pr): - """ - - :param pr: - :return: PaginatedList[Commit] | List[Commit] - """ - try: - # This requires at least PyGithub 1.23.0. Making it optional for the moment. - return pr.get_commits().reversed - except: # noqa - # This seems to fail for more than 250 commits. Not sure if the - # problem is github itself or the bindings. - try: - return reversed(list(pr.get_commits())) - except IndexError: - print("Index error: May be PR with no commits") - return [] - - def get_package_categories(package): cats = [] for cat, packages in list(CMSSW_CATEGORIES.items()): @@ -376,7 +353,9 @@ def find_last_comment(issue, user, match): def modify_comment(comment, match, replace, dryRun): - comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" + comment_msg = ( + comment.body.encode("ascii", "ignore").decode() if comment.body else "" + ) if match: new_comment_msg = re.sub(match, replace, comment_msg) else: @@ -466,7 +445,9 @@ def check_extra_labels(first_line, extra_labels): def check_type_labels(first_line, extra_labels): ex_labels = {} rem_labels = {} - for type_cmd in [x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip()]: + for type_cmd in [ + x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip() + ]: valid_lab = False rem_lab = type_cmd[0] == "-" if type_cmd[0] in ["-", "+"]: @@ -695,7 +676,9 @@ def cmssw_file2Package(repo_config, filename): def get_jenkins_job(issue): test_line = "" - for line in [l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n")]: + for line in [ + l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n") + ]: if line.startswith("Build logs are available at:"): test_line = line if test_line: @@ -774,16 +757,21 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers = [] # Process Pull Request pkg_categories = set([]) - REGEX_TYPE_CMDS = "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" - REGEX_EX_CMDS = "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" % ( - repo.full_name + REGEX_TYPE_CMDS = ( + "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" + ) + REGEX_EX_CMDS = ( + "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" + % (repo.full_name) ) known_ignore_tests = "%s" % MULTILINE_COMMENTS_MAP["ignore_test(s|)"][0] REGEX_EX_IGNORE_CHKS = "^ignore\s+((%s)(\s*,\s*(%s))*|none)$" % ( known_ignore_tests, known_ignore_tests, ) - REGEX_EX_ENABLE_TESTS = "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] + REGEX_EX_ENABLE_TESTS = ( + "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] + ) L2_DATA = init_l2_data(cms_repo) last_commit_date = None last_commit_obj = None @@ -852,7 +840,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F cms_status_prefix = "cms/%s" % prId bot_status = get_status(bot_status_name, commit_statuses) code_checks_status = [ - s for s in commit_statuses if s.context == "%s/code-checks" % cms_status_prefix + s + for s in commit_statuses + if s.context == "%s/code-checks" % cms_status_prefix ] print("PR Statuses:", commit_statuses) print("#PR Statuses:", len(commit_statuses)) @@ -878,7 +868,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if (not dryRun) and add_labels: - labels = [x.name.encode("ascii", "ignore").decode() for x in issue.labels] + labels = [ + x.name.encode("ascii", "ignore").decode() for x in issue.labels + ] if not "future-commit" in labels: labels.append("future-commit") issue.edit(labels=labels) @@ -901,7 +893,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if repository == CMSSW_REPO_NAME and re.match(CREATE_REPO, issue.title): - with open("query-new-data-repo-issues-" + str(issue.number) + ".properties", "w") as f: + with open( + "query-new-data-repo-issues-" + str(issue.number) + ".properties", "w" + ) as f: f.write("ISSUE_NUMBER=" + str(issue.number) + "\n") # Process the issue comments @@ -953,12 +947,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F commenter_categories = get_commenter_categories( commenter, int(comment.created_at.strftime("%s")) ) - valid_commenter = (commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org]) or ( - len(commenter_categories) > 0 - ) + valid_commenter = ( + commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org] + ) or (len(commenter_categories) > 0) if (not valid_commenter) and (requestor != commenter): continue - comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" + comment_msg = ( + comment.body.encode("ascii", "ignore").decode() if comment.body else "" + ) # The first line is an invariant. comment_lines = [l.strip() for l in comment_msg.split("\n") if l.strip()] first_line = comment_lines[0:1] @@ -982,7 +978,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): technical_comment = comment - if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: + if ( + commenter == cmsbuild_user + ) and "This PR contains too many commits" in first_line: warned_too_many_commits = True if commenter in l2s and re.match( @@ -992,7 +990,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F assign_type, new_cats = get_assign_categories(first_line) if new_cats: - if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user): + if (assign_type == "new categories assigned:") and ( + commenter == cmsbuild_user + ): for ex_cat in new_cats: if ex_cat in assign_cats: assign_cats[ex_cat] = 1 @@ -1014,7 +1014,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Some of the special users can say "hold" prevent automatic merging of # fully signed PRs. if re.match("^hold$", first_line, re.I): - if commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): + if commenter_categories or ( + commenter in releaseManagers + PR_HOLD_MANAGERS + ): hold[commenter] = 1 continue if re.match(REGEX_EX_CMDS, first_line, re.I): @@ -1035,7 +1037,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F continue if re.match(REGEX_EX_ENABLE_TESTS, first_line, re.I): if valid_commenter: - enable_tests, ignore = check_enable_bot_tests(first_line.split(" ", 1)[-1]) + enable_tests, ignore = check_enable_bot_tests( + first_line.split(" ", 1)[-1] + ) if not dryRun: set_comment_emoji(comment.id, repository, emoji="+1") continue @@ -1050,11 +1054,15 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if re.match("^unhold$", first_line, re.I): if "orp" in commenter_categories: hold = {} - elif commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): + elif commenter_categories or ( + commenter in releaseManagers + PR_HOLD_MANAGERS + ): if commenter in hold: del hold[commenter] continue - if (commenter == cmsbuild_user) and (re.match("^" + HOLD_MSG + ".+", first_line)): + if (commenter == cmsbuild_user) and ( + re.match("^" + HOLD_MSG + ".+", first_line) + ): for u in first_line.split(HOLD_MSG, 2)[1].split(","): u = u.strip().lstrip("@") if u in hold: @@ -1073,7 +1081,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F (not issue.pull_request) and (commenter in CMSSW_ISSUES_TRACKERS) ): mustClose = False - if (issue.state == "closed") and (comment.created_at >= issue.closed_at): + if (issue.state == "closed") and ( + comment.created_at >= issue.closed_at + ): reOpen = True print("==>Reopen request received from %s" % commenter) continue @@ -1091,7 +1101,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params = dict(test_params) if "ENABLE_BOT_TESTS" in global_test_params: enable_tests = global_test_params["ENABLE_BOT_TESTS"] - test_params_msg = str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) + test_params_msg = ( + str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) + ) continue if cmssw_repo: @@ -1136,7 +1148,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[first_line] = "pending" if first_line not in pre_checks + extra_pre_checks: extra_pre_checks.append(first_line) - if code_checks_status and (code_checks_status[0].updated_at >= comment.created_at): + if code_checks_status and ( + code_checks_status[0].updated_at >= comment.created_at + ): continue if first_line in pre_checks: if pre_checks_state["code-checks"] in ["pending", ""]: @@ -1172,7 +1186,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F IGNORING_TESTS_MSG, first_line ): signatures["tests"] = "pending" - elif re.match("Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line): + elif re.match( + "Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line + ): pull_request_updated = False elif re.match(TRIGERING_TESTS_MSG, first_line) or re.match( TRIGERING_TESTS_MSG1, first_line @@ -1202,7 +1218,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F comp_warnings = False if "+1" in first_line: signatures["tests"] = "approved" - comp_warnings = any("Compilation Warnings: Yes" in l for l in comment_lines) + comp_warnings = any( + "Compilation Warnings: Yes" in l for l in comment_lines + ) pre_checks_url["tests"] = comment.html_url elif "-1" in first_line: signatures["tests"] = "rejected" @@ -1216,9 +1234,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if issue.pull_request or push_test_issue: # Check if the release manager asked for merging this. - if ((commenter in releaseManagers) or ("orp" in commenter_categories)) and re.match( - "^\s*(merge)\s*$", first_line, re.I - ): + if ( + (commenter in releaseManagers) or ("orp" in commenter_categories) + ) and re.match("^\s*(merge)\s*$", first_line, re.I): mustMerge = True mustClose = False if ("orp" in commenter_categories) and ("orp" in signatures): @@ -1227,7 +1245,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Check if the someone asked to trigger the tests if valid_commenter: - ok, v2, v3, v4 = check_test_cmd(first_line, repository, global_test_params) + ok, v2, v3, v4 = check_test_cmd( + first_line, repository, global_test_params + ) if ok: test_comment = comment abort_test = None @@ -1249,7 +1269,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Comment message:", first_line) signatures["tests"] = "pending" continue - elif REGEX_TEST_ABORT.match(first_line) and (signatures["tests"] == "pending"): + elif REGEX_TEST_ABORT.match(first_line) and ( + signatures["tests"] == "pending" + ): abort_test = comment test_comment = None signatures["tests"] = "pending" @@ -1348,14 +1370,17 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signing_categories.add("code-checks") updateMilestone(repo, issue, pr, dryRun) - packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) + packages = sorted( + list(set(cmssw_file2Package(repo_config, f) for f in chg_files)) + ) for pkg_file in chg_files: for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): for regex in pkgs_regexp: if regex.match(pkg_file): extra_labels["mtype"].append(ex_lab) print( - "Non-Blocking label:%s:%s:%s" % (ex_lab, regex.pattern, pkg_file) + "Non-Blocking label:%s:%s:%s" + % (ex_lab, regex.pattern, pkg_file) ) break if not extra_labels["mtype"]: @@ -1369,7 +1394,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ex_pkg = external_to_package(repository) if ex_pkg: packages.add(ex_pkg) - if (repo_org != GH_CMSSW_ORGANIZATION) or (repo_name in VALID_CMS_SW_REPOS_FOR_TESTS): + if (repo_org != GH_CMSSW_ORGANIZATION) or ( + repo_name in VALID_CMS_SW_REPOS_FOR_TESTS + ): create_test_property = True if (repo_name == GH_CMSDIST_REPO) and ( not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) @@ -1408,9 +1435,13 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ] has_category = all([package in all_packages for package in packages]) if not has_category: - new_package_message = "\nThe following packages do not have a category, yet:\n\n" + new_package_message = ( + "\nThe following packages do not have a category, yet:\n\n" + ) new_package_message += ( - "\n".join([package for package in packages if not package in all_packages]) + "\n".join( + [package for package in packages if not package in all_packages] + ) + "\n" ) new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" @@ -1471,7 +1502,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F elif event["type"] == "commit": chg_categories = [ - x for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) + x + for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) ] signatures["orp"] = "pending" for cat in chg_categories: @@ -1595,7 +1627,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): signatures["tests"] = "started" if ( - get_status_state("%s/unknown/release" % cms_status_prefix, commit_statuses) + get_status_state( + "%s/unknown/release" % cms_status_prefix, commit_statuses + ) == "error" ): signatures["tests"] = "pending" @@ -1618,7 +1652,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F for s in [ i for i in commit_statuses - if ((i.context == scontext) or (i.context.startswith(scontext + "/"))) + if ( + (i.context == scontext) + or (i.context.startswith(scontext + "/")) + ) ]: if (not result_url) and ("/jenkins-artifacts/" in s.target_url): xdata = s.target_url.split("/") @@ -1745,7 +1782,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) else: new_msg = "" - for l in already_seen.body.encode("ascii", "ignore").decode().split("\n"): + for l in ( + already_seen.body.encode("ascii", "ignore").decode().split("\n") + ): if BACKPORT_STR in l: continue new_msg += l + "\n" @@ -1894,7 +1933,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # get release managers SUPER_USERS = read_repo_file(repo_config, "super-users.yaml", []) - releaseManagersList = ", ".join([gh_user_char + x for x in set(releaseManagers + SUPER_USERS)]) + releaseManagersList = ", ".join( + [gh_user_char + x for x in set(releaseManagers + SUPER_USERS)] + ) if cmssw_prs: global_test_params["PULL_REQUESTS"] = cmssw_prs @@ -1939,7 +1980,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) if not dryRun: set_comment_emoji(test_comment.id, repository) - elif abort_test and bot_status and (not bot_status.description.startswith("Aborted")): + elif ( + abort_test + and bot_status + and (not bot_status.description.startswith("Aborted")) + ): if not has_user_emoji(abort_test, repository, "+1", cmsbuild_user): create_properties_file_tests( repository, prId, global_test_params, dryRun, abort=True @@ -2021,7 +2066,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F gh_user_char + name for name, l2_categories in list(CMSSW_L2.items()) for signature in signing_categories - if signature in l2_categories and signature in unsigned and signature not in ["orp"] + if signature in l2_categories + and signature in unsigned + and signature not in ["orp"] ] missing_notifications = set(missing_notifications) @@ -2061,7 +2108,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pkg_msg = [] for pkg in packages: if pkg in package_categories: - pkg_msg.append("- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg]))) + pkg_msg.append( + "- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg])) + ) else: pkg_msg.append("- %s (**new**)" % pkg) messageNewPR = format( @@ -2149,7 +2198,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pre_check not in signatures: signatures[pre_check] = "pending" print( - "PRE CHECK: %s,%s,%s" % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) + "PRE CHECK: %s,%s,%s" + % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) ) if signatures[pre_check] != "pending": if pre_checks_state[pre_check] in ["pending", ""]: @@ -2221,7 +2271,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if not dryRun: emoji = "-1" if "ERRORS: " in test_params_msg else "+1" if e and (e["content"] != emoji): - delete_comment_emoji(str(e["id"]), test_params_comment.id, repository) + delete_comment_emoji( + str(e["id"]), test_params_comment.id, repository + ) state = "success" if emoji == "+1" else "error" last_commit_obj.create_status( state, From 1fe6a6594614c07c5085b326c4294d90cf4c7dde Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 16 Nov 2023 15:01:35 +0100 Subject: [PATCH 12/46] Code-format --- github_utils.py | 61 ++++++++++++------------------------------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/github_utils.py b/github_utils.py index d27ee9cded7f..2ca912b1c24f 100644 --- a/github_utils.py +++ b/github_utils.py @@ -135,9 +135,7 @@ def api_rate_limits(gh, msg=True, when_slow=False, prefix=""): def get_ported_PRs(repo, src_branch, des_branch): done_prs_id = {} - prRe = re.compile( - "Automatically ported from " + src_branch + " #(\d+)\s+.*", re.MULTILINE - ) + prRe = re.compile("Automatically ported from " + src_branch + " #(\d+)\s+.*", re.MULTILINE) for pr in repo.get_pulls(base=des_branch): body = pr.body.encode("ascii", "ignore") if sys.version_info[0] == 3: @@ -265,14 +263,9 @@ def port_pr(repo, pr_num, des_branch, dryRun=False): print(newHead) print(newBody) if not dryRun: - newPR = repo.create_pull( - title=pr.title, body=newBody, base=des_branch, head=newHead - ) + newPR = repo.create_pull(title=pr.title, body=newBody, base=des_branch, head=newHead) else: - print( - "DryRun: should have created Pull Request for %s using %s" - % (des_branch, newHead) - ) + print("DryRun: should have created Pull Request for %s using %s" % (des_branch, newHead)) print("Every thing looks good") git_cmd = format( "cd %(clone_dir)s; git branch -d %(new_branch)s", @@ -448,9 +441,7 @@ def get_failed_pending_members(org): def get_delete_pending_members(org, invitation_id): - return github_api( - "/orgs/%s/invitations/%s" % (org, invitation_id), method="DELETE", raw=True - ) + return github_api("/orgs/%s/invitations/%s" % (org, invitation_id), method="DELETE", raw=True) def get_organization_members(org, role="all", filter="all"): @@ -492,9 +483,7 @@ def edit_pr(repo, pr_num, title=None, body=None, state=None, base=None): params["base"] = base if state: params["state"] = state - return github_api( - uri="/repos/%s/pulls/%s" % (repo, pr_num), params=params, method="PATCH" - ) + return github_api(uri="/repos/%s/pulls/%s" % (repo, pr_num), params=params, method="PATCH") def create_issue_comment(repo, issue_num, body): @@ -506,9 +495,7 @@ def create_issue_comment(repo, issue_num, body): def get_issue_labels(repo, issue_num): get_gh_token(repo) - return github_api( - uri="/repos/%s/issues/%s/labels" % (repo, issue_num), method="GET" - ) + return github_api(uri="/repos/%s/issues/%s/labels" % (repo, issue_num), method="GET") def add_issue_labels(repo, issue_num, labels=[]): @@ -571,11 +558,7 @@ def github_api( headers = {} url = "https://api.github.com%s" % uri data = "" - if ( - per_page - and ("per_page" not in params) - and (not method in ["POST", "PATCH", "PUT"]) - ): + if per_page and ("per_page" not in params) and (not method in ["POST", "PATCH", "PUT"]): params["per_page"] = per_page if method == "GET": if params: @@ -719,9 +702,7 @@ def set_gh_user(user): def get_combined_statuses(commit, repository): get_gh_token(repository) - return github_api( - "/repos/%s/commits/%s/status" % (repository, commit), method="GET" - ) + return github_api("/repos/%s/commits/%s/status" % (repository, commit), method="GET") def get_pr_commits(pr, repository, per_page=None, last_page=False): @@ -776,9 +757,7 @@ def get_repository_issues( def get_issue_comments(repository, issue_num): get_gh_token(repository) - return github_api( - "/repos/%s/issues/%s/comments" % (repository, issue_num), method="GET" - ) + return github_api("/repos/%s/issues/%s/comments" % (repository, issue_num), method="GET") def get_issue(repository, issue_num): @@ -813,8 +792,7 @@ def get_comment_emojis(comment_id, repository): def delete_comment_emoji(emoji_id, comment_id, repository): get_gh_token(repository) return github_api( - "/repos/%s/issues/comments/%s/reactions/%s" - % (repository, comment_id, emoji_id), + "/repos/%s/issues/comments/%s/reactions/%s" % (repository, comment_id, emoji_id), method="DELETE", raw=True, ) @@ -854,9 +832,7 @@ def mark_commit_status( for s in statuses["statuses"]: if s["context"].startswith(context + "/"): params["context"] = s["context"] - github_api( - "/repos/%s/statuses/%s" % (repository, commit), params=params - ) + github_api("/repos/%s/statuses/%s" % (repository, commit), params=params) return @@ -868,9 +844,7 @@ def get_branch(repository, branch_name): def get_git_tag(repository, tag_name): get_gh_token(repository) - data = github_api( - "/repos/%s/git/ref/tags/%s" % (repository, tag_name), method="GET" - ) + data = github_api("/repos/%s/git/ref/tags/%s" % (repository, tag_name), method="GET") return data @@ -922,13 +896,10 @@ def get_org_package_versions(org, package, package_type="container", token_file= ) -def get_org_package_version( - org, package, version_id, package_type="container", token_file=None -): +def get_org_package_version(org, package, version_id, package_type="container", token_file=None): get_gh_token(token_file=token_file) return github_api( - "/orgs/%s/packages/%s/%s/versions/%s" - % (org, package_type, package, version_id), + "/orgs/%s/packages/%s/%s/versions/%s" % (org, package_type, package, version_id), method="GET", ) @@ -951,9 +922,7 @@ def get_commits(repository, branch, until, per_page=1): def find_tags(repository, name): get_gh_token(repository) - data = github_api( - "/repos/%s/git/matching-refs/tags/%s" % (repository, name), method="GET" - ) + data = github_api("/repos/%s/git/matching-refs/tags/%s" % (repository, name), method="GET") return data From 7a2145e5ba3eb147b5f22d64426182e53d6c9efe Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 16 Nov 2023 15:22:12 +0100 Subject: [PATCH 13/46] Code-format --- process_pr.py | 173 ++++++++++++++------------------------------------ 1 file changed, 47 insertions(+), 126 deletions(-) diff --git a/process_pr.py b/process_pr.py index a0df00902983..553864ec4b23 100644 --- a/process_pr.py +++ b/process_pr.py @@ -114,12 +114,8 @@ def format(s, **kwds): arch=ARCH_PATTERN, ) RELVAL_OPTS = "[-][a-zA-Z0-9_.,\s/'-]+" -CLOSE_REQUEST = re.compile( - "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I -) -REOPEN_REQUEST = re.compile( - "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I -) +CLOSE_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I) +REOPEN_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I) CMS_PR_PATTERN = format( "(#[1-9][0-9]*|(%(cmsorgs)s)/+[a-zA-Z0-9_-]+#[1-9][0-9]*|https://+github.com/+(%(cmsorgs)s)/+[a-zA-Z0-9_-]+/+pull/+[1-9][0-9]*)", cmsorgs="|".join(EXTERNAL_REPOS), @@ -353,9 +349,7 @@ def find_last_comment(issue, user, match): def modify_comment(comment, match, replace, dryRun): - comment_msg = ( - comment.body.encode("ascii", "ignore").decode() if comment.body else "" - ) + comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" if match: new_comment_msg = re.sub(match, replace, comment_msg) else: @@ -445,9 +439,7 @@ def check_extra_labels(first_line, extra_labels): def check_type_labels(first_line, extra_labels): ex_labels = {} rem_labels = {} - for type_cmd in [ - x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip() - ]: + for type_cmd in [x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip()]: valid_lab = False rem_lab = type_cmd[0] == "-" if type_cmd[0] in ["-", "+"]: @@ -676,9 +668,7 @@ def cmssw_file2Package(repo_config, filename): def get_jenkins_job(issue): test_line = "" - for line in [ - l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n") - ]: + for line in [l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n")]: if line.startswith("Build logs are available at:"): test_line = line if test_line: @@ -757,21 +747,16 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers = [] # Process Pull Request pkg_categories = set([]) - REGEX_TYPE_CMDS = ( - "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" - ) - REGEX_EX_CMDS = ( - "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" - % (repo.full_name) + REGEX_TYPE_CMDS = "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" + REGEX_EX_CMDS = "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" % ( + repo.full_name ) known_ignore_tests = "%s" % MULTILINE_COMMENTS_MAP["ignore_test(s|)"][0] REGEX_EX_IGNORE_CHKS = "^ignore\s+((%s)(\s*,\s*(%s))*|none)$" % ( known_ignore_tests, known_ignore_tests, ) - REGEX_EX_ENABLE_TESTS = ( - "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] - ) + REGEX_EX_ENABLE_TESTS = "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] L2_DATA = init_l2_data(cms_repo) last_commit_date = None last_commit_obj = None @@ -840,9 +825,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F cms_status_prefix = "cms/%s" % prId bot_status = get_status(bot_status_name, commit_statuses) code_checks_status = [ - s - for s in commit_statuses - if s.context == "%s/code-checks" % cms_status_prefix + s for s in commit_statuses if s.context == "%s/code-checks" % cms_status_prefix ] print("PR Statuses:", commit_statuses) print("#PR Statuses:", len(commit_statuses)) @@ -868,9 +851,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if (not dryRun) and add_labels: - labels = [ - x.name.encode("ascii", "ignore").decode() for x in issue.labels - ] + labels = [x.name.encode("ascii", "ignore").decode() for x in issue.labels] if not "future-commit" in labels: labels.append("future-commit") issue.edit(labels=labels) @@ -893,9 +874,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if repository == CMSSW_REPO_NAME and re.match(CREATE_REPO, issue.title): - with open( - "query-new-data-repo-issues-" + str(issue.number) + ".properties", "w" - ) as f: + with open("query-new-data-repo-issues-" + str(issue.number) + ".properties", "w") as f: f.write("ISSUE_NUMBER=" + str(issue.number) + "\n") # Process the issue comments @@ -947,14 +926,12 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F commenter_categories = get_commenter_categories( commenter, int(comment.created_at.strftime("%s")) ) - valid_commenter = ( - commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org] - ) or (len(commenter_categories) > 0) + valid_commenter = (commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org]) or ( + len(commenter_categories) > 0 + ) if (not valid_commenter) and (requestor != commenter): continue - comment_msg = ( - comment.body.encode("ascii", "ignore").decode() if comment.body else "" - ) + comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" # The first line is an invariant. comment_lines = [l.strip() for l in comment_msg.split("\n") if l.strip()] first_line = comment_lines[0:1] @@ -978,9 +955,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): technical_comment = comment - if ( - commenter == cmsbuild_user - ) and "This PR contains too many commits" in first_line: + if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: warned_too_many_commits = True if commenter in l2s and re.match( @@ -990,9 +965,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F assign_type, new_cats = get_assign_categories(first_line) if new_cats: - if (assign_type == "new categories assigned:") and ( - commenter == cmsbuild_user - ): + if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user): for ex_cat in new_cats: if ex_cat in assign_cats: assign_cats[ex_cat] = 1 @@ -1014,9 +987,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Some of the special users can say "hold" prevent automatic merging of # fully signed PRs. if re.match("^hold$", first_line, re.I): - if commenter_categories or ( - commenter in releaseManagers + PR_HOLD_MANAGERS - ): + if commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): hold[commenter] = 1 continue if re.match(REGEX_EX_CMDS, first_line, re.I): @@ -1037,9 +1008,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F continue if re.match(REGEX_EX_ENABLE_TESTS, first_line, re.I): if valid_commenter: - enable_tests, ignore = check_enable_bot_tests( - first_line.split(" ", 1)[-1] - ) + enable_tests, ignore = check_enable_bot_tests(first_line.split(" ", 1)[-1]) if not dryRun: set_comment_emoji(comment.id, repository, emoji="+1") continue @@ -1054,15 +1023,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if re.match("^unhold$", first_line, re.I): if "orp" in commenter_categories: hold = {} - elif commenter_categories or ( - commenter in releaseManagers + PR_HOLD_MANAGERS - ): + elif commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): if commenter in hold: del hold[commenter] continue - if (commenter == cmsbuild_user) and ( - re.match("^" + HOLD_MSG + ".+", first_line) - ): + if (commenter == cmsbuild_user) and (re.match("^" + HOLD_MSG + ".+", first_line)): for u in first_line.split(HOLD_MSG, 2)[1].split(","): u = u.strip().lstrip("@") if u in hold: @@ -1081,9 +1046,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F (not issue.pull_request) and (commenter in CMSSW_ISSUES_TRACKERS) ): mustClose = False - if (issue.state == "closed") and ( - comment.created_at >= issue.closed_at - ): + if (issue.state == "closed") and (comment.created_at >= issue.closed_at): reOpen = True print("==>Reopen request received from %s" % commenter) continue @@ -1101,9 +1064,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params = dict(test_params) if "ENABLE_BOT_TESTS" in global_test_params: enable_tests = global_test_params["ENABLE_BOT_TESTS"] - test_params_msg = ( - str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) - ) + test_params_msg = str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) continue if cmssw_repo: @@ -1148,9 +1109,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[first_line] = "pending" if first_line not in pre_checks + extra_pre_checks: extra_pre_checks.append(first_line) - if code_checks_status and ( - code_checks_status[0].updated_at >= comment.created_at - ): + if code_checks_status and (code_checks_status[0].updated_at >= comment.created_at): continue if first_line in pre_checks: if pre_checks_state["code-checks"] in ["pending", ""]: @@ -1186,9 +1145,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F IGNORING_TESTS_MSG, first_line ): signatures["tests"] = "pending" - elif re.match( - "Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line - ): + elif re.match("Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line): pull_request_updated = False elif re.match(TRIGERING_TESTS_MSG, first_line) or re.match( TRIGERING_TESTS_MSG1, first_line @@ -1218,9 +1175,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F comp_warnings = False if "+1" in first_line: signatures["tests"] = "approved" - comp_warnings = any( - "Compilation Warnings: Yes" in l for l in comment_lines - ) + comp_warnings = any("Compilation Warnings: Yes" in l for l in comment_lines) pre_checks_url["tests"] = comment.html_url elif "-1" in first_line: signatures["tests"] = "rejected" @@ -1234,9 +1189,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if issue.pull_request or push_test_issue: # Check if the release manager asked for merging this. - if ( - (commenter in releaseManagers) or ("orp" in commenter_categories) - ) and re.match("^\s*(merge)\s*$", first_line, re.I): + if ((commenter in releaseManagers) or ("orp" in commenter_categories)) and re.match( + "^\s*(merge)\s*$", first_line, re.I + ): mustMerge = True mustClose = False if ("orp" in commenter_categories) and ("orp" in signatures): @@ -1245,9 +1200,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Check if the someone asked to trigger the tests if valid_commenter: - ok, v2, v3, v4 = check_test_cmd( - first_line, repository, global_test_params - ) + ok, v2, v3, v4 = check_test_cmd(first_line, repository, global_test_params) if ok: test_comment = comment abort_test = None @@ -1269,9 +1222,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Comment message:", first_line) signatures["tests"] = "pending" continue - elif REGEX_TEST_ABORT.match(first_line) and ( - signatures["tests"] == "pending" - ): + elif REGEX_TEST_ABORT.match(first_line) and (signatures["tests"] == "pending"): abort_test = comment test_comment = None signatures["tests"] = "pending" @@ -1370,17 +1321,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signing_categories.add("code-checks") updateMilestone(repo, issue, pr, dryRun) - packages = sorted( - list(set(cmssw_file2Package(repo_config, f) for f in chg_files)) - ) + packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) for pkg_file in chg_files: for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): for regex in pkgs_regexp: if regex.match(pkg_file): extra_labels["mtype"].append(ex_lab) print( - "Non-Blocking label:%s:%s:%s" - % (ex_lab, regex.pattern, pkg_file) + "Non-Blocking label:%s:%s:%s" % (ex_lab, regex.pattern, pkg_file) ) break if not extra_labels["mtype"]: @@ -1394,9 +1342,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ex_pkg = external_to_package(repository) if ex_pkg: packages.add(ex_pkg) - if (repo_org != GH_CMSSW_ORGANIZATION) or ( - repo_name in VALID_CMS_SW_REPOS_FOR_TESTS - ): + if (repo_org != GH_CMSSW_ORGANIZATION) or (repo_name in VALID_CMS_SW_REPOS_FOR_TESTS): create_test_property = True if (repo_name == GH_CMSDIST_REPO) and ( not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) @@ -1435,13 +1381,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ] has_category = all([package in all_packages for package in packages]) if not has_category: - new_package_message = ( - "\nThe following packages do not have a category, yet:\n\n" - ) + new_package_message = "\nThe following packages do not have a category, yet:\n\n" new_package_message += ( - "\n".join( - [package for package in packages if not package in all_packages] - ) + "\n".join([package for package in packages if not package in all_packages]) + "\n" ) new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" @@ -1502,8 +1444,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F elif event["type"] == "commit": chg_categories = [ - x - for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) + x for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) ] signatures["orp"] = "pending" for cat in chg_categories: @@ -1627,9 +1568,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): signatures["tests"] = "started" if ( - get_status_state( - "%s/unknown/release" % cms_status_prefix, commit_statuses - ) + get_status_state("%s/unknown/release" % cms_status_prefix, commit_statuses) == "error" ): signatures["tests"] = "pending" @@ -1652,10 +1591,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F for s in [ i for i in commit_statuses - if ( - (i.context == scontext) - or (i.context.startswith(scontext + "/")) - ) + if ((i.context == scontext) or (i.context.startswith(scontext + "/"))) ]: if (not result_url) and ("/jenkins-artifacts/" in s.target_url): xdata = s.target_url.split("/") @@ -1782,9 +1718,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) else: new_msg = "" - for l in ( - already_seen.body.encode("ascii", "ignore").decode().split("\n") - ): + for l in already_seen.body.encode("ascii", "ignore").decode().split("\n"): if BACKPORT_STR in l: continue new_msg += l + "\n" @@ -1933,9 +1867,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # get release managers SUPER_USERS = read_repo_file(repo_config, "super-users.yaml", []) - releaseManagersList = ", ".join( - [gh_user_char + x for x in set(releaseManagers + SUPER_USERS)] - ) + releaseManagersList = ", ".join([gh_user_char + x for x in set(releaseManagers + SUPER_USERS)]) if cmssw_prs: global_test_params["PULL_REQUESTS"] = cmssw_prs @@ -1980,11 +1912,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) if not dryRun: set_comment_emoji(test_comment.id, repository) - elif ( - abort_test - and bot_status - and (not bot_status.description.startswith("Aborted")) - ): + elif abort_test and bot_status and (not bot_status.description.startswith("Aborted")): if not has_user_emoji(abort_test, repository, "+1", cmsbuild_user): create_properties_file_tests( repository, prId, global_test_params, dryRun, abort=True @@ -2066,9 +1994,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F gh_user_char + name for name, l2_categories in list(CMSSW_L2.items()) for signature in signing_categories - if signature in l2_categories - and signature in unsigned - and signature not in ["orp"] + if signature in l2_categories and signature in unsigned and signature not in ["orp"] ] missing_notifications = set(missing_notifications) @@ -2108,9 +2034,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pkg_msg = [] for pkg in packages: if pkg in package_categories: - pkg_msg.append( - "- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg])) - ) + pkg_msg.append("- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg]))) else: pkg_msg.append("- %s (**new**)" % pkg) messageNewPR = format( @@ -2198,8 +2122,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pre_check not in signatures: signatures[pre_check] = "pending" print( - "PRE CHECK: %s,%s,%s" - % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) + "PRE CHECK: %s,%s,%s" % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) ) if signatures[pre_check] != "pending": if pre_checks_state[pre_check] in ["pending", ""]: @@ -2271,9 +2194,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if not dryRun: emoji = "-1" if "ERRORS: " in test_params_msg else "+1" if e and (e["content"] != emoji): - delete_comment_emoji( - str(e["id"]), test_params_comment.id, repository - ) + delete_comment_emoji(str(e["id"]), test_params_comment.id, repository) state = "success" if emoji == "+1" else "error" last_commit_obj.create_status( state, From 995050b9c22faff8b1df3925d959c3f23829091d Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 20 Nov 2023 10:06:29 +0100 Subject: [PATCH 14/46] Fix too-many-commits handling --- process_pr.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/process_pr.py b/process_pr.py index 553864ec4b23..d4d98443f6e8 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1246,11 +1246,13 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F and not ok_too_many_commits and pr.commits >= MAX_INITIAL_COMMITS_IN_PR ): - issue.create_comment( - f"This PR contains too many commits ({pr.commits} > {MAX_INITIAL_COMMITS_IN_PR}). " - "Make sure you chose the right target branch.\n" - f"{l2s}, you can override this check with `+commit-count`." - ) + if not dryRun: + issue.create_comment( + f"This PR contains too many commits ({pr.commits} > {MAX_INITIAL_COMMITS_IN_PR}). " + "Make sure you chose the right target branch.\n" + f"{l2s}, you can override this check with `+commit-count`." + ) + return # Get the commit cache from `already_seen` commit or technical commit print("Recalculating signatures") From d785a37c3fde2f763720921093639e524d2fcf35 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Fri, 24 Nov 2023 16:36:55 +0100 Subject: [PATCH 15/46] Convert packages to categories; fix timestamp handling --- process_pr.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/process_pr.py b/process_pr.py index d4d98443f6e8..229e3e78424d 100644 --- a/process_pr.py +++ b/process_pr.py @@ -30,7 +30,7 @@ from githublabels import TYPE_COMMANDS, TEST_IGNORE_REASON from repo_config import GH_REPO_ORGANIZATION import re, time -from datetime import datetime +from datetime import datetime, timezone from os.path import join, exists, dirname from os import environ from github_utils import edit_pr, api_rate_limits, get_pr_commits_reversed @@ -843,7 +843,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Latest commit sha: ", last_commit.sha) print("PR update time", pr.updated_at) print("Time UTC:", datetime.utcnow()) - if last_commit_date > datetime.utcnow(): + if last_commit_date > datetime.now(timezone.utc): print("==== Future commit found ====") add_labels = True try: @@ -1281,7 +1281,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F } cache_entry = commit_cache[commit.sha] - events[datetime.fromtimestamp(cache_entry["time"])] = { + events[datetime.fromtimestamp(cache_entry["time"], timezone.utc)] = { "type": "commit", "value": cache_entry["files"], } @@ -1445,9 +1445,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F mustClose = False elif event["type"] == "commit": - chg_categories = [ - x for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) - ] + chg_categories = set() + for fn in event["value"]: + chg_categories.update(get_package_categories(cmssw_file2Package(repo_config, fn))) + signatures["orp"] = "pending" for cat in chg_categories: signatures[cat] = "pending" From 758e56a2011f76773b38e038ec36b995a2cb3514 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Fri, 24 Nov 2023 17:14:06 +0100 Subject: [PATCH 16/46] Use API to get list of changed files --- process_pr.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/process_pr.py b/process_pr.py index 229e3e78424d..498e542aa842 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1270,7 +1270,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if seen_commits_match: commit_cache = loads(seen_commits_match[1]) - chg_files = set() + chg_files = get_changed_files(repo, pr) if pr.commits < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: for commit in all_commits: @@ -1286,8 +1286,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F "value": cache_entry["files"], } - chg_files.update(cache_entry["files"]) - old_body = cache_comment.body if cache_comment else CMSBOT_TECHNICAL_MSG new_body = ( REGEX_COMMITS_CACHE.sub("", old_body) @@ -1451,7 +1449,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures["orp"] = "pending" for cat in chg_categories: - signatures[cat] = "pending" + if cat in signing_categories: + signatures[cat] = "pending" if push_test_issue: auto_close_push_test_issue = True From 9ac15d3f6ac51391c481b527ae719efe2ddd0252 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Fri, 1 Dec 2023 13:54:10 +0100 Subject: [PATCH 17/46] Fix tests, code-checks signatures; restorre special handling for ORP signatures --- process_pr.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/process_pr.py b/process_pr.py index 498e542aa842..b496389bf82e 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1091,6 +1091,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ctype = first_line[0] + "1" selected_cats = [category_name] + if "orp" in selected_cats: + # Lines 1281, 1286 of original code + mustClose = False + if selected_cats: events[comment.created_at] = { "type": "sign", @@ -1135,9 +1139,25 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F elif "-code-checks" == first_line: signatures["code-checks"] = "rejected" pre_checks_url["code-checks"] = comment.html_url + events[comment.created_at] = { + "type": "sign", + "value": { + "ctype": "-1", + "selected_cats": ["code-checks"], + "comment": comment, + }, + } elif "+code-checks" == first_line: signatures["code-checks"] = "approved" pre_checks_url["code-checks"] = comment.html_url + events[comment.created_at] = { + "type": "sign", + "value": { + "ctype": "+1", + "selected_cats": ["code-checks"], + "comment": comment, + }, + } elif re.match("^Comparison not run.+", first_line): if ("tests" in signatures) and signatures["tests"] != "pending": comparison_notrun = True @@ -1177,9 +1197,26 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures["tests"] = "approved" comp_warnings = any("Compilation Warnings: Yes" in l for l in comment_lines) pre_checks_url["tests"] = comment.html_url + events[comment.created_at] = { + "type": "sign", + "value": { + "ctype": "+1", + "selected_cats": ["tests"], + "comment": comment, + }, + } + elif "-1" in first_line: signatures["tests"] = "rejected" pre_checks_url["tests"] = comment.html_url + events[comment.created_at] = { + "type": "sign", + "value": { + "ctype": "-1", + "selected_cats": ["tests"], + "comment": comment, + }, + } else: signatures["tests"] = "pending" print( @@ -1196,6 +1233,13 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F mustClose = False if ("orp" in commenter_categories) and ("orp" in signatures): signatures["orp"] = "approved" + events[comment.created_at] = { + "type": "sign", + "value": { + "ctype": "+1", + "selected_cats": ["orp"], + "comment": comment, + },} continue # Check if the someone asked to trigger the tests @@ -1255,7 +1299,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F return # Get the commit cache from `already_seen` commit or technical commit - print("Recalculating signatures") + print("Loading commit cache") cache_comment = None if technical_comment: @@ -1422,8 +1466,13 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers = set([gh_user_char + u for u in watchers]) print("Watchers " + ", ".join(watchers)) + print("Recalculating signatures") + print("signatures:", signatures) + print("signing_categories:", signing_categories) + for event in events.values(): if event["type"] == "sign": + print("Sign:", event["value"]) selected_cats = event["value"]["selected_cats"] ctype = event["value"]["ctype"] if any(x in signing_categories for x in selected_cats): @@ -1434,20 +1483,20 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F (repository in auto_test_repo) or ("*" in auto_test_repo) ): test_comment = event["value"]["comment"] - if sign == "orp": - mustClose = False + # if sign == "orp": + # mustClose = False elif ctype == "-1": for sign in selected_cats: signatures[sign] = "rejected" if sign == "orp": mustClose = False - + else: + print(f"Ignoring event: {signing_categories}, {selected_cats}") elif event["type"] == "commit": - chg_categories = set() + chg_categories = {"orp", "tests", "code-checks"} for fn in event["value"]: chg_categories.update(get_package_categories(cmssw_file2Package(repo_config, fn))) - signatures["orp"] = "pending" for cat in chg_categories: if cat in signing_categories: signatures[cat] = "pending" From c057d8ed849d78b259c2c514a89c61b17246fdef Mon Sep 17 00:00:00 2001 From: iarspider Date: Fri, 1 Dec 2023 14:06:23 +0100 Subject: [PATCH 18/46] Move code to determine signing-cats back to where it was --- process_pr.py | 278 +++++++++++++++++++++++++------------------------- 1 file changed, 140 insertions(+), 138 deletions(-) diff --git a/process_pr.py b/process_pr.py index b496389bf82e..148c627e1c8e 100644 --- a/process_pr.py +++ b/process_pr.py @@ -808,6 +808,124 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if is_closed_branch(pr.base.ref): mustClose = True + if issue.pull_request: + pr = repo.get_pull(prId) + + # Process the changes for the given pull request so that we can determine the + # signatures it requires. + if cmssw_repo or not external_repo: + if cmssw_repo: + if pr.base.ref == "master": + signing_categories.add("code-checks") + updateMilestone(repo, issue, pr, dryRun) + + chg_files = get_changed_files(repo, pr) + packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) + for pkg_file in chg_files: + for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): + for regex in pkgs_regexp: + if regex.match(pkg_file): + extra_labels["mtype"].append(ex_lab) + print( + "Non-Blocking label:%s:%s:%s" + % (ex_lab, regex.pattern, pkg_file) + ) + break + if not extra_labels["mtype"]: + del extra_labels["mtype"] + print("Extra non-blocking labels:", extra_labels) + print("First Package: ", packages[0]) + create_test_property = True + else: + add_external_category = True + packages = {"externals/" + repository} + ex_pkg = external_to_package(repository) + if ex_pkg: + packages.add(ex_pkg) + if (repo_org != GH_CMSSW_ORGANIZATION) or ( + repo_name in VALID_CMS_SW_REPOS_FOR_TESTS + ): + create_test_property = True + if (repo_name == GH_CMSDIST_REPO) and ( + not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) + ): + print("Skipping PR as it does not belong to valid CMSDIST branch") + return + + print("Following packages affected:") + print("\n".join(packages)) + for package in packages: + package_categories[package] = set([]) + for category in get_package_categories(package): + package_categories[package].add(category) + pkg_categories.add(category) + signing_categories.update(pkg_categories) + + # For PR, we always require tests. + signing_categories.add("tests") + if add_external_category: + signing_categories.add("externals") + if cms_repo: + print("This pull request requires ORP approval") + signing_categories.add("orp") + + print("Following categories affected:") + print("\n".join(signing_categories)) + + signatures = dict([(x, "pending") for x in signing_categories]) + + if cmssw_repo: + # If there is a new package, add also a dummy "new" category. + all_packages = [ + package + for category_packages in list(CMSSW_CATEGORIES.values()) + for package in category_packages + ] + has_category = all([package in all_packages for package in packages]) + if not has_category: + new_package_message = ( + "\nThe following packages do not have a category, yet:\n\n" + ) + new_package_message += ( + "\n".join([package for package in packages if not package in all_packages]) + + "\n" + ) + new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" + print(new_package_message) + signing_categories.add("new-package") + + # Add watchers.yaml information to the WATCHERS dict. + WATCHERS = read_repo_file(repo_config, "watchers.yaml", {}) + # Given the files modified by the PR, check if there are additional developers watching one or more. + author = pr.user.login + watchers = set( + [ + user + for chg_file in chg_files + for user, watched_regexp in list(WATCHERS.items()) + for regexp in watched_regexp + if re.match("^" + regexp + ".*", chg_file) and user != author + ] + ) + # Handle category watchers + catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {}) + non_block_cats = [] if not "mtype" in extra_labels else extra_labels["mtype"] + for user, cats in list(catWatchers.items()): + for cat in cats: + if (cat in signing_categories) or (cat in non_block_cats): + print("Added ", user, " to watch due to cat", cat) + watchers.add(user) + + # Handle watchers + watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) + for watcher in [x for x in watchers]: + if watcher not in watchingGroups: + continue + watchers.remove(watcher) + watchers.update(set(watchingGroups[watcher])) + watchers = set([gh_user_char + u for u in watchers]) + print("Watchers " + ", ".join(watchers)) + all_commits = get_pr_commits_reversed(pr) if all_commits: @@ -1198,24 +1316,24 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F comp_warnings = any("Compilation Warnings: Yes" in l for l in comment_lines) pre_checks_url["tests"] = comment.html_url events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "+1", - "selected_cats": ["tests"], - "comment": comment, - }, - } + "type": "sign", + "value": { + "ctype": "+1", + "selected_cats": ["tests"], + "comment": comment, + }, + } elif "-1" in first_line: signatures["tests"] = "rejected" pre_checks_url["tests"] = comment.html_url events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "-1", - "selected_cats": ["tests"], - "comment": comment, - }, + "type": "sign", + "value": { + "ctype": "-1", + "selected_cats": ["tests"], + "comment": comment, + }, } else: signatures["tests"] = "pending" @@ -1234,12 +1352,13 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if ("orp" in commenter_categories) and ("orp" in signatures): signatures["orp"] = "approved" events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "+1", - "selected_cats": ["orp"], - "comment": comment, - },} + "type": "sign", + "value": { + "ctype": "+1", + "selected_cats": ["orp"], + "comment": comment, + }, + } continue # Check if the someone asked to trigger the tests @@ -1299,7 +1418,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F return # Get the commit cache from `already_seen` commit or technical commit - print("Loading commit cache") + print("Searching for commit cache") cache_comment = None if technical_comment: @@ -1309,13 +1428,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F cache_comment = already_seen if cache_comment: - print("Loading commit cache") seen_commits_match = REGEX_COMMITS_CACHE.search(cache_comment.body) if seen_commits_match: + print("Loading commit cache") commit_cache = loads(seen_commits_match[1]) - chg_files = get_changed_files(repo, pr) - if pr.commits < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: for commit in all_commits: if commit.sha not in commit_cache: @@ -1353,122 +1470,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F events = dict(sorted(events.items())) print("Events:", events) - - if issue.pull_request: - pr = repo.get_pull(prId) - - # Process the changes for the given pull request so that we can determine the - # signatures it requires. - if cmssw_repo or not external_repo: - if cmssw_repo: - if pr.base.ref == "master": - signing_categories.add("code-checks") - updateMilestone(repo, issue, pr, dryRun) - - packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) - for pkg_file in chg_files: - for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): - for regex in pkgs_regexp: - if regex.match(pkg_file): - extra_labels["mtype"].append(ex_lab) - print( - "Non-Blocking label:%s:%s:%s" % (ex_lab, regex.pattern, pkg_file) - ) - break - if not extra_labels["mtype"]: - del extra_labels["mtype"] - print("Extra non-blocking labels:", extra_labels) - print("First Package: ", packages[0]) - create_test_property = True - else: - add_external_category = True - packages = {"externals/" + repository} - ex_pkg = external_to_package(repository) - if ex_pkg: - packages.add(ex_pkg) - if (repo_org != GH_CMSSW_ORGANIZATION) or (repo_name in VALID_CMS_SW_REPOS_FOR_TESTS): - create_test_property = True - if (repo_name == GH_CMSDIST_REPO) and ( - not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) - ): - print("Skipping PR as it does not belong to valid CMSDIST branch") - return - - print("Following packages affected:") - print("\n".join(packages)) - for package in packages: - package_categories[package] = set([]) - for category in get_package_categories(package): - package_categories[package].add(category) - pkg_categories.add(category) - signing_categories.update(pkg_categories) - - # For PR, we always require tests. - signing_categories.add("tests") - if add_external_category: - signing_categories.add("externals") - if cms_repo: - print("This pull request requires ORP approval") - signing_categories.add("orp") - - print("Following categories affected:") - print("\n".join(signing_categories)) - - signatures = dict([(x, "pending") for x in signing_categories]) - - if cmssw_repo: - # If there is a new package, add also a dummy "new" category. - all_packages = [ - package - for category_packages in list(CMSSW_CATEGORIES.values()) - for package in category_packages - ] - has_category = all([package in all_packages for package in packages]) - if not has_category: - new_package_message = "\nThe following packages do not have a category, yet:\n\n" - new_package_message += ( - "\n".join([package for package in packages if not package in all_packages]) - + "\n" - ) - new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" - print(new_package_message) - signing_categories.add("new-package") - - # Add watchers.yaml information to the WATCHERS dict. - WATCHERS = read_repo_file(repo_config, "watchers.yaml", {}) - # Given the files modified by the PR, check if there are additional developers watching one or more. - author = pr.user.login - watchers = set( - [ - user - for chg_file in chg_files - for user, watched_regexp in list(WATCHERS.items()) - for regexp in watched_regexp - if re.match("^" + regexp + ".*", chg_file) and user != author - ] - ) - # Handle category watchers - catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {}) - non_block_cats = [] if not "mtype" in extra_labels else extra_labels["mtype"] - for user, cats in list(catWatchers.items()): - for cat in cats: - if (cat in signing_categories) or (cat in non_block_cats): - print("Added ", user, " to watch due to cat", cat) - watchers.add(user) - - # Handle watchers - watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) - for watcher in [x for x in watchers]: - if watcher not in watchingGroups: - continue - watchers.remove(watcher) - watchers.update(set(watchingGroups[watcher])) - watchers = set([gh_user_char + u for u in watchers]) - print("Watchers " + ", ".join(watchers)) - print("Recalculating signatures") - print("signatures:", signatures) - print("signing_categories:", signing_categories) for event in events.values(): if event["type"] == "sign": From e5bc0f7531954b1f47e40497639940742bad32b2 Mon Sep 17 00:00:00 2001 From: iarspider Date: Fri, 1 Dec 2023 15:20:07 +0100 Subject: [PATCH 19/46] Remove extra check --- process_pr.py | 222 ++++++++++++++++++++++++-------------------------- 1 file changed, 107 insertions(+), 115 deletions(-) diff --git a/process_pr.py b/process_pr.py index 148c627e1c8e..c9d810ad4d0a 100644 --- a/process_pr.py +++ b/process_pr.py @@ -808,123 +808,115 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if is_closed_branch(pr.base.ref): mustClose = True - if issue.pull_request: - pr = repo.get_pull(prId) - - # Process the changes for the given pull request so that we can determine the - # signatures it requires. - if cmssw_repo or not external_repo: - if cmssw_repo: - if pr.base.ref == "master": - signing_categories.add("code-checks") - updateMilestone(repo, issue, pr, dryRun) - - chg_files = get_changed_files(repo, pr) - packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) - for pkg_file in chg_files: - for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): - for regex in pkgs_regexp: - if regex.match(pkg_file): - extra_labels["mtype"].append(ex_lab) - print( - "Non-Blocking label:%s:%s:%s" - % (ex_lab, regex.pattern, pkg_file) - ) - break - if not extra_labels["mtype"]: - del extra_labels["mtype"] - print("Extra non-blocking labels:", extra_labels) - print("First Package: ", packages[0]) + # Process the changes for the given pull request so that we can determine the + # signatures it requires. + if cmssw_repo or not external_repo: + if cmssw_repo: + if pr.base.ref == "master": + signing_categories.add("code-checks") + updateMilestone(repo, issue, pr, dryRun) + + chg_files = get_changed_files(repo, pr) + packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) + for pkg_file in chg_files: + for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): + for regex in pkgs_regexp: + if regex.match(pkg_file): + extra_labels["mtype"].append(ex_lab) + print( + "Non-Blocking label:%s:%s:%s" % (ex_lab, regex.pattern, pkg_file) + ) + break + if not extra_labels["mtype"]: + del extra_labels["mtype"] + print("Extra non-blocking labels:", extra_labels) + print("First Package: ", packages[0]) + create_test_property = True + else: + add_external_category = True + packages = {"externals/" + repository} + ex_pkg = external_to_package(repository) + if ex_pkg: + packages.add(ex_pkg) + if (repo_org != GH_CMSSW_ORGANIZATION) or (repo_name in VALID_CMS_SW_REPOS_FOR_TESTS): create_test_property = True - else: - add_external_category = True - packages = {"externals/" + repository} - ex_pkg = external_to_package(repository) - if ex_pkg: - packages.add(ex_pkg) - if (repo_org != GH_CMSSW_ORGANIZATION) or ( - repo_name in VALID_CMS_SW_REPOS_FOR_TESTS - ): - create_test_property = True - if (repo_name == GH_CMSDIST_REPO) and ( - not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) - ): - print("Skipping PR as it does not belong to valid CMSDIST branch") - return - - print("Following packages affected:") - print("\n".join(packages)) - for package in packages: - package_categories[package] = set([]) - for category in get_package_categories(package): - package_categories[package].add(category) - pkg_categories.add(category) - signing_categories.update(pkg_categories) - - # For PR, we always require tests. - signing_categories.add("tests") - if add_external_category: - signing_categories.add("externals") - if cms_repo: - print("This pull request requires ORP approval") - signing_categories.add("orp") - - print("Following categories affected:") - print("\n".join(signing_categories)) - - signatures = dict([(x, "pending") for x in signing_categories]) + if (repo_name == GH_CMSDIST_REPO) and ( + not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) + ): + print("Skipping PR as it does not belong to valid CMSDIST branch") + return + + print("Following packages affected:") + print("\n".join(packages)) + for package in packages: + package_categories[package] = set([]) + for category in get_package_categories(package): + package_categories[package].add(category) + pkg_categories.add(category) + signing_categories.update(pkg_categories) + + # For PR, we always require tests. + signing_categories.add("tests") + if add_external_category: + signing_categories.add("externals") + if cms_repo: + print("This pull request requires ORP approval") + signing_categories.add("orp") + + print("Following categories affected:") + print("\n".join(signing_categories)) + + signatures = dict([(x, "pending") for x in signing_categories]) - if cmssw_repo: - # If there is a new package, add also a dummy "new" category. - all_packages = [ - package - for category_packages in list(CMSSW_CATEGORIES.values()) - for package in category_packages - ] - has_category = all([package in all_packages for package in packages]) - if not has_category: - new_package_message = ( - "\nThe following packages do not have a category, yet:\n\n" - ) - new_package_message += ( - "\n".join([package for package in packages if not package in all_packages]) - + "\n" - ) - new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" - print(new_package_message) - signing_categories.add("new-package") - - # Add watchers.yaml information to the WATCHERS dict. - WATCHERS = read_repo_file(repo_config, "watchers.yaml", {}) - # Given the files modified by the PR, check if there are additional developers watching one or more. - author = pr.user.login - watchers = set( - [ - user - for chg_file in chg_files - for user, watched_regexp in list(WATCHERS.items()) - for regexp in watched_regexp - if re.match("^" + regexp + ".*", chg_file) and user != author - ] - ) - # Handle category watchers - catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {}) - non_block_cats = [] if not "mtype" in extra_labels else extra_labels["mtype"] - for user, cats in list(catWatchers.items()): - for cat in cats: - if (cat in signing_categories) or (cat in non_block_cats): - print("Added ", user, " to watch due to cat", cat) - watchers.add(user) - - # Handle watchers - watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) - for watcher in [x for x in watchers]: - if watcher not in watchingGroups: - continue - watchers.remove(watcher) - watchers.update(set(watchingGroups[watcher])) - watchers = set([gh_user_char + u for u in watchers]) - print("Watchers " + ", ".join(watchers)) + if cmssw_repo: + # If there is a new package, add also a dummy "new" category. + all_packages = [ + package + for category_packages in list(CMSSW_CATEGORIES.values()) + for package in category_packages + ] + has_category = all([package in all_packages for package in packages]) + if not has_category: + new_package_message = "\nThe following packages do not have a category, yet:\n\n" + new_package_message += ( + "\n".join([package for package in packages if not package in all_packages]) + + "\n" + ) + new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" + print(new_package_message) + signing_categories.add("new-package") + + # Add watchers.yaml information to the WATCHERS dict. + WATCHERS = read_repo_file(repo_config, "watchers.yaml", {}) + # Given the files modified by the PR, check if there are additional developers watching one or more. + author = pr.user.login + watchers = set( + [ + user + for chg_file in chg_files + for user, watched_regexp in list(WATCHERS.items()) + for regexp in watched_regexp + if re.match("^" + regexp + ".*", chg_file) and user != author + ] + ) + # Handle category watchers + catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {}) + non_block_cats = [] if not "mtype" in extra_labels else extra_labels["mtype"] + for user, cats in list(catWatchers.items()): + for cat in cats: + if (cat in signing_categories) or (cat in non_block_cats): + print("Added ", user, " to watch due to cat", cat) + watchers.add(user) + + # Handle watchers + watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) + for watcher in [x for x in watchers]: + if watcher not in watchingGroups: + continue + watchers.remove(watcher) + watchers.update(set(watchingGroups[watcher])) + watchers = set([gh_user_char + u for u in watchers]) + print("Watchers " + ", ".join(watchers)) all_commits = get_pr_commits_reversed(pr) From 0ee9c9ea275e4846a2308e38a109e3950a5cedc3 Mon Sep 17 00:00:00 2001 From: iarspider Date: Fri, 1 Dec 2023 15:38:09 +0100 Subject: [PATCH 20/46] Change wording --- process_pr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process_pr.py b/process_pr.py index c9d810ad4d0a..1cf80446c067 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1410,7 +1410,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F return # Get the commit cache from `already_seen` commit or technical commit - print("Searching for commit cache") + print("Checking for commit cache") cache_comment = None if technical_comment: From 042b550f7d879d52c9ec497bc5c94934ca163d07 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 4 Dec 2023 10:59:37 +0100 Subject: [PATCH 21/46] Remove extra formatting --- github_utils.py | 49 +++--------- process_pr.py | 194 ++++++++++++++++-------------------------------- 2 files changed, 73 insertions(+), 170 deletions(-) diff --git a/github_utils.py b/github_utils.py index 2ca912b1c24f..416e61039c38 100644 --- a/github_utils.py +++ b/github_utils.py @@ -45,12 +45,7 @@ def get_page_range(): def _check_rate_limits( - rate_limit, - rate_limit_max, - rate_limiting_resettime, - msg=True, - when_slow=False, - prefix="", + rate_limit, rate_limit_max, rate_limiting_resettime, msg=True, when_slow=False, prefix="" ): global GH_TOKENS, GH_TOKEN_INDEX from calendar import timegm @@ -103,12 +98,7 @@ def _check_rate_limits( def check_rate_limits(msg=True, when_slow=False, prefix=""): _check_rate_limits( - GH_RATE_LIMIT[0], - GH_RATE_LIMIT[1], - GH_RATE_LIMIT[2], - msg, - when_slow, - prefix=prefix, + GH_RATE_LIMIT[0], GH_RATE_LIMIT[1], GH_RATE_LIMIT[2], msg, when_slow, prefix=prefix ) @@ -466,9 +456,7 @@ def add_organization_member(org, member, role="member"): def invite_organization_member(org, member, role="direct_member"): return github_api( - "/orgs/%s/invitations" % org, - params={"role": role, "invitee_id": member}, - method="POST", + "/orgs/%s/invitations" % org, params={"role": role, "invitee_id": member}, method="POST" ) @@ -519,9 +507,7 @@ def set_issue_labels(repo, issue_num, labels=[]): def remove_issue_labels_all(repo, issue_num): get_gh_token(repo) return github_api( - uri="/repos/%s/issues/%s/labels" % (repo, issue_num), - method="DELETE", - status=[204], + uri="/repos/%s/issues/%s/labels" % (repo, issue_num), method="DELETE", status=[204] ) @@ -621,14 +607,7 @@ def github_api( if max_pages > 0 and page > max_pages: break data += github_api( - uri, - params, - method, - headers, - page, - raw=raw, - per_page=per_page, - all_pages=False, + uri, params, method, headers, page, raw=raw, per_page=per_page, all_pages=False ) return data @@ -737,8 +716,7 @@ def set_comment_emoji(comment_id, repository, emoji="+1", reset_other=True): get_gh_token(repository) params = {"content": emoji} return github_api( - "/repos/%s/issues/comments/%s/reactions" % (repository, comment_id), - params=params, + "/repos/%s/issues/comments/%s/reactions" % (repository, comment_id), params=params ) @@ -784,8 +762,7 @@ def get_release_by_tag(repository, tag): def get_comment_emojis(comment_id, repository): get_gh_token(repository) return github_api( - "/repos/%s/issues/comments/%s/reactions" % (repository, comment_id), - method="GET", + "/repos/%s/issues/comments/%s/reactions" % (repository, comment_id), method="GET" ) @@ -813,12 +790,7 @@ def mark_commit_status( reset=False, ): get_gh_token(repository) - params = { - "state": state, - "target_url": url, - "description": description, - "context": context, - } + params = {"state": state, "target_url": url, "description": description, "context": context} github_api("/repos/%s/statuses/%s" % (repository, commit), params=params) if reset: statuses = get_combined_statuses(commit, repository) @@ -881,9 +853,7 @@ def get_org_packages(org, package_type="container", visibility=None, token_file= def get_org_package(org, package, package_type="container", token_file=None): get_gh_token(token_file=token_file) return github_api( - "/orgs/%s/packages/%s/%s" % (org, package_type, package), - method="GET", - all_pages=True, + "/orgs/%s/packages/%s/%s" % (org, package_type, package), method="GET", all_pages=True ) @@ -943,7 +913,6 @@ def get_last_commit(pr): def get_pr_commits_reversed(pr): """ - :param pr: :return: PaginatedList[Commit] | List[Commit] """ diff --git a/process_pr.py b/process_pr.py index 1cf80446c067..ac0184ca561a 100644 --- a/process_pr.py +++ b/process_pr.py @@ -8,11 +8,7 @@ CMSDIST_REPOS, ) from categories import CMSSW_CATEGORIES -from releases import ( - RELEASE_BRANCH_MILESTONE, - RELEASE_BRANCH_PRODUCTION, - CMSSW_DEVEL_BRANCH, -) +from releases import RELEASE_BRANCH_MILESTONE, RELEASE_BRANCH_PRODUCTION, CMSSW_DEVEL_BRANCH from cms_static import ( VALID_CMSDIST_BRANCHES, NEW_ISSUE_PREFIX, @@ -34,12 +30,7 @@ from os.path import join, exists, dirname from os import environ from github_utils import edit_pr, api_rate_limits, get_pr_commits_reversed -from github_utils import ( - set_comment_emoji, - get_comment_emojis, - delete_comment_emoji, - set_gh_user, -) +from github_utils import set_comment_emoji, get_comment_emojis, delete_comment_emoji, set_gh_user from socket import setdefaulttimeout from _py2with3compatibility import run_cmd from json import dumps, load, loads @@ -109,9 +100,7 @@ def format(s, **kwds): CMSSW_PACKAGE_PATTERN = "[A-Z][a-zA-Z0-9]+(/[a-zA-Z0-9]+|)" ARCH_PATTERN = "[a-z0-9]+_[a-z0-9]+_[a-z0-9]+" CMSSW_RELEASE_QUEUE_PATTERN = format( - "(%(cmssw)s|%(arch)s|%(cmssw)s/%(arch)s)", - cmssw=CMSSW_QUEUE_PATTERN, - arch=ARCH_PATTERN, + "(%(cmssw)s|%(arch)s|%(cmssw)s/%(arch)s)", cmssw=CMSSW_QUEUE_PATTERN, arch=ARCH_PATTERN ) RELVAL_OPTS = "[-][a-zA-Z0-9_.,\s/'-]+" CLOSE_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I) @@ -137,8 +126,6 @@ def format(s, **kwds): r"^\s*(?:(?:@|)cmsbuild\s*[,]*\s+|)(?:please\s*[,]*\s+|)ignore\s+tests-rejected\s+(?:with|)([a-z -]+)$", re.I, ) -REGEX_COMMITS_CACHE = re.compile(r"") -REGEX_IGNORE_COMMIT_COUNT = "\+commit-count" TEST_WAIT_GAP = 720 ALL_CHECK_FUNCTIONS = None EXTRA_RELVALS_TESTS = ["threading", "gpu", "high-stats", "nano"] @@ -150,10 +137,7 @@ def format(s, **kwds): MULTILINE_COMMENTS_MAP = { "(workflow|relval)(s|)(" + EXTRA_RELVALS_TESTS_OPTS - + "|)": [ - format("%(workflow)s(\s*,\s*%(workflow)s|)*", workflow=WF_PATTERN), - "MATRIX_EXTRAS", - ], + + "|)": [format("%(workflow)s(\s*,\s*%(workflow)s|)*", workflow=WF_PATTERN), "MATRIX_EXTRAS"], "(workflow|relval)(s|)_profiling": [ format("%(workflow)s(\s*,\s*%(workflow)s|)*", workflow=WF_PATTERN), "PROFILING_WORKFLOWS", @@ -166,16 +150,10 @@ def format(s, **kwds): "disable_poison": ["true|false", "DISABLE_POISON"], "use_ib_tag": ["true|false", "USE_IB_TAG"], "baseline": ["self|default", "USE_BASELINE"], - "skip_test(s|)": [ - format("(%(tests)s)(\s*,\s*(%(tests)s))*", tests=SKIP_TESTS), - "SKIP_TESTS", - ], + "skip_test(s|)": [format("(%(tests)s)(\s*,\s*(%(tests)s))*", tests=SKIP_TESTS), "SKIP_TESTS"], "dry_run": ["true|false", "DRY_RUN"], "jenkins_(slave|node)": [JENKINS_NODES, "RUN_ON_SLAVE"], - "(arch(itecture(s|))|release|release/arch)": [ - CMSSW_RELEASE_QUEUE_PATTERN, - "RELEASE_FORMAT", - ], + "(arch(itecture(s|))|release|release/arch)": [CMSSW_RELEASE_QUEUE_PATTERN, "RELEASE_FORMAT"], ENABLE_TEST_PTRN: [ format("(%(tests)s)(\s*,\s*(%(tests)s))*", tests=EXTRA_TESTS), "ENABLE_BOT_TESTS", @@ -264,13 +242,7 @@ def read_repo_file(repo_config, repo_file, default=None): def create_properties_file_tests( - repository, - pr_number, - parameters, - dryRun, - abort=False, - req_type="tests", - repo_config=None, + repository, pr_number, parameters, dryRun, abort=False, req_type="tests", repo_config=None ): if abort: req_type = "abort" @@ -334,17 +306,11 @@ def find_last_comment(issue, user, match): if (user != comment.user.login) or (not comment.body): continue if not re.match( - match, - comment.body.encode("ascii", "ignore").decode().strip("\n\t\r "), - re.MULTILINE, + match, comment.body.encode("ascii", "ignore").decode().strip("\n\t\r "), re.MULTILINE ): continue last_comment = comment - print( - "Matched comment from ", - comment.user.login + " with comment id ", - comment.id, - ) + print("Matched comment from ", comment.user.login + " with comment id ", comment.id) return last_comment @@ -717,14 +683,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pass if not cmsbuild_user: cmsbuild_user = repo_config.CMSBUILD_USER - print( - "Working on ", - repo.full_name, - " for PR/Issue ", - prId, - "with admin user", - cmsbuild_user, - ) + print("Working on ", repo.full_name, " for PR/Issue ", prId, "with admin user", cmsbuild_user) print("Notify User: ", gh_user_char) set_gh_user(cmsbuild_user) cmssw_repo = repo_name == GH_CMSSW_REPO @@ -735,6 +694,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F create_test_property = False repo_cache = {repository: repo} packages = set([]) + chg_files = [] package_categories = {} extra_labels = {"mtype": []} add_external_category = False @@ -776,7 +736,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # For future pre_checks # if prId>=somePRNumber: default_pre_checks+=["some","new","checks"] pre_checks_url = {} - events = {} commit_cache = {} all_commits = [] @@ -784,7 +743,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F warned_too_many_commits = False l2s = ", ".join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS]) - if issue.pull_request: pr = repo.get_pull(prId) if pr.changed_files == 0: @@ -807,7 +765,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # A pull request is by default closed if the branch is a closed one. if is_closed_branch(pr.base.ref): mustClose = True - # Process the changes for the given pull request so that we can determine the # signatures it requires. if cmssw_repo or not external_repo: @@ -815,9 +772,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pr.base.ref == "master": signing_categories.add("code-checks") updateMilestone(repo, issue, pr, dryRun) - chg_files = get_changed_files(repo, pr) - packages = sorted(list(set(cmssw_file2Package(repo_config, f) for f in chg_files))) + packages = sorted( + [x for x in set([cmssw_file2Package(repo_config, f) for f in chg_files])] + ) for pkg_file in chg_files: for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): for regex in pkgs_regexp: @@ -834,7 +792,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F create_test_property = True else: add_external_category = True - packages = {"externals/" + repository} + packages = set(["externals/" + repository]) ex_pkg = external_to_package(repository) if ex_pkg: packages.add(ex_pkg) @@ -866,8 +824,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Following categories affected:") print("\n".join(signing_categories)) - signatures = dict([(x, "pending") for x in signing_categories]) - if cmssw_repo: # If there is a new package, add also a dummy "new" category. all_packages = [ @@ -911,7 +867,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Handle watchers watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) for watcher in [x for x in watchers]: - if watcher not in watchingGroups: + if not watcher in watchingGroups: continue watchers.remove(watcher) watchers.update(set(watchingGroups[watcher])) @@ -938,7 +894,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F s for s in commit_statuses if s.context == "%s/code-checks" % cms_status_prefix ] print("PR Statuses:", commit_statuses) - print("#PR Statuses:", len(commit_statuses)) + print(len(commit_statuses)) last_commit_date = last_commit.committer.date print( "Latest commit by ", @@ -946,14 +902,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F " at ", last_commit_date, ) - print( - "Latest commit message: ", - last_commit.message.encode("ascii", "ignore").decode(), - ) + print("Latest commit message: ", last_commit.message.encode("ascii", "ignore").decode()) print("Latest commit sha: ", last_commit.sha) print("PR update time", pr.updated_at) - print("Time UTC:", datetime.utcnow()) - if last_commit_date > datetime.now(timezone.utc): + print("Time UTC:", datetime.now(timezone.utc)()) + if last_commit_date > datetime.now(timezone.utc)(): print("==== Future commit found ====") add_labels = True try: @@ -988,6 +941,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F f.write("ISSUE_NUMBER=" + str(issue.number) + "\n") # Process the issue comments + signatures = dict([(x, "pending") for x in signing_categories]) extra_pre_checks = [] pre_checks = [] if issue.pull_request: @@ -1082,7 +1036,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if commenter_categories or (commenter in CMSSW_ISSUES_TRACKERS): if assign_type == "assign": for ex_cat in new_cats: - if ex_cat not in signing_categories: + if not ex_cat in signing_categories: assign_cats[ex_cat] = 0 signing_categories.add(ex_cat) signatures[ex_cat] = "pending" @@ -1161,11 +1115,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("==>Reopen request received from %s" % commenter) continue if valid_commenter: - ( - valid_multiline_comment, - test_params, - test_params_m, - ) = multiline_check_function(first_line, comment_lines, repository) + valid_multiline_comment, test_params, test_params_m = multiline_check_function( + first_line, comment_lines, repository + ) if test_params_m: test_params_msg = str(comment.id) + ":" + test_params_m test_params_comment = comment @@ -1187,33 +1139,33 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if m.group(2): code_check_apply_patch = True - selected_cats = [] - - if re.match("^([+]1|approve[d]?|sign|signed)$", first_line, re.I): - ctype = "+1" - selected_cats = commenter_categories - elif re.match("^([-]1|reject|rejected)$", first_line, re.I): - ctype = "-1" - selected_cats = commenter_categories - elif re.match("^[+-][a-z][a-z0-9-]+$", first_line, re.I): - category_name = first_line[1:].lower() - if category_name in commenter_categories: - ctype = first_line[0] + "1" - selected_cats = [category_name] - - if "orp" in selected_cats: - # Lines 1281, 1286 of original code - mustClose = False - - if selected_cats: - events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": ctype, - "selected_cats": selected_cats, - "comment": comment, - }, - } + selected_cats = [] + + if re.match("^([+]1|approve[d]?|sign|signed)$", first_line, re.I): + ctype = "+1" + selected_cats = commenter_categories + elif re.match("^([-]1|reject|rejected)$", first_line, re.I): + ctype = "-1" + selected_cats = commenter_categories + elif re.match("^[+-][a-z][a-z0-9-]+$", first_line, re.I): + category_name = first_line[1:].lower() + if category_name in commenter_categories: + ctype = first_line[0] + "1" + selected_cats = [category_name] + + if "orp" in selected_cats: + # Lines 1281, 1286 of original code + mustClose = False + + if selected_cats: + events[comment.created_at] = { + "type": "sign", + "value": { + "ctype": ctype, + "selected_cats": selected_cats, + "comment": comment, + }, + } # Ignore all other messages which are before last commit. if issue.pull_request and (comment.created_at < last_commit_date): @@ -1305,7 +1257,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F comp_warnings = False if "+1" in first_line: signatures["tests"] = "approved" - comp_warnings = any("Compilation Warnings: Yes" in l for l in comment_lines) + comp_warnings = ( + len([1 for l in comment_lines if "Compilation Warnings: Yes" in l]) > 0 + ) pre_checks_url["tests"] = comment.html_url events[comment.created_at] = { "type": "sign", @@ -1315,7 +1269,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F "comment": comment, }, } - elif "-1" in first_line: signatures["tests"] = "rejected" pre_checks_url["tests"] = comment.html_url @@ -1393,7 +1346,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F override_tests_failure = reason if not dryRun: set_comment_emoji(comment.id, repository) - # end of parsing comments section if ( @@ -1499,7 +1451,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F auto_close_push_test_issue = True try: auto_close_push_test_issue = repo_config.AUTO_CLOSE_PUSH_TESTS_ISSUE - except AttributeError: + except: pass if ( auto_close_push_test_issue @@ -1592,19 +1544,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print(desc) if not dryRun: last_commit_obj.create_status( - "success", - description=desc, - target_url=turl, - context=bot_status_name, + "success", description=desc, target_url=turl, context=bot_status_name ) set_comment_emoji(test_comment.id, repository) if bot_status: - print( - bot_status.target_url, - turl, - signatures["tests"], - bot_status.description, - ) + print(bot_status.target_url, turl, signatures["tests"], bot_status.description) if ( bot_status and bot_status.target_url == turl @@ -1757,10 +1701,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if already_seen: if dryRun: - print( - "Update PR seen message to include backport PR number", - backport_pr_num, - ) + print("Update PR seen message to include backport PR number", backport_pr_num) else: new_msg = "" for l in already_seen.body.encode("ascii", "ignore").decode().split("\n"): @@ -1890,6 +1831,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F uname = "" if issue.user.name: uname = issue.user.name.encode("ascii", "ignore").decode() + l2s = ", ".join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS]) issueMessage = format( "%(msgPrefix)s %(gh_user_char)s%(user)s" " %(name)s.\n\n" @@ -1948,12 +1890,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params["CONTEXT_PREFIX"] = cms_status_prefix if trigger_test: create_properties_file_tests( - repository, - prId, - global_test_params, - dryRun, - abort=False, - repo_config=repo_config, + repository, prId, global_test_params, dryRun, abort=False, repo_config=repo_config ) if not dryRun: set_comment_emoji(test_comment.id, repository) @@ -2160,7 +2097,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if commentMsg and dryRun: print("The following comment will be made:") try: - print(commentMsg.encode("ascii", "replace").decode()) + print(commentMsg.decode("ascii", "replace")) except: pass for pre_check in pre_checks + extra_pre_checks: @@ -2183,10 +2120,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) continue if (not dryRunOrig) and (pre_checks_state[pre_check] == ""): - params = { - "PULL_REQUEST": "%s" % (prId), - "CONTEXT_PREFIX": cms_status_prefix, - } + params = {"PULL_REQUEST": "%s" % (prId), "CONTEXT_PREFIX": cms_status_prefix} if pre_check == "code-checks": params["CMSSW_TOOL_CONF"] = code_checks_tools params["APPLY_PATCH"] = str(code_check_apply_patch).lower() @@ -2218,7 +2152,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F mustMerge = True else: print("This pull request will not be automatically merged.") - if mustMerge: + if mustMerge == True: print("This pull request must be merged.") if not dryRun and (pr.state == "open"): pr.merge() From 1138c0b4e753a1c9ff6515e32c491bf8f46e790a Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 4 Dec 2023 11:27:24 +0100 Subject: [PATCH 22/46] Fix extra braces --- process_pr.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/process_pr.py b/process_pr.py index ac0184ca561a..189886749874 100644 --- a/process_pr.py +++ b/process_pr.py @@ -26,7 +26,7 @@ from githublabels import TYPE_COMMANDS, TEST_IGNORE_REASON from repo_config import GH_REPO_ORGANIZATION import re, time -from datetime import datetime, timezone +from datetime import datetime from os.path import join, exists, dirname from os import environ from github_utils import edit_pr, api_rate_limits, get_pr_commits_reversed @@ -126,6 +126,8 @@ def format(s, **kwds): r"^\s*(?:(?:@|)cmsbuild\s*[,]*\s+|)(?:please\s*[,]*\s+|)ignore\s+tests-rejected\s+(?:with|)([a-z -]+)$", re.I, ) +REGEX_COMMITS_CACHE = re.compile(r"") +REGEX_IGNORE_COMMIT_COUNT = "\+commit-count" TEST_WAIT_GAP = 720 ALL_CHECK_FUNCTIONS = None EXTRA_RELVALS_TESTS = ["threading", "gpu", "high-stats", "nano"] @@ -905,8 +907,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Latest commit message: ", last_commit.message.encode("ascii", "ignore").decode()) print("Latest commit sha: ", last_commit.sha) print("PR update time", pr.updated_at) - print("Time UTC:", datetime.now(timezone.utc)()) - if last_commit_date > datetime.now(timezone.utc)(): + print("Time UTC:", datetime.utcnow()) + if last_commit_date > datetime.utcnow(): print("==== Future commit found ====") add_labels = True try: @@ -1386,7 +1388,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F } cache_entry = commit_cache[commit.sha] - events[datetime.fromtimestamp(cache_entry["time"], timezone.utc)] = { + events[datetime.fromtimestamp(cache_entry["time"])] = { "type": "commit", "value": cache_entry["files"], } @@ -1412,13 +1414,12 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("DRY RUN: Creating technical comment with text") print(new_body) - events = dict(sorted(events.items())) print("Events:", events) + events = dict(sorted(events.items())) print("Recalculating signatures") for event in events.values(): if event["type"] == "sign": - print("Sign:", event["value"]) selected_cats = event["value"]["selected_cats"] ctype = event["value"]["ctype"] if any(x in signing_categories for x in selected_cats): From 417d6094eead69078c8727ef910b1497f6423be8 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 4 Dec 2023 16:07:24 +0100 Subject: [PATCH 23/46] Reset all signatures on commit in external/user repos --- process_pr.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/process_pr.py b/process_pr.py index 189886749874..ca2a69b2bac2 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1393,6 +1393,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F "value": cache_entry["files"], } + print("Saving commit cache") old_body = cache_comment.body if cache_comment else CMSBOT_TECHNICAL_MSG new_body = ( REGEX_COMMITS_CACHE.sub("", old_body) @@ -1414,8 +1415,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("DRY RUN: Creating technical comment with text") print(new_body) - print("Events:", events) events = dict(sorted(events.items())) + import pprint + + print("Events:", pprint.pformat(events)) print("Recalculating signatures") for event in events.values(): @@ -1440,12 +1443,19 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F else: print(f"Ignoring event: {signing_categories}, {selected_cats}") elif event["type"] == "commit": - chg_categories = {"orp", "tests", "code-checks"} - for fn in event["value"]: - chg_categories.update(get_package_categories(cmssw_file2Package(repo_config, fn))) + if cmssw_repo: + chg_categories = {"orp", "tests", "code-checks"} + for fn in event["value"]: + chg_categories.update( + get_package_categories(cmssw_file2Package(repo_config, fn)) + ) - for cat in chg_categories: - if cat in signing_categories: + for cat in chg_categories: + if cat in signing_categories: + signatures[cat] = "pending" + else: + print("Resetting all signatures for external/user repos") + for cat in signing_categories: signatures[cat] = "pending" if push_test_issue: @@ -1553,7 +1563,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if ( bot_status and bot_status.target_url == turl - and signatures["tests"] == "pending" + # and signatures["tests"] == "pending" and (" requested by " in bot_status.description) ): signatures["tests"] = "started" From a2239254ad8133e14d45b24cc8939d13daa9ac25 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Wed, 6 Dec 2023 11:11:27 +0100 Subject: [PATCH 24/46] Do not trigger tests on code-checks signature --- process_pr.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/process_pr.py b/process_pr.py index ca2a69b2bac2..0dfae2a4dfa6 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1429,8 +1429,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if ctype == "+1": for sign in selected_cats: signatures[sign] = "approved" - if (test_comment is None) and ( - (repository in auto_test_repo) or ("*" in auto_test_repo) + if ( + (test_comment is None) + and ((repository in auto_test_repo) or ("*" in auto_test_repo)) + and sign != "code-checks" ): test_comment = event["value"]["comment"] # if sign == "orp": From 2657914b55ae9cfc5e8672591a6f21ae96d53eb3 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Wed, 6 Dec 2023 11:29:59 +0100 Subject: [PATCH 25/46] Exclude orp and tests as well --- process_pr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process_pr.py b/process_pr.py index 0dfae2a4dfa6..d19ee83b2e45 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1384,7 +1384,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if commit.sha not in commit_cache: commit_cache[commit.sha] = { "time": int(commit.commit.committer.date.timestamp()), - "files": get_changed_files_in_commit(repo, commit), + "files": sorted(get_changed_files_in_commit(repo, commit)), } cache_entry = commit_cache[commit.sha] @@ -1432,7 +1432,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if ( (test_comment is None) and ((repository in auto_test_repo) or ("*" in auto_test_repo)) - and sign != "code-checks" + and sign not in ("code-checks", "tests", "orp") ): test_comment = event["value"]["comment"] # if sign == "orp": From c97e30906caad238d00ed0e848a8c96a77849a31 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Wed, 6 Dec 2023 11:31:55 +0100 Subject: [PATCH 26/46] Check for rate-limits before checking for / loading commit cache --- process_pr.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/process_pr.py b/process_pr.py index d19ee83b2e45..0a2a2b429137 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1363,6 +1363,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) return + api_rate_limits(gh) + # Get the commit cache from `already_seen` commit or technical commit print("Checking for commit cache") cache_comment = None From 3aee7599f9fae1e0d7c1946ee5d41ff2c4fe20ba Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Wed, 6 Dec 2023 11:37:32 +0100 Subject: [PATCH 27/46] Apply suggestions from code-review --- process_pr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/process_pr.py b/process_pr.py index 0a2a2b429137..f17ff8ebf8cc 100644 --- a/process_pr.py +++ b/process_pr.py @@ -744,7 +744,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ok_too_many_commits = False warned_too_many_commits = False - l2s = ", ".join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS]) if issue.pull_request: pr = repo.get_pull(prId) if pr.changed_files == 0: @@ -1024,7 +1023,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: warned_too_many_commits = True - if commenter in l2s and re.match( + if commenter in CMSSW_ISSUES_TRACKERS and re.match( r"^\s*" + REGEX_IGNORE_COMMIT_COUNT + r"\s*$", first_line ): ok_too_many_commits = True @@ -1359,7 +1358,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F issue.create_comment( f"This PR contains too many commits ({pr.commits} > {MAX_INITIAL_COMMITS_IN_PR}). " "Make sure you chose the right target branch.\n" - f"{l2s}, you can override this check with `+commit-count`." + f"{', '.join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS])}, " + "you can override this check with `+commit-count`." ) return From 0cb4fe9b8772549bece746e450125eb88262cccb Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Wed, 6 Dec 2023 14:29:01 +0100 Subject: [PATCH 28/46] More changes from review --- process_pr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/process_pr.py b/process_pr.py index f17ff8ebf8cc..fa5101a26b3d 100644 --- a/process_pr.py +++ b/process_pr.py @@ -661,7 +661,7 @@ def get_status_state(context, statuses): def dumps_compact(value): - return dumps(value, separators=(",", ":")) + return dumps(dict(sorted(value.items())), separators=(",", ":")) def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=False): @@ -1027,6 +1027,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F r"^\s*" + REGEX_IGNORE_COMMIT_COUNT + r"\s*$", first_line ): ok_too_many_commits = True + continue assign_type, new_cats = get_assign_categories(first_line) if new_cats: From a39283a47199d8d1d9b6b3eff3b77a886df431af Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Wed, 6 Dec 2023 16:11:43 +0100 Subject: [PATCH 29/46] Compress commit cache as needed; handle commits with many changed files --- process_pr.py | 61 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/process_pr.py b/process_pr.py index fa5101a26b3d..157a7c1888b8 100644 --- a/process_pr.py +++ b/process_pr.py @@ -26,10 +26,16 @@ from githublabels import TYPE_COMMANDS, TEST_IGNORE_REASON from repo_config import GH_REPO_ORGANIZATION import re, time +import zlib, base64 from datetime import datetime from os.path import join, exists, dirname from os import environ -from github_utils import edit_pr, api_rate_limits, get_pr_commits_reversed +from github_utils import ( + edit_pr, + api_rate_limits, + get_pr_commits_reversed, + get_commit, +) from github_utils import set_comment_emoji, get_comment_emojis, delete_comment_emoji, set_gh_user from socket import setdefaulttimeout from _py2with3compatibility import run_cmd @@ -207,11 +213,6 @@ def get_commenter_categories(commenter, comment_date): return [] -def get_changed_files_in_commit(repo, commit_obj): - commit = repo.get_commit(commit_obj.commit.sha) - return [x.filename for x in commit.files] - - def get_package_categories(package): cats = [] for cat, packages in list(CMSSW_CATEGORIES.items()): @@ -660,8 +661,19 @@ def get_status_state(context, statuses): return "" -def dumps_compact(value): - return dumps(dict(sorted(value.items())), separators=(",", ":")) +def dumps_maybe_compress(value): + json_ = dumps(value, separators=(",", ":"), sort_keys=True) + if len(json_) > 32000: + return "b64:" + base64.encodebytes(zlib.compress(json_.encode())).decode("ascii", "ignore") + else: + return json_ + + +def loads_maybe_decompress(data): + if data.startswith("b64:"): + data = zlib.decompress(base64.decodebytes(data[4:].encode())).decode() + + return loads(data) def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=False): @@ -1380,14 +1392,16 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F seen_commits_match = REGEX_COMMITS_CACHE.search(cache_comment.body) if seen_commits_match: print("Loading commit cache") - commit_cache = loads(seen_commits_match[1]) + commit_cache = loads_maybe_decompress(seen_commits_match[1]) if pr.commits < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: for commit in all_commits: if commit.sha not in commit_cache: commit_cache[commit.sha] = { "time": int(commit.commit.committer.date.timestamp()), - "files": sorted(get_changed_files_in_commit(repo, commit)), + "files": sorted( + x["filename"] for x in get_commit(repo.full_name, commit.sha)["files"] + ), } cache_entry = commit_cache[commit.sha] @@ -1401,22 +1415,25 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F new_body = ( REGEX_COMMITS_CACHE.sub("", old_body) + "" ) - if not dryRun: - if cache_comment: - if old_body != new_body: - cache_comment.edit(new_body) + if len(new_body) <= 65535: + if not dryRun: + if cache_comment: + if old_body != new_body: + cache_comment.edit(new_body) + else: + issue.create_comment(new_body) else: - issue.create_comment(new_body) + if cache_comment: + print("DRY RUN: Updating existing comment with text") + print(new_body) + else: + print("DRY RUN: Creating technical comment with text") + print(new_body) else: - if cache_comment: - print("DRY RUN: Updating existing comment with text") - print(new_body) - else: - print("DRY RUN: Creating technical comment with text") - print(new_body) + raise RuntimeError(f"Updated comment body too long: {len(new_body)} > 65535") events = dict(sorted(events.items())) import pprint From 98e950096c79b20e28b32ed03b633e910491d2b0 Mon Sep 17 00:00:00 2001 From: iarspider Date: Wed, 6 Dec 2023 17:21:47 +0100 Subject: [PATCH 30/46] Reset signatures for manually-assigned categories on commit; other changes from review --- process_pr.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/process_pr.py b/process_pr.py index 157a7c1888b8..6063d5ae2559 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1031,9 +1031,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F and not technical_comment ): technical_comment = comment + continue if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: warned_too_many_commits = True + continue if commenter in CMSSW_ISSUES_TRACKERS and re.match( r"^\s*" + REGEX_IGNORE_COMMIT_COUNT + r"\s*$", first_line @@ -1435,13 +1437,13 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F else: raise RuntimeError(f"Updated comment body too long: {len(new_body)} > 65535") - events = dict(sorted(events.items())) + events = sorted(events.items()) import pprint print("Events:", pprint.pformat(events)) print("Recalculating signatures") - for event in events.values(): + for _, event in events: if event["type"] == "sign": selected_cats = event["value"]["selected_cats"] ctype = event["value"]["ctype"] @@ -1472,6 +1474,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F get_package_categories(cmssw_file2Package(repo_config, fn)) ) + chg_categories.update(assign_cats.keys()) + for cat in chg_categories: if cat in signing_categories: signatures[cat] = "pending" From 5fcd33c6aa16ce14273e3eeabce2d401a6c5aea1 Mon Sep 17 00:00:00 2001 From: iarspider Date: Wed, 6 Dec 2023 17:47:04 +0100 Subject: [PATCH 31/46] Missed one comment --- process_pr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/process_pr.py b/process_pr.py index 6063d5ae2559..017066baf9c3 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1399,6 +1399,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pr.commits < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: for commit in all_commits: if commit.sha not in commit_cache: + api_rate_limits(gh) commit_cache[commit.sha] = { "time": int(commit.commit.committer.date.timestamp()), "files": sorted( From 5f4f8446c00d8b101cf874da4475c1a488821342 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 7 Dec 2023 10:19:52 +0100 Subject: [PATCH 32/46] Fix for github_utils --- github_utils.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/github_utils.py b/github_utils.py index 416e61039c38..3e977e61a7f8 100644 --- a/github_utils.py +++ b/github_utils.py @@ -522,6 +522,29 @@ def get_rate_limits(): return github_api(uri="/rate_limit", method="GET") +def merge_dicts(old, new): + for k, v in new.items(): + if k not in old: + old[k] = new[k] + continue + + if isinstance(v, dict): + old[k] = merge_dicts(old[k], new[k]) + continue + + if isinstance(v, list): + old[k].extend(v) + continue + + if old[k] != new[k]: + raise RuntimeError( + f"Unable to merge dictionaries: value for key {k} differs. " + "Old {old[k]} {type(old[k])}, new {new[k]}, {type(new[k])}" + ) + + return old + + def github_api( uri, params=None, @@ -604,11 +627,12 @@ def github_api( all_pages=False, ) for page in GH_PAGE_RANGE: - if max_pages > 0 and page > max_pages: + if max_pages > 0 and page > max_pages: # noqa for readability break - data += github_api( + new_data = github_api( uri, params, method, headers, page, raw=raw, per_page=per_page, all_pages=False ) + data = merge_dicts(data, new_data) return data @@ -647,6 +671,10 @@ def pr_get_changed_files(pr): return rez +def get_commit(repository, commit_sha): + return github_api(f"/repos/{repository}/commits/{commit_sha}", method="GET") + + def get_unix_time(data_obj): return data_obj.strftime("%s") From fc24b327168cc0bfbb973463848486479b71b87b Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 7 Dec 2023 10:22:50 +0100 Subject: [PATCH 33/46] Fix for fix for github_utils --- github_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/github_utils.py b/github_utils.py index 3e977e61a7f8..5da97ff4f3a0 100644 --- a/github_utils.py +++ b/github_utils.py @@ -632,7 +632,10 @@ def github_api( new_data = github_api( uri, params, method, headers, page, raw=raw, per_page=per_page, all_pages=False ) - data = merge_dicts(data, new_data) + if isinstance(data, dict): + data = merge_dicts(data, new_data) + else: + data += new_data return data From 7539456ede28f936527e97a264f83bd2ed665de0 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 7 Dec 2023 10:51:24 +0100 Subject: [PATCH 34/46] Check rates at every call to github_api() --- github_utils.py | 9 +++++++-- process_pr.py | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/github_utils.py b/github_utils.py index 5da97ff4f3a0..31df1f058b6a 100644 --- a/github_utils.py +++ b/github_utils.py @@ -556,10 +556,15 @@ def github_api( last_page=False, all_pages=True, max_pages=-1, - status=[], + status=None, ): + if status is None: + status = [] + + check_rate_limits(msg=False) + global GH_RATE_LIMIT, GH_PAGE_RANGE - if max_pages > 0 and page > max_pages: + if max_pages > 0 and page > max_pages: # noqa for readability return "[]" if raw else [] if not params: params = {} diff --git a/process_pr.py b/process_pr.py index 017066baf9c3..6063d5ae2559 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1399,7 +1399,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pr.commits < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: for commit in all_commits: if commit.sha not in commit_cache: - api_rate_limits(gh) commit_cache[commit.sha] = { "time": int(commit.commit.committer.date.timestamp()), "files": sorted( From 77095becc9d0f150aa67d3dba08af392c70677c0 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 7 Dec 2023 10:53:08 +0100 Subject: [PATCH 35/46] Changes from review --- process_pr.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/process_pr.py b/process_pr.py index 6063d5ae2559..e06ef6140226 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1378,8 +1378,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) return - api_rate_limits(gh) - # Get the commit cache from `already_seen` commit or technical commit print("Checking for commit cache") cache_comment = None From 4b453f046c0a47dc08a15ef82f8b7de19c5baed0 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 7 Dec 2023 11:16:47 +0100 Subject: [PATCH 36/46] Only check commits if the issue is a PR --- process_pr.py | 129 +++++++++++++++++++++++++------------------------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/process_pr.py b/process_pr.py index e06ef6140226..fbaf6e3223b1 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1364,76 +1364,77 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F set_comment_emoji(comment.id, repository) # end of parsing comments section - if ( - (not warned_too_many_commits) - and not ok_too_many_commits - and pr.commits >= MAX_INITIAL_COMMITS_IN_PR - ): - if not dryRun: - issue.create_comment( - f"This PR contains too many commits ({pr.commits} > {MAX_INITIAL_COMMITS_IN_PR}). " - "Make sure you chose the right target branch.\n" - f"{', '.join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS])}, " - "you can override this check with `+commit-count`." - ) - return + if issue.pull_request: + if ( + (not warned_too_many_commits) + and not ok_too_many_commits + and pr.commits >= MAX_INITIAL_COMMITS_IN_PR + ): + if not dryRun: + issue.create_comment( + f"This PR contains too many commits ({pr.commits} > {MAX_INITIAL_COMMITS_IN_PR}). " + "Make sure you chose the right target branch.\n" + f"{', '.join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS])}, " + "you can override this check with `+commit-count`." + ) + return - # Get the commit cache from `already_seen` commit or technical commit - print("Checking for commit cache") - cache_comment = None + # Get the commit cache from `already_seen` commit or technical commit + print("Checking for commit cache") + cache_comment = None - if technical_comment: - cache_comment = technical_comment - else: - if already_seen: - cache_comment = already_seen - - if cache_comment: - seen_commits_match = REGEX_COMMITS_CACHE.search(cache_comment.body) - if seen_commits_match: - print("Loading commit cache") - commit_cache = loads_maybe_decompress(seen_commits_match[1]) - - if pr.commits < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: - for commit in all_commits: - if commit.sha not in commit_cache: - commit_cache[commit.sha] = { - "time": int(commit.commit.committer.date.timestamp()), - "files": sorted( - x["filename"] for x in get_commit(repo.full_name, commit.sha)["files"] - ), - } + if technical_comment: + cache_comment = technical_comment + else: + if already_seen: + cache_comment = already_seen + + if cache_comment: + seen_commits_match = REGEX_COMMITS_CACHE.search(cache_comment.body) + if seen_commits_match: + print("Loading commit cache") + commit_cache = loads_maybe_decompress(seen_commits_match[1]) + + if pr.commits < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: + for commit in all_commits: + if commit.sha not in commit_cache: + commit_cache[commit.sha] = { + "time": int(commit.commit.committer.date.timestamp()), + "files": sorted( + x["filename"] for x in get_commit(repo.full_name, commit.sha)["files"] + ), + } - cache_entry = commit_cache[commit.sha] - events[datetime.fromtimestamp(cache_entry["time"])] = { - "type": "commit", - "value": cache_entry["files"], - } + cache_entry = commit_cache[commit.sha] + events[datetime.fromtimestamp(cache_entry["time"])] = { + "type": "commit", + "value": cache_entry["files"], + } - print("Saving commit cache") - old_body = cache_comment.body if cache_comment else CMSBOT_TECHNICAL_MSG - new_body = ( - REGEX_COMMITS_CACHE.sub("", old_body) - + "" - ) - if len(new_body) <= 65535: - if not dryRun: - if cache_comment: - if old_body != new_body: - cache_comment.edit(new_body) + print("Saving commit cache") + old_body = cache_comment.body if cache_comment else CMSBOT_TECHNICAL_MSG + new_body = ( + REGEX_COMMITS_CACHE.sub("", old_body) + + "" + ) + if len(new_body) <= 65535: + if not dryRun: + if cache_comment: + if old_body != new_body: + cache_comment.edit(new_body) + else: + issue.create_comment(new_body) else: - issue.create_comment(new_body) + if cache_comment: + print("DRY RUN: Updating existing comment with text") + print(new_body) + else: + print("DRY RUN: Creating technical comment with text") + print(new_body) else: - if cache_comment: - print("DRY RUN: Updating existing comment with text") - print(new_body) - else: - print("DRY RUN: Creating technical comment with text") - print(new_body) - else: - raise RuntimeError(f"Updated comment body too long: {len(new_body)} > 65535") + raise RuntimeError(f"Updated comment body too long: {len(new_body)} > 65535") events = sorted(events.items()) import pprint From 6a2218570fc7bf3e35e3dbd1b8456c7d00ff064f Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Thu, 7 Dec 2023 15:04:50 +0100 Subject: [PATCH 37/46] Hack to set tests signature to pending --- process_pr.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/process_pr.py b/process_pr.py index fbaf6e3223b1..434b571db146 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1345,11 +1345,28 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) print("Comment message:", first_line) signatures["tests"] = "pending" + events[comment.created_at] = { + "type": "sign", + "value": { + "ctype": "pending", + "selected_cats": ["tests"], + "comment": comment, + }, + } continue elif REGEX_TEST_ABORT.match(first_line) and (signatures["tests"] == "pending"): abort_test = comment test_comment = None signatures["tests"] = "pending" + events[comment.created_at] = { + "type": "sign", + "value": { + "ctype": "pending", + "selected_cats": ["tests"], + "comment": comment, + }, + } + continue elif REGEX_TEST_IGNORE.match(first_line): reason = REGEX_TEST_IGNORE.match(first_line)[1].strip() if reason not in TEST_IGNORE_REASON: @@ -1357,7 +1374,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if not dryRun: set_comment_emoji(comment.id, repository, "-1") reason = "" - if reason: override_tests_failure = reason if not dryRun: @@ -1463,8 +1479,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[sign] = "rejected" if sign == "orp": mustClose = False + elif ctype == "pending": + if selected_cats != ["tests"]: + print(f"Invalid signature: {event['value']}") + exit(1) + else: + signatures["tests"] = "pending" else: - print(f"Ignoring event: {signing_categories}, {selected_cats}") + print(f"Ignoring event: {selected_cats} includes none of {signing_categories}") elif event["type"] == "commit": if cmssw_repo: chg_categories = {"orp", "tests", "code-checks"} @@ -1588,7 +1610,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if ( bot_status and bot_status.target_url == turl - # and signatures["tests"] == "pending" + and signatures["tests"] == "pending" and (" requested by " in bot_status.description) ): signatures["tests"] = "started" From c8530bb860122ab9e51b92e31a82d66d31b40793 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 11 Dec 2023 10:59:13 +0100 Subject: [PATCH 38/46] Don't recalculate tests signature --- process_pr.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/process_pr.py b/process_pr.py index 434b571db146..fb968b76e6d4 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1345,27 +1345,11 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) print("Comment message:", first_line) signatures["tests"] = "pending" - events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "pending", - "selected_cats": ["tests"], - "comment": comment, - }, - } continue elif REGEX_TEST_ABORT.match(first_line) and (signatures["tests"] == "pending"): abort_test = comment test_comment = None signatures["tests"] = "pending" - events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "pending", - "selected_cats": ["tests"], - "comment": comment, - }, - } continue elif REGEX_TEST_IGNORE.match(first_line): reason = REGEX_TEST_IGNORE.match(first_line)[1].strip() @@ -1458,6 +1442,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Events:", pprint.pformat(events)) print("Recalculating signatures") + # Save tests signature + test_signature = signatures.get("tests", None) + + # Process commits and signatures for _, event in events: if event["type"] == "sign": selected_cats = event["value"]["selected_cats"] @@ -1489,7 +1477,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print(f"Ignoring event: {selected_cats} includes none of {signing_categories}") elif event["type"] == "commit": if cmssw_repo: - chg_categories = {"orp", "tests", "code-checks"} + chg_categories = {"orp", "code-checks"} for fn in event["value"]: chg_categories.update( get_package_categories(cmssw_file2Package(repo_config, fn)) @@ -1505,6 +1493,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F for cat in signing_categories: signatures[cat] = "pending" + # Restore "tests" signature + if test_signature is not None: + signatures["tests"] = test_signature + if push_test_issue: auto_close_push_test_issue = True try: From 524a16fefb2014f3b0f1d07d6a5502d222998c4e Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 11 Dec 2023 11:37:23 +0100 Subject: [PATCH 39/46] Remove dead branch --- process_pr.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/process_pr.py b/process_pr.py index fb968b76e6d4..9ec4d1903779 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1467,12 +1467,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[sign] = "rejected" if sign == "orp": mustClose = False - elif ctype == "pending": - if selected_cats != ["tests"]: - print(f"Invalid signature: {event['value']}") - exit(1) - else: - signatures["tests"] = "pending" else: print(f"Ignoring event: {selected_cats} includes none of {signing_categories}") elif event["type"] == "commit": From 2e1d4ce9fad65346a80f670ef48bca1006bc640a Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 11 Dec 2023 16:04:30 +0100 Subject: [PATCH 40/46] Changes from review --- process_pr.py | 86 +++++++++++++-------------------------------------- 1 file changed, 22 insertions(+), 64 deletions(-) diff --git a/process_pr.py b/process_pr.py index 9ec4d1903779..bbd7eec0ef10 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1,3 +1,5 @@ +import copy + from categories import ( CMSSW_L2, CMSSW_L1, @@ -41,6 +43,7 @@ from _py2with3compatibility import run_cmd from json import dumps, load, loads + try: from categories import CMSSW_LABELS except: @@ -751,6 +754,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # if prId>=somePRNumber: default_pre_checks+=["some","new","checks"] pre_checks_url = {} events = {} + commit_cache = {} all_commits = [] ok_too_many_commits = False @@ -1159,10 +1163,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if re.match("^([+]1|approve[d]?|sign|signed)$", first_line, re.I): ctype = "+1" - selected_cats = commenter_categories + selected_cats = copy.copy(commenter_categories) elif re.match("^([-]1|reject|rejected)$", first_line, re.I): ctype = "-1" - selected_cats = commenter_categories + selected_cats = copy.copy(commenter_categories) elif re.match("^[+-][a-z][a-z0-9-]+$", first_line, re.I): category_name = first_line[1:].lower() if category_name in commenter_categories: @@ -1173,6 +1177,12 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Lines 1281, 1286 of original code mustClose = False + if "tests" in selected_cats: + selected_cats.remove("tests") + + if "code-checks" in selected_cats: + selected_cats.remove("code-checks") + if selected_cats: events[comment.created_at] = { "type": "sign", @@ -1217,25 +1227,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F elif "-code-checks" == first_line: signatures["code-checks"] = "rejected" pre_checks_url["code-checks"] = comment.html_url - events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "-1", - "selected_cats": ["code-checks"], - "comment": comment, - }, - } elif "+code-checks" == first_line: signatures["code-checks"] = "approved" pre_checks_url["code-checks"] = comment.html_url - events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "+1", - "selected_cats": ["code-checks"], - "comment": comment, - }, - } elif re.match("^Comparison not run.+", first_line): if ("tests" in signatures) and signatures["tests"] != "pending": comparison_notrun = True @@ -1277,25 +1271,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F len([1 for l in comment_lines if "Compilation Warnings: Yes" in l]) > 0 ) pre_checks_url["tests"] = comment.html_url - events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "+1", - "selected_cats": ["tests"], - "comment": comment, - }, - } elif "-1" in first_line: signatures["tests"] = "rejected" pre_checks_url["tests"] = comment.html_url - events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "-1", - "selected_cats": ["tests"], - "comment": comment, - }, - } else: signatures["tests"] = "pending" print( @@ -1312,14 +1290,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F mustClose = False if ("orp" in commenter_categories) and ("orp" in signatures): signatures["orp"] = "approved" - events[comment.created_at] = { - "type": "sign", - "value": { - "ctype": "+1", - "selected_cats": ["orp"], - "comment": comment, - }, - } continue # Check if the someone asked to trigger the tests @@ -1440,10 +1410,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F import pprint print("Events:", pprint.pformat(events)) - print("Recalculating signatures") - - # Save tests signature - test_signature = signatures.get("tests", None) + print("Recalculating signatures", signatures) # Process commits and signatures for _, event in events: @@ -1460,37 +1427,28 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F and sign not in ("code-checks", "tests", "orp") ): test_comment = event["value"]["comment"] - # if sign == "orp": - # mustClose = False elif ctype == "-1": for sign in selected_cats: signatures[sign] = "rejected" - if sign == "orp": - mustClose = False else: print(f"Ignoring event: {selected_cats} includes none of {signing_categories}") elif event["type"] == "commit": if cmssw_repo: - chg_categories = {"orp", "code-checks"} - for fn in event["value"]: - chg_categories.update( - get_package_categories(cmssw_file2Package(repo_config, fn)) - ) - - chg_categories.update(assign_cats.keys()) - - for cat in chg_categories: + # chg_categories = {"orp", "code-checks"} + # for fn in event["value"]: + # chg_categories.update( + # get_package_categories(cmssw_file2Package(repo_config, fn)) + # ) + # + # chg_categories.update(assign_cats.keys()) + + for cat in assign_cats.keys(): if cat in signing_categories: signatures[cat] = "pending" else: - print("Resetting all signatures for external/user repos") for cat in signing_categories: signatures[cat] = "pending" - # Restore "tests" signature - if test_signature is not None: - signatures["tests"] = test_signature - if push_test_issue: auto_close_push_test_issue = True try: From 4f437412ace6da67a70eb8ce51bfc41bfd2b5750 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Tue, 12 Dec 2023 08:39:42 +0100 Subject: [PATCH 41/46] Fix --- process_pr.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/process_pr.py b/process_pr.py index bbd7eec0ef10..177ec333a2af 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1434,15 +1434,13 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print(f"Ignoring event: {selected_cats} includes none of {signing_categories}") elif event["type"] == "commit": if cmssw_repo: - # chg_categories = {"orp", "code-checks"} - # for fn in event["value"]: - # chg_categories.update( - # get_package_categories(cmssw_file2Package(repo_config, fn)) - # ) - # - # chg_categories.update(assign_cats.keys()) - - for cat in assign_cats.keys(): + for fn in event["value"]: + chg_categories.update( + get_package_categories(cmssw_file2Package(repo_config, fn)) + ) + chg_categories.update(assign_cats.keys()) + + for cat in chg_categories: if cat in signing_categories: signatures[cat] = "pending" else: From 2654bf7ce736ab20605da74bc20aa0eb28f2c5b7 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Tue, 12 Dec 2023 11:19:24 +0100 Subject: [PATCH 42/46] Fix for fix --- process_pr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/process_pr.py b/process_pr.py index 177ec333a2af..2dc5012ee59a 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1434,6 +1434,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print(f"Ignoring event: {selected_cats} includes none of {signing_categories}") elif event["type"] == "commit": if cmssw_repo: + chg_categories = set() for fn in event["value"]: chg_categories.update( get_package_categories(cmssw_file2Package(repo_config, fn)) From 712cb46568df65d5bbddb50fde8e19bbb9e0b32b Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Tue, 12 Dec 2023 11:38:49 +0100 Subject: [PATCH 43/46] Changes from review --- process_pr.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/process_pr.py b/process_pr.py index 2dc5012ee59a..b99597c525d6 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1,5 +1,3 @@ -import copy - from categories import ( CMSSW_L2, CMSSW_L1, @@ -1163,10 +1161,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if re.match("^([+]1|approve[d]?|sign|signed)$", first_line, re.I): ctype = "+1" - selected_cats = copy.copy(commenter_categories) + selected_cats = commenter_categories[:] elif re.match("^([-]1|reject|rejected)$", first_line, re.I): ctype = "-1" - selected_cats = copy.copy(commenter_categories) + selected_cats = commenter_categories[:] elif re.match("^[+-][a-z][a-z0-9-]+$", first_line, re.I): category_name = first_line[1:].lower() if category_name in commenter_categories: From 377fa08f8bffae1a6edf7a4863c23b24ff29273f Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Tue, 12 Dec 2023 11:49:19 +0100 Subject: [PATCH 44/46] Add protection from printing non-ascii characters --- process_pr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/process_pr.py b/process_pr.py index b99597c525d6..fcd7bae2d96d 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1397,17 +1397,17 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F else: if cache_comment: print("DRY RUN: Updating existing comment with text") - print(new_body) + print(new_body.encode("ascii", "ignore").decode()) else: print("DRY RUN: Creating technical comment with text") - print(new_body) + print(new_body.encode("ascii", "ignore").decode()) else: raise RuntimeError(f"Updated comment body too long: {len(new_body)} > 65535") events = sorted(events.items()) import pprint - print("Events:", pprint.pformat(events)) + print("Events:", pprint.pformat(events).encode("ascii", "ignore").decode()) print("Recalculating signatures", signatures) # Process commits and signatures From d29f967e6d9a0476d16eca794ce3d2ad2394efbb Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Tue, 12 Dec 2023 23:10:12 +0100 Subject: [PATCH 45/46] reset test_comment on every new commit --- process_pr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/process_pr.py b/process_pr.py index fcd7bae2d96d..a5a31d1e6bbb 100644 --- a/process_pr.py +++ b/process_pr.py @@ -1431,6 +1431,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F else: print(f"Ignoring event: {selected_cats} includes none of {signing_categories}") elif event["type"] == "commit": + test_comment = None if cmssw_repo: chg_categories = set() for fn in event["value"]: From 95d76a02c1853416f63ccfea39ca5b5450c58a25 Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Tue, 12 Dec 2023 23:30:30 +0100 Subject: [PATCH 46/46] Update process-pull-request.py --- process-pull-request.py | 1 + 1 file changed, 1 insertion(+) diff --git a/process-pull-request.py b/process-pull-request.py index 74c18e757d6d..e4e92a7aa007 100755 --- a/process-pull-request.py +++ b/process-pull-request.py @@ -81,3 +81,4 @@ from process_pr import process_pr process_pr(repo_config, gh, repo, repo.get_issue(prId), opts.dryRun, force=opts.force) + api_rate_limits(gh)