diff --git a/.ci/generate_bear_requirements.py b/.ci/generate_bear_requirements.py index 808e6ce044..e20f48422e 100755 --- a/.ci/generate_bear_requirements.py +++ b/.ci/generate_bear_requirements.py @@ -14,13 +14,14 @@ # along with this program. If not, see . import argparse +import copy import itertools import json import os import sys -from collections import OrderedDict -from yaml import dump +from ruamel.yaml import YAML, RoundTripDumper +from ruamel.yaml.comments import CommentedMap from pyprint.NullPrinter import NullPrinter from coalib.bears.BEAR_KIND import BEAR_KIND @@ -30,6 +31,10 @@ from dependency_management.requirements.NpmRequirement import NpmRequirement from dependency_management.requirements.PipRequirement import PipRequirement +yaml = YAML(typ='rt') +yaml.default_flow_style = False +yaml.Dumper = RoundTripDumper + BEAR_REQUIREMENTS_YAML = "bear-requirements.yaml" _VERSION_OPERATORS = ('<', '>', '~', '=', '-', '!') @@ -51,6 +56,11 @@ def get_args(): parser.add_argument('--bear-dirs', '-d', nargs='+', metavar='DIR', help='additional directories which may contain bears') + parser.add_argument('--check', '-c', action='store_true', + help='performs a dry run, and reports differences.') + parser.add_argument('--update', '-u', action='store_true', + help='updates "bear-requirements.yaml" ' + 'instead of overwriting') args = parser.parse_args() return args @@ -144,6 +154,47 @@ def get_pip_requirements(requirements): return _get_requirements(requirements, '~=', inherited_requirements) +def deep_update(target, src): + for key, value in src.items(): + if key not in target: + target[key] = copy.deepcopy(value) + else: + if isinstance(value, list): + target[key].extend(value) + elif isinstance(value, dict): + deep_update(target[key], value) + elif isinstance(value, set): + target[key].update(value.copy()) + else: + target[key] = copy.copy(value) + + +def deep_diff(target, src): + errors = [] + for key, value in src.items(): + if key not in target: + errors.append((key, 'Missing')) + elif target[key] != value: + if isinstance(value, list): + if [x for x in value if x not in target[key]]: + errors.append(key) + elif isinstance(value, dict): + if target[key] != value: + errors.append((key, deep_diff(target[key], value))) + elif isinstance(value, set): + if set(target[key]).symmetric_difference(value): + errors.append(key) + else: + errors.append((key, target[key])) + return errors + + +def sort_requirements(req_dict): + for key in ['pip_requirements', 'npm_requirements', 'gem_requirements']: + req_dict[key] = CommentedMap(sorted(req_dict[key].items(), + key=lambda t: t[0])) + + if __name__ == '__main__': args = get_args() @@ -155,18 +206,45 @@ def get_pip_requirements(requirements): pip_reqs, npm_reqs, gem_reqs = ( get_all_requirements(get_all_bears(bear_dirs))) - requirements = {} + requirements = CommentedMap() + requirements.yaml_set_start_comment( + "This is an automatically generated file.\n " + "And should not be edited by hand.") + requirements['overrides'] = 'coala-build.yaml' - requirements['pip_requirements'] = get_pip_requirements(pip_reqs) - requirements['npm_requirements'] = get_npm_requirements(npm_reqs) requirements['gem_requirements'] = get_gem_requirements(gem_reqs) + requirements['npm_requirements'] = get_npm_requirements(npm_reqs) + requirements['pip_requirements'] = get_pip_requirements(pip_reqs) + + if args.update or args.check: + input_file_path = os.path.join(PROJECT_DIR, BEAR_REQUIREMENTS_YAML) + + try: + input_file = open(input_file_path, 'r') + except FileNotFoundError: + print("bear-requirements.yaml not found. " + "Run without flags to generate it.") + exit(1) + + input_requirments = yaml.load(input_file) + + new_requirments = copy.deepcopy(input_requirments) + deep_update(new_requirments, requirements) + + if args.update: + requirements = new_requirments - output = None + if args.check: + changed = deep_diff(input_requirments, new_requirments) + if changed: + yaml.dump(changed, sys.stdout) + exit(1) if args.output == '-': output = sys.stdout else: output = open(args.output, 'w') - dump(requirements, output, default_flow_style=False) + sort_requirements(requirements) + yaml.dump(requirements, output) output.close() diff --git a/.moban.dt/bears-test-requirements.txt.jj2 b/.moban.dt/bears-test-requirements.txt.jj2 new file mode 100644 index 0000000000..01ec1125a2 --- /dev/null +++ b/.moban.dt/bears-test-requirements.txt.jj2 @@ -0,0 +1,2 @@ +{% include 'test-requirements.txt.jj2' %} +ruamel.yaml~=0.15.42 diff --git a/.moban.yaml b/.moban.yaml index 0edc788467..77375c40cb 100644 --- a/.moban.yaml +++ b/.moban.yaml @@ -26,7 +26,7 @@ configuration: targets: - setup.py: bears-setup.py.jj2 - requirements.txt: requirements.txt.jj2 - - test-requirements.txt: test-requirements.txt.jj2 + - test-requirements.txt: bears-test-requirements.txt.jj2 - bears/VERSION: VERSION.jj2 - Gemfile: Gemfile.jj2 - package.json: package.json.jj2 diff --git a/.travis.yml b/.travis.yml index 2d85acf334..e03952344d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -159,7 +159,7 @@ script: - pip install $(ls ./dist/*.whl)"[alldeps]" - bash .ci/tests.sh # Ensure bear requirements are in sync with the bear PipRequirement - - .ci/generate_bear_requirements.py + - .ci/generate_bear_requirements.py --check --update - git diff --exit-code - coala --non-interactive - codecov diff --git a/bear-requirements.yaml b/bear-requirements.yaml index 22d3702b05..93ccf93693 100644 --- a/bear-requirements.yaml +++ b/bear-requirements.yaml @@ -1,3 +1,6 @@ +# This is an automatically generated file. +# And should not be edited by hand. +overrides: coala-build.yaml gem_requirements: brakeman: version: ~>4.1.1 @@ -114,7 +117,6 @@ npm_requirements: version: '>=1.7.3' write-good: version: ~0.9.1 -overrides: coala-build.yaml pip_requirements: HTTPolice: version: ~=0.5.2 diff --git a/test-requirements.txt b/test-requirements.txt index ee59554934..ee6c751799 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,3 +17,4 @@ pytest-xdist~=1.15 requests-mock~=1.2 pip!=9.0.2, !=10.0.* wheel~=0.29 +ruamel.yaml~=0.15.42