From 9c7b14e49cd721cb585998f0d6c8ad34617c7ffb Mon Sep 17 00:00:00 2001 From: Arjun Nemani Date: Wed, 18 Apr 2018 22:32:50 +0530 Subject: [PATCH] .ci/generate_bear_requirements: Add Update & Check This commits updates the script to use ruamel.yaml which is better fork of PyYaml. Also adds Update and Check mode for diffing and updating Bear-Requirements yaml. Closes https://github.com/coala/coala-bears/issues/2444 --- .ci/generate_bear_requirements.py | 84 +++++++++++++++++++++++++++---- .travis.yml | 3 +- bear-requirements.yaml | 2 +- test-requirements.txt | 1 + 4 files changed, 77 insertions(+), 13 deletions(-) diff --git a/.ci/generate_bear_requirements.py b/.ci/generate_bear_requirements.py index b6fbfcdc9a..cc9366c613 100755 --- a/.ci/generate_bear_requirements.py +++ b/.ci/generate_bear_requirements.py @@ -18,9 +18,10 @@ import json import os import sys -from collections import OrderedDict +import copy -from yaml import dump +from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedMap from pyprint.NullPrinter import NullPrinter from coalib.bears.BEAR_KIND import BEAR_KIND @@ -30,6 +31,9 @@ from dependency_management.requirements.NpmRequirement import NpmRequirement from dependency_management.requirements.PipRequirement import PipRequirement +yaml = YAML(typ="rt") +yaml.default_flow_style = False + BEAR_REQUIREMENTS_YAML = "bear-requirements.yaml" THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -54,6 +58,11 @@ def get_args(): parser.add_argument('--bear-dirs', '-d', nargs='+', metavar='DIR', help='additional directories which may contain bears') + parser.add_argument('--check', '-c', action='store_true', + help='performs a dry run, and reports differences.') + parser.add_argument('--update', '-u', action='store_true', + help='updates "bear-requirements.yaml" ' + 'instead of overwriting') args = parser.parse_args() return args @@ -126,7 +135,7 @@ def _to_entry(obj, version=None): def get_gem_requirements(requirements): - gem_dependencies = {} + gem_dependencies = CommentedMap({}) for requirement in requirements: gem_dependencies[requirement.package] = _to_entry(requirement) @@ -135,7 +144,7 @@ def get_gem_requirements(requirements): def get_npm_requirements(requirements): - npm_dependencies = {} + npm_dependencies = CommentedMap({}) for requirement in requirements: package_name = requirement.package @@ -151,7 +160,7 @@ def get_npm_requirements(requirements): def get_pip_requirements(requirements): inherited_requirements = get_inherited_requirements() - pip_requirements = {} + pip_requirements = CommentedMap({}) for requirement in requirements: if requirement.package in inherited_requirements: @@ -167,6 +176,46 @@ def get_pip_requirements(requirements): return pip_requirements +def deepupdate(target, src): + for key, value in src.items(): + if isinstance(value, list): + if not key in target: + target[key] = copy.deepcopy(value) + else: + target[key].extend(value) + elif isinstance(value, dict): + if not key in target: + target[key] = copy.deepcopy(value) + else: + deepupdate(target[key], value) + elif isinstance(value, set): + if not key in target: + target[key] = value.copy() + else: + target[key].update(value.copy()) + else: + target[key] = copy.copy(value) + + +def deepdiff(target, src): + errors = [] + for key, value in src.items(): + if not key in target: + errors.append((key, "Missing")) + elif target[key] != value: + if isinstance(value, list): + if [x for x in value if x not in target[key]]: + errors.append(key) + elif isinstance(value, dict): + if target[key] != value: + e = deepdiff(target[key], value) + errors.append((key, e)) + elif isinstance(value, set): + if set(target[key]).symmetric_difference(value): + errors.append(key) + else: + errors.append((key, target[key])) + return errors if __name__ == '__main__': args = get_args() @@ -179,18 +228,31 @@ def get_pip_requirements(requirements): pip_reqs, npm_reqs, gem_reqs = ( get_all_requirements(get_all_bears(bear_dirs))) - requirements = {} - requirements['overrides'] = 'coala-build.yaml' - requirements['pip_requirements'] = get_pip_requirements(pip_reqs) - requirements['npm_requirements'] = get_npm_requirements(npm_reqs) + requirements = CommentedMap({}) + requirements['gem_requirements'] = get_gem_requirements(gem_reqs) + requirements['npm_requirements'] = get_npm_requirements(npm_reqs) + requirements['pip_requirements'] = get_pip_requirements(pip_reqs) + + if args.update or args.check: + inputFile = open(os.path.join(PROJECT_DIR, BEAR_REQUIREMENTS_YAML),'r') + inputRequirments = yaml.load(inputFile) + newRequirments = copy.deepcopy(inputRequirments) + deepupdate(newRequirments, requirements) + + if args.update: + requirements = newRequirments - output = None + if args.check: + y = deepdiff(inputRequirments, newRequirments) + if y: + yaml.dump(y, sys.stdout) + exit() if args.output == '-': output = sys.stdout else: output = open(args.output, 'w') - dump(requirements, output, default_flow_style=False) + yaml.dump(requirements, output) output.close() diff --git a/.travis.yml b/.travis.yml index 00c9e2d33d..8cb3fb4d79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -115,7 +115,8 @@ script: - pip install $(ls ./dist/*.whl)"[alldeps]" - bash .ci/tests.sh # Ensure bear requirements are in sync with the bear PipRequirement - - .ci/generate_bear_requirements.py + - .ci/generate_bear_requirements.py --check + - .ci/generate_bear_requirements.py --update - git clone https://github.com/moremoban/setupmobans ../setupmobans - git clone https://gitlab.com/coala/mobans ../coala-mobans - moban diff --git a/bear-requirements.yaml b/bear-requirements.yaml index 5e9cd1ce41..3176d60c2a 100644 --- a/bear-requirements.yaml +++ b/bear-requirements.yaml @@ -1,3 +1,4 @@ +overrides: coala-build.yaml gem_requirements: brakeman: version: 4.1.1 @@ -113,7 +114,6 @@ npm_requirements: version: '>=1.7.3' write-good: version: ~0.9.1 -overrides: coala-build.yaml pip_requirements: HTTPolice: version: ~=0.5.2 diff --git a/test-requirements.txt b/test-requirements.txt index f9c02926a7..7370b91d73 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,3 +9,4 @@ pytest-timeout~=1.2 pytest-xdist~=1.15 requests_mock~=0.7.0 wheel~=0.29 +ruamel.yaml==0.15.37