.ci/generate_bear_requirements: Add Update & Check

This commits updates the script to use ruamel.yaml
which is better fork of PyYaml.
Also adds Update and Check mode for diffing and
updating Bear-Requirements yaml.

Closes coala#2444

.ci/generate_bear_requirements.py

@@ -18,9 +18,10 @@
 import json
 import os
 import sys
-from collections import OrderedDict
+import copy
 
-from yaml import dump
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
 from pyprint.NullPrinter import NullPrinter
 
 from coalib.bears.BEAR_KIND import BEAR_KIND
@@ -30,6 +31,9 @@
 from dependency_management.requirements.NpmRequirement import NpmRequirement
 from dependency_management.requirements.PipRequirement import PipRequirement
 
+yaml = YAML(typ="rt")
+yaml.default_flow_style = False
+
 BEAR_REQUIREMENTS_YAML = "bear-requirements.yaml"
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -54,6 +58,11 @@ def get_args():
     parser.add_argument('--bear-dirs', '-d', nargs='+', metavar='DIR',
                         help='additional directories which may contain bears')
 
+    parser.add_argument('--check', '-c', action='store_true',
+                        help='performs a dry run, and reports differences.')
+    parser.add_argument('--update', '-u', action='store_true',
+                        help='updates "bear-requirements.yaml" '
+                             'instead of overwriting')
     args = parser.parse_args()
 
     return args
@@ -126,7 +135,7 @@ def _to_entry(obj, version=None):
 
 
 def get_gem_requirements(requirements):
-    gem_dependencies = {}
+    gem_dependencies = CommentedMap({})
 
     for requirement in requirements:
         gem_dependencies[requirement.package] = _to_entry(requirement)
@@ -135,7 +144,7 @@ def get_gem_requirements(requirements):
 
 
 def get_npm_requirements(requirements):
-    npm_dependencies = {}
+    npm_dependencies = CommentedMap({})
 
     for requirement in requirements:
         package_name = requirement.package
@@ -151,7 +160,7 @@ def get_npm_requirements(requirements):
 def get_pip_requirements(requirements):
     inherited_requirements = get_inherited_requirements()
 
-    pip_requirements = {}
+    pip_requirements = CommentedMap({})
 
     for requirement in requirements:
         if requirement.package in inherited_requirements:
@@ -167,6 +176,46 @@ def get_pip_requirements(requirements):
 
     return pip_requirements
 
+def deepupdate(target, src):
+    for key, value in src.items():
+        if isinstance(value, list):
+            if not key in target:
+                target[key] = copy.deepcopy(value)
+            else:
+                target[key].extend(value)
+        elif isinstance(value, dict):
+            if not key in target:
+                target[key] = copy.deepcopy(value)
+            else:
+                deepupdate(target[key], value)
+        elif isinstance(value, set):
+            if not key in target:
+                target[key] = value.copy()
+            else:
+                target[key].update(value.copy())
+        else:
+            target[key] = copy.copy(value)
+
+
+def deepdiff(target, src):
+    errors = []
+    for key, value in src.items():
+        if not key in target:
+                errors.append((key, "Missing"))
+        elif target[key] != value:
+            if isinstance(value, list):
+                if [x for x in value if x not in target[key]]:
+                    errors.append(key)
+            elif isinstance(value, dict):
+                if target[key] != value:
+                    e = deepdiff(target[key], value)
+                    errors.append((key, e))
+            elif isinstance(value, set):
+                if set(target[key]).symmetric_difference(value):
+                    errors.append(key)
+            else:
+                errors.append((key, target[key]))
+    return errors
 
 if __name__ == '__main__':
     args = get_args()
@@ -179,18 +228,31 @@ def get_pip_requirements(requirements):
     pip_reqs, npm_reqs, gem_reqs = (
         get_all_requirements(get_all_bears(bear_dirs)))
 
-    requirements = {}
-    requirements['overrides'] = 'coala-build.yaml'
-    requirements['pip_requirements'] = get_pip_requirements(pip_reqs)
-    requirements['npm_requirements'] = get_npm_requirements(npm_reqs)
+    requirements = CommentedMap({})
+
     requirements['gem_requirements'] = get_gem_requirements(gem_reqs)
+    requirements['npm_requirements'] = get_npm_requirements(npm_reqs)
+    requirements['pip_requirements'] = get_pip_requirements(pip_reqs)
+
+    if args.update or args.check:
+        inputFile = open(os.path.join(PROJECT_DIR, BEAR_REQUIREMENTS_YAML),'r')
+        inputRequirments = yaml.load(inputFile)
+        newRequirments = copy.deepcopy(inputRequirments)
+        deepupdate(newRequirments, requirements)
+
+        if args.update:
+            requirements = newRequirments
 
-    output = None
+        if args.check:
+            y = deepdiff(inputRequirments, newRequirments)
+            if y:
+                yaml.dump(y, sys.stdout)
+            exit()
 
     if args.output == '-':
         output = sys.stdout
     else:
         output = open(args.output, 'w')
 
-    dump(requirements, output, default_flow_style=False)
+    yaml.dump(requirements, output)
     output.close()

.travis.yml

@@ -115,7 +115,8 @@ script:
   - pip install $(ls ./dist/*.whl)"[alldeps]"
   - bash .ci/tests.sh
   # Ensure bear requirements are in sync with the bear PipRequirement
-  - .ci/generate_bear_requirements.py
+  - .ci/generate_bear_requirements.py --check
+  - .ci/generate_bear_requirements.py --update
   - git clone https://github.com/moremoban/setupmobans ../setupmobans
   - git clone https://gitlab.com/coala/mobans ../coala-mobans
   - moban

bear-requirements.yaml

@@ -1,3 +1,4 @@
+overrides: coala-build.yaml
 gem_requirements:
   brakeman:
     version: 4.1.1
@@ -113,7 +114,6 @@ npm_requirements:
     version: '>=1.7.3'
   write-good:
     version: ~0.9.1
-overrides: coala-build.yaml
 pip_requirements:
   HTTPolice:
     version: ~=0.5.2

test-requirements.txt

@@ -9,3 +9,4 @@ pytest-timeout~=1.2
 pytest-xdist~=1.15
 requests_mock~=0.7.0
 wheel~=0.29
+ruamel.yaml==0.15.37