[WIP] .ci/generate_bear_requirements: Changes

This commits updates the script to use ruamel.yaml
which is better fork of PyYaml.
Add Update and Check mode for diffing and updating
Bear-Requirments yaml.

Closes coala#2444

.ci/generate_bear_requirements.py

@@ -18,9 +18,10 @@
 import json
 import os
 import sys
-from collections import OrderedDict
+import copy
 
-from yaml import dump
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
 from pyprint.NullPrinter import NullPrinter
 
 from coalib.bears.BEAR_KIND import BEAR_KIND
@@ -30,6 +31,9 @@
 from dependency_management.requirements.NpmRequirement import NpmRequirement
 from dependency_management.requirements.PipRequirement import PipRequirement
 
+yaml = YAML(typ="rt")
+yaml.default_flow_style = False
+
 BEAR_REQUIREMENTS_YAML = "bear-requirements.yaml"
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -54,6 +58,10 @@ def get_args():
     parser.add_argument('--bear-dirs', '-d', nargs='+', metavar='DIR',
                         help='additional directories which may contain bears')
 
+    parser.add_argument('--check', '-c', action='store_true',
+                        help='performs a dry run, and reports differences.')
+    parser.add_argument('--update', '-u',action='store_true',
+                        help='updates "bear-requirements.yaml" instaead of overwriting')
     args = parser.parse_args()
 
     return args
@@ -126,7 +134,7 @@ def _to_entry(obj, version=None):
 
 
 def get_gem_requirements(requirements):
-    gem_dependencies = {}
+    gem_dependencies = CommentedMap({})
 
     for requirement in requirements:
         gem_dependencies[requirement.package] = _to_entry(requirement)
@@ -135,7 +143,7 @@ def get_gem_requirements(requirements):
 
 
 def get_npm_requirements(requirements):
-    npm_dependencies = {}
+    npm_dependencies = CommentedMap({})
 
     for requirement in requirements:
         package_name = requirement.package
@@ -151,7 +159,7 @@ def get_npm_requirements(requirements):
 def get_pip_requirements(requirements):
     inherited_requirements = get_inherited_requirements()
 
-    pip_requirements = {}
+    pip_requirements = CommentedMap({})
 
     for requirement in requirements:
         if requirement.package in inherited_requirements:
@@ -167,6 +175,46 @@ def get_pip_requirements(requirements):
 
     return pip_requirements
 
+def deepupdate(target, src):
+    for key, value in src.items():
+        if isinstance(value, list):
+            if not key in target:
+                target[key] = copy.deepcopy(value)
+            else:
+                target[key].extend(value)
+        elif isinstance(value, dict):
+            if not key in target:
+                target[key] = copy.deepcopy(value)
+            else:
+                deepupdate(target[key], value)
+        elif isinstance(value, set):
+            if not key in target:
+                target[key] = value.copy()
+            else:
+                target[key].update(value.copy())
+        else:
+            target[key] = copy.copy(value)
+
+
+def deepdiff(target, src):
+    errors = []
+    for key, value in src.items():
+        if not key in target:
+                errors.append((key, "Missing"))
+        elif target[key] != value:
+            if isinstance(value, list):
+                if [x for x in value if x not in target[key]]:
+                    errors.append(key)
+            elif isinstance(value, dict):
+                if target[key] != value:
+                    e = deepdiff(target[key], value)
+                    errors.append((key, e))
+            elif isinstance(value, set):
+                if set(target[key]).symmetric_difference(value):
+                    errors.append(key)
+            else:
+                errors.append((key, target[key]))
+    return errors
 
 if __name__ == '__main__':
     args = get_args()
@@ -179,18 +227,31 @@ def get_pip_requirements(requirements):
     pip_reqs, npm_reqs, gem_reqs = (
         get_all_requirements(get_all_bears(bear_dirs)))
 
-    requirements = {}
-    requirements['overrides'] = 'coala-build.yaml'
-    requirements['pip_requirements'] = get_pip_requirements(pip_reqs)
-    requirements['npm_requirements'] = get_npm_requirements(npm_reqs)
+    requirements = CommentedMap({})
+
     requirements['gem_requirements'] = get_gem_requirements(gem_reqs)
+    requirements['npm_requirements'] = get_npm_requirements(npm_reqs)
+    requirements['pip_requirements'] = get_pip_requirements(pip_reqs)
+
+    if args.update or args.check:
+        inputFile = open(os.path.join(PROJECT_DIR, BEAR_REQUIREMENTS_YAML),'r')
+        inputRequirments = yaml.load(inputFile)
+        newRequirments = copy.deepcopy(inputRequirments)
+        deepupdate(newRequirments, requirements)
+
+        if args.update:
+            requirements = newRequirments
 
-    output = None
+        if args.check:
+            y = deepdiff(inputRequirments, newRequirments)
+            if y:
+                yaml.dump(y, sys.stdout)
+            exit()
 
     if args.output == '-':
         output = sys.stdout
     else:
         output = open(args.output, 'w')
 
-    dump(requirements, output, default_flow_style=False)
+    yaml.dump(requirements, output)
     output.close()

.travis.yml

@@ -115,7 +115,8 @@ script:
   - pip install $(ls ./dist/*.whl)"[alldeps]"
   - bash .ci/tests.sh
   # Ensure bear requirements are in sync with the bear PipRequirement
-  - .ci/generate_bear_requirements.py
+  - .ci/generate_bear_requirements.py --check
+  - .ci/generate_bear_requirements.py --update
   - git clone https://github.com/moremoban/setupmobans ../setupmobans
   - git clone https://gitlab.com/coala/mobans ../coala-mobans
   - moban

bear-requirements.yaml

@@ -1,3 +1,84 @@
+# This is a comment!
+overrides: coala-build.yaml
+pip_requirements:
+  HTTPolice:
+    version: ~=0.5.2
+  aenum:
+    version: ~=2.0.8
+  apertium-lint:
+    version: ~=0.29
+  autoflake:
+    version: ~=0.7
+  autopep8:
+    version: ~=1.2
+  bandit:
+    version: ~=1.2
+  cmakelint:
+    version: ~=1.3
+  cppclean:
+    version: ~=0.12.0
+  cpplint:
+    version: ~=1.3
+  dennis:
+    version: ~=0.9
+  docutils-ast-writer:
+    version: ~=0.1.2
+  eradicate:
+    version: ~=0.1.6
+  guess-language-spirit:
+    version: ~=0.5.2
+  html-linter:
+    version: ~=0.4.0
+  isort:
+    version: ~=4.2
+  language-check:
+    version: ~=1.0
+  libclang-py3:
+    version: ~=3.4.0
+  lxml:
+    version: ==3.6.0
+  memento_client:
+    version: ~=0.6.1
+  munkres3:
+    version: ~=1.0
+  mypy-lang:
+    version: ~=0.4.6
+  nbformat:
+    version: ~=4.1
+  nltk:
+    version: ~=3.2
+  proselint:
+    version: ~=0.7.0
+  pycodestyle:
+    version: ~=2.2
+  pydocstyle:
+    version: ~=2.0
+  pyflakes:
+    version: ~=1.5.0
+  pylint:
+    version: ~=1.7.2
+  pyroma:
+    version: ~=2.2.0
+  pyyaml:
+    version: ~=3.12
+  radon:
+    version: ==1.4.0
+  restructuredtext-lint:
+    version: ~=1.0.0
+  rstcheck:
+    version: ~=3.1
+  safety:
+    version: ~=0.5.1
+  scspell3k:
+    version: ~=2.0
+  vim-vint:
+    version: ~=0.3.12
+  vulture:
+    version: ~=0.25.0
+  yamllint:
+    version: ~=1.6.1
+  yapf:
+    version: ~=0.16.0
 gem_requirements:
   csvlint: true
   haml_lint:
@@ -109,83 +190,3 @@ npm_requirements:
     version: '>=1.7.3'
   write-good:
     version: ~0.9.1
-overrides: coala-build.yaml
-pip_requirements:
-  HTTPolice:
-    version: ~=0.5.2
-  aenum:
-    version: ~=2.0.8
-  apertium-lint:
-    version: ~=0.29
-  autoflake:
-    version: ~=0.7
-  autopep8:
-    version: ~=1.2
-  bandit:
-    version: ~=1.2
-  cmakelint:
-    version: ~=1.3
-  cppclean:
-    version: ~=0.12.0
-  cpplint:
-    version: ~=1.3
-  dennis:
-    version: ~=0.9
-  docutils-ast-writer:
-    version: ~=0.1.2
-  eradicate:
-    version: ~=0.1.6
-  guess-language-spirit:
-    version: ~=0.5.2
-  html-linter:
-    version: ~=0.4.0
-  isort:
-    version: ~=4.2
-  language-check:
-    version: ~=1.0
-  libclang-py3:
-    version: ~=3.4.0
-  lxml:
-    version: ==3.6.0
-  memento_client:
-    version: ~=0.6.1
-  munkres3:
-    version: ~=1.0
-  mypy-lang:
-    version: ~=0.4.6
-  nbformat:
-    version: ~=4.1
-  nltk:
-    version: ~=3.2
-  proselint:
-    version: ~=0.7.0
-  pycodestyle:
-    version: ~=2.2
-  pydocstyle:
-    version: ~=2.0
-  pyflakes:
-    version: ~=1.5.0
-  pylint:
-    version: ~=1.7.2
-  pyroma:
-    version: ~=2.2.0
-  pyyaml:
-    version: ~=3.12
-  radon:
-    version: ==1.4.0
-  restructuredtext-lint:
-    version: ~=1.0.0
-  rstcheck:
-    version: ~=3.1
-  safety:
-    version: ~=0.5.1
-  scspell3k:
-    version: ~=2.0
-  vim-vint:
-    version: ~=0.3.12
-  vulture:
-    version: ~=0.25.0
-  yamllint:
-    version: ~=1.6.1
-  yapf:
-    version: ~=0.16.0

test-requirements.txt

@@ -9,3 +9,4 @@ pytest-timeout~=1.2
 pytest-xdist~=1.15
 requests_mock~=0.7.0
 wheel~=0.29
+ruamel.yaml==0.15.37