.ci/generate_bear_requirements: Add Update & Check

This commits updates the script to use ruamel.yaml
which is better fork of PyYaml.
Also adds Update and Check mode for diffing and
updating Bear-Requirements yaml.

Closes #2444

.ci/generate_bear_requirements.py

@@ -14,13 +14,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
+import copy
 import itertools
 import json
 import os
 import sys
-from collections import OrderedDict
 
-from yaml import dump
+from ruamel.yaml import YAML, RoundTripDumper
+from ruamel.yaml.comments import CommentedMap
 from pyprint.NullPrinter import NullPrinter
 
 from coalib.bears.BEAR_KIND import BEAR_KIND
@@ -30,6 +31,10 @@
 from dependency_management.requirements.NpmRequirement import NpmRequirement
 from dependency_management.requirements.PipRequirement import PipRequirement
 
+yaml = YAML(typ='rt')
+yaml.default_flow_style = False
+yaml.Dumper = RoundTripDumper
+
 BEAR_REQUIREMENTS_YAML = "bear-requirements.yaml"
 _VERSION_OPERATORS = ('<', '>', '~', '=', '-', '!')
 
@@ -51,6 +56,11 @@ def get_args():
     parser.add_argument('--bear-dirs', '-d', nargs='+', metavar='DIR',
                         help='additional directories which may contain bears')
 
+    parser.add_argument('--check', '-c', action='store_true',
+                        help='performs a dry run, and reports differences.')
+    parser.add_argument('--update', '-u', action='store_true',
+                        help='updates "bear-requirements.yaml" '
+                             'instead of overwriting')
     args = parser.parse_args()
 
     return args
@@ -144,6 +154,47 @@ def get_pip_requirements(requirements):
     return _get_requirements(requirements, '~=', inherited_requirements)
 
 
+def deep_update(target, src):
+    for key, value in src.items():
+        if key not in target:
+            target[key] = copy.deepcopy(value)
+        else:
+            if isinstance(value, list):
+                target[key].extend(value)
+            elif isinstance(value, dict):
+                deep_update(target[key], value)
+            elif isinstance(value, set):
+                target[key].update(value.copy())
+            else:
+                target[key] = copy.copy(value)
+
+
+def deep_diff(target, src):
+    errors = []
+    for key, value in src.items():
+        if key not in target:
+            errors.append((key, 'Missing'))
+        elif target[key] != value:
+            if isinstance(value, list):
+                if [x for x in value if x not in target[key]]:
+                    errors.append(key)
+            elif isinstance(value, dict):
+                if target[key] != value:
+                    errors.append((key, deep_diff(target[key], value)))
+            elif isinstance(value, set):
+                if set(target[key]).symmetric_difference(value):
+                    errors.append(key)
+            else:
+                errors.append((key, target[key]))
+    return errors
+
+
+def sort_requirements(req_dict):
+    for key in ['pip_requirements', 'npm_requirements', 'gem_requirements']:
+        req_dict[key] = CommentedMap(sorted(req_dict[key].items(),
+                                            key=lambda t: t[0]))
+
+
 if __name__ == '__main__':
     args = get_args()
 
@@ -155,18 +206,45 @@ def get_pip_requirements(requirements):
     pip_reqs, npm_reqs, gem_reqs = (
         get_all_requirements(get_all_bears(bear_dirs)))
 
-    requirements = {}
+    requirements = CommentedMap()
+    requirements.yaml_set_start_comment(
+                    "This is an automatically generated file.\n "
+                    "And should not be edited by hand.")
+
     requirements['overrides'] = 'coala-build.yaml'
-    requirements['pip_requirements'] = get_pip_requirements(pip_reqs)
-    requirements['npm_requirements'] = get_npm_requirements(npm_reqs)
     requirements['gem_requirements'] = get_gem_requirements(gem_reqs)
+    requirements['npm_requirements'] = get_npm_requirements(npm_reqs)
+    requirements['pip_requirements'] = get_pip_requirements(pip_reqs)
+
+    if args.update or args.check:
+        input_file_path = os.path.join(PROJECT_DIR, BEAR_REQUIREMENTS_YAML)
+
+        try:
+            input_file = open(input_file_path, 'r')
+        except FileNotFoundError:
+            print("bear-requirements.yaml not found. "
+                  "Run without flags to generate it.")
+            exit(1)
+
+        input_requirments = yaml.load(input_file)
+
+        new_requirments = copy.deepcopy(input_requirments)
+        deep_update(new_requirments, requirements)
+
+        if args.update:
+            requirements = new_requirments
 
-    output = None
+        if args.check:
+            changed = deep_diff(input_requirments, new_requirments)
+            if changed:
+                yaml.dump(changed, sys.stdout)
+            exit(1)
 
     if args.output == '-':
         output = sys.stdout
     else:
         output = open(args.output, 'w')
 
-    dump(requirements, output, default_flow_style=False)
+    sort_requirements(requirements)
+    yaml.dump(requirements, output)
     output.close()

.moban.dt/bears-test-requirements.txt.jj2

@@ -0,0 +1,2 @@
+{% include 'test-requirements.txt.jj2' %}
+ruamel.yaml~=0.15.42

.moban.yaml

@@ -26,7 +26,7 @@ configuration:
 targets:
   - setup.py: bears-setup.py.jj2
   - requirements.txt: requirements.txt.jj2
-  - test-requirements.txt: test-requirements.txt.jj2
+  - test-requirements.txt: bears-test-requirements.txt.jj2
   - bears/VERSION: VERSION.jj2
   - Gemfile: Gemfile.jj2
   - package.json: package.json.jj2

.travis.yml

@@ -159,7 +159,7 @@ script:
   - pip install $(ls ./dist/*.whl)"[alldeps]"
   - bash .ci/tests.sh
   # Ensure bear requirements are in sync with the bear PipRequirement
-  - .ci/generate_bear_requirements.py
+  - .ci/generate_bear_requirements.py --check --update
   - git diff --exit-code
   - coala --non-interactive
   - codecov

bear-requirements.yaml

@@ -1,3 +1,6 @@
+# This is an automatically generated file.
+# And should not be edited by hand.
+overrides: coala-build.yaml
 gem_requirements:
   brakeman:
     version: ~>4.1.1
@@ -114,7 +117,6 @@ npm_requirements:
     version: '>=1.7.3'
   write-good:
     version: ~0.9.1
-overrides: coala-build.yaml
 pip_requirements:
   HTTPolice:
     version: ~=0.5.2

test-requirements.txt

@@ -17,3 +17,4 @@ pytest-xdist~=1.15
 requests-mock~=1.2
 pip!=9.0.2, !=10.0.*
 wheel~=0.29
+ruamel.yaml~=0.15.42