.travis.yml: Build on xenial using template

Implements https://gitlab.com/coala/mobans/merge_requests/135
working around the lxml dependency problem on Python 3.4.

coala#6053 is only
occurring in the Travis trusty environment.
Upgrading to xenial bypasses that problem, but
requires a few workarounds, especially replacing the
Python 3.4.2 job with Python '3.4' which causes
Travis to provide Python 3.4.8.

Add gravis to template to switch jdk versions on Xenial.

Uninstall setuptools multiple times on Xenial for the same
reason it was required once on Trusty.  Also remove
unnecessary pipenv which causes pip to complain about
incompatibilities.

Bump setuptools dependency to match the version defined
in mobans.

.misc/check_moban.sh is no longer needed as the
Travis template provides a moban job, and the latest
moban handles branches.

Related to coala#4708
Related to coala#6053
Related to coala/coala-bears#2940

.moban.yaml

@@ -18,12 +18,17 @@ docs_toc: false
 test_py36: true
 allow_generic_no_cover: true
 test_timeout: 60
+travis_sentinel: true
 python_versions:
   - 3.6
   - 3.5
   - 3.4.4
   - 3.4.2
 # 3.7 excluded due to https://github.com/coala/coala/issues/6028 and others
+test_unsupported_python_versions:
+  - 2.7
+  - 3.3
+  - 3.4.1
 
 entry_points:
   console_scripts:
@@ -44,15 +49,18 @@ dependencies:
   - Pygments~=2.1
   - PyPrint~=0.2.6
   - requests~=2.12
-  - setuptools>=21.0.0
+  - setuptools>=21.2.2
   - unidiff~=0.5.2
 
 gitignore_extra_rulesets:
   - node
 
 requires:
-  - https://gitlab.com/coala/mobans.git
+  - type: git
+    url: https://gitlab.com/jayvdb/mobans
+    branch: custom-travis-template
   - https://github.com/NAStools/homeassistant.git
+
 configuration:
   template_dir:
     - .moban.dt/
@@ -82,11 +90,11 @@ targets:
   - coalib/VERSION: VERSION.jj2
   - runtime.txt: runtime.txt
   - netlify.toml: docs/netlify.toml
+  - .travis.yml: ci/coala-travis.yml.jj2
   - .misc/appveyor.yml: ci/appveyor.yml.jj2
   - .misc/run_with_env.cmd: run_with_env.cmd
   - .misc/check_unsupported.sh: ci/check_unsupported.sh.jj2
 copy:
-  - .misc/check_moban.sh: ci/check_moban.sh
   - .misc/check_setuptools.py: ci/check_setuptools.py
   # homeassistant license MIT
   - coalib/misc/Asyncio.py: async.py

.travis.yml

@@ -1,9 +1,11 @@
 sudo: false
+dist: xenial
 language: python
+notifications:
+  email: false
 python:
-  - 3.4.4
   - 3.5
-  - 3.4.2
+  - 3.4
 
 stages:
   - name: sentinel
@@ -14,19 +16,27 @@ stages:
     if: branch = master AND type = push
 
 .disable_global: &disable_global
+  addons: false
+  cache: false
+  env: {}
+  python: false
   before_install: false
-  install: true
+  install: false
   before_script: false
+  script: false
   after_success: false
   after_failure: false
+  before_deploy: false
+  deploy: false
 
-.check_moban: &check_moban
+.moban: &moban
   <<: *disable_global
   python: 3.6
   stage: moban
-  install: pip install moban
-  script: .misc/check_moban.sh
-  if: branch != master OR type = pull_request
+  install: pip install moban>=0.0.4
+  script:
+    - moban
+    - git diff --exit-code
 
 jobs:
   include:
@@ -54,28 +64,32 @@ jobs:
       <<: *disable_global
       stage: unsupported
       script: .misc/check_unsupported.sh
-    - python: 3.6
-      stage: moban
-      install: pip install moban
-      script: .misc/check_moban.sh
+    - stage: moban
+      <<: *moban
       if: branch = master AND type = push
-    - *check_moban
+    - *moban
   allow_failures:
-    - *check_moban
+    - *moban
+
+stage: test
 
 env:
   global:
     - PATH="$PATH:$TRAVIS_BUILD_DIR/node_modules/.bin"
     - BEARS_ZIP_URL=https://codeload.github.com/coala/coala-bears/zip
+    - GRAVIS="https://raw.githubusercontent.com/DanySK/Gravis-CI/master/"
+    - JDK="[email protected]"
 
 cache:
   pip: true
   directories:
     - node_modules
     - docs/_build
+    - ~/nltk_data
+    - ~/.jabba
 
 before_install:
-  - if [[ $TRAVIS_OS_NAME == osx ]]; then TRAVIS_PYTHON_VERSION=3.5; fi
+  - curl "${GRAVIS}.install-jdk-travis.sh" --output ~/.install-jdk-travis.sh
 
   - >
     if [[ $TRAVIS_OS_NAME == osx ]]; then
@@ -84,12 +98,17 @@ before_install:
       sudo ln -sf $(which pip3) $(which pip 2>/dev/null || echo /usr/bin/pip)
     fi
 
+  # See rationale in .ci/deps.python-packages.ps1 for pre-installing these
+  - pip install --prefer-binary cffi lxml
   # Use pip 9, so that setuptools can be downgraded.
   # https://github.com/coala/coala/issues/4708
   # Apart from builds with python 2.7 and 3.5, it installs setuptools
   # twice. So, it is required to uninstall one version manually.
   - pip uninstall setuptools --yes
-  - pip install pip==9.0.1 setuptools==21
+  - pip uninstall setuptools --yes || true
+  - pip uninstall setuptools --yes || true
+  - pip uninstall pipenv --yes || true
+  - pip install pip==9.0.1 setuptools==21.2.2
   - python .misc/check_setuptools.py
 
   # https://github.com/coala/coala/issues/3183
@@ -104,6 +123,7 @@ before_install:
 
   # For bears in .coafile requiring npm dependencies
   - npm install --no-save
+  - source ~/.install-jdk-travis.sh
 
 before_script:
   # Restore the original requirements.txt
@@ -113,9 +133,9 @@ before_script:
 script:
   - py.test
   - python setup.py bdist_wheel
-  - pip install ./dist/coala-*.whl
+  - pip install ./dist/*.whl
   - curl -fsSL -o coala-bears.zip $BEARS_ZIP_URL/master
-  - pip install coala-bears.zip[alldeps] --pre -U
+  - pip install coala-bears.zip[alldeps]
   - coala --non-interactive
   - python setup.py docs
   - .ci/check_man.sh
@@ -126,9 +146,6 @@ after_success:
 after_failure:
   - codecov
 
-notifications:
-  email: false
-
 branches:
   exclude:
     - /^sils\//

coalib/bearlib/abstractions/Linter.py

@@ -415,14 +415,22 @@ def process_output_corrected(self,
                 An iterator returning results containing patches for the
                 file to correct.
             """
-            return self.process_diff(
-                Diff.from_string_arrays(
-                    file,
-                    output.splitlines(keepends=True)),
-                filename,
-                diff_severity,
-                result_message,
-                diff_distance)
+            if 'XML' not in result_message:
+                return self.process_diff(
+                    Diff.from_string_arrays(
+                        file,
+                        output.splitlines(keepends=True)),
+                    filename,
+                    diff_severity,
+                    result_message,
+                    diff_distance)
+            else:
+                output = output.splitlines(keepends=True)
+                xml_diff = Diff.from_xml_diff(output, file)
+                yield Result(self,
+                             result_message,
+                             diffs={filename: xml_diff},
+                             severity=diff_severity)
 
         def process_output_unified_diff(self,
                                         output,

coalib/processes/Processing.py

@@ -421,8 +421,7 @@ def instantiate_processes(section,
     # run only for the changed files if caching is enabled.
 
     # Start tracking all the files
-    if cache and (loaded_valid_local_bears_count == loaded_local_bears_count
-                  and not use_raw_files):
+    if cache and (loaded_valid_local_bears_count == loaded_local_bears_count):
         cache.track_files(set(complete_filename_list))
         changed_files = cache.get_uncached_files(
             set(filename_list)) if cache else filename_list
@@ -806,7 +805,7 @@ def execute_section(section,
 
     for runner in processes:
         runner.start()
-
+        
     try:
         return (process_queues(processes,
                                arg_dict['control_queue'],

coalib/results/Diff.py

@@ -1,8 +1,11 @@
+import os
+
 import copy
 import difflib
 import logging
 
 from unidiff import PatchSet
+from xmldiff import main, formatting
 
 from coalib.results.LineDiff import LineDiff, ConflictError
 from coalib.results.SourceRange import SourceRange
@@ -155,6 +158,30 @@ def from_unified_diff(cls, unified_diff, original_file):
 
         return diff
 
+    @classmethod
+    def from_xml_diff(cls, new_file, original_file):
+        diff = Diff(original_file)
+
+        #diff.xml_diff(new_file, original_file)
+
+        formatter = formatting.XMLFormatter(normalize=formatting.WS_BOTH,
+                                            pretty_print=True)
+        new_content = "".join(new_file)
+        original_content = "".join(original_file)
+        current_dir = os.getcwd()
+        file1 = os.path.join(current_dir, 'file1')
+        file2 = os.path.join(current_dir, 'file2')
+        with open(file1, 'w') as f1:
+            f1.write(original_content)
+        with open(file2, 'w') as f2:
+            f2.write(new_content)
+        xml_diff = main.diff_files(file1, file2,
+                               formatter=formatter)
+        diff._changes['diff'] = xml_diff
+        os.remove(file1)
+        os.remove(file2)
+        return diff
+
     def _get_change(self, line_nr, min_line=1):
         if not isinstance(line_nr, int):
             raise TypeError('line_nr needs to be an integer.')
@@ -284,6 +311,14 @@ def unified_diff(self):
 
         return ''.join(self._generate_linebreaks(list_unified_diff))
 
+    @property
+    def xml_diff(self, filename, output_filename):
+        formatter = formatting.XMLFormatter(normalize=formatting.WS_BOTH,
+                                            pretty_print=True)
+        diff = main.diff_files('xml_file1.xml', 'xml_file2.xml',
+                               formatter=formatter)
+        self._changes['diff'] = diff
+
     def __json__(self):
         """
         Override JSON export, using the unified diff is the easiest thing for

requirements.txt

@@ -8,5 +8,6 @@ packaging>=16.8
 Pygments~=2.1
 PyPrint~=0.2.6
 requests~=2.12
-setuptools>=21.0.0
+setuptools>=21.2.2
 unidiff~=0.5.2
+git+https://github.com/Shoobx/xmldiff#egg=xmldiff