From 8dcd9b8b78d063bff606840146c6e539335df4a1 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 22 Jul 2019 20:16:43 +0700 Subject: [PATCH] .travis.yml: Build on xenial using template Implements https://gitlab.com/coala/mobans/merge_requests/135 working around the lxml dependency problem on Python 3.4. https://github.com/coala/coala/issues/6053 is only occurring in the Travis trusty environment. Upgrading to xenial bypasses that problem, but requires a few workarounds, especially replacing the Python 3.4.2 job with Python '3.4' which causes Travis to provide Python 3.4.8. Add gravis to template to switch jdk versions on Xenial. Uninstall setuptools multiple times on Xenial for the same reason it was required once on Trusty. Also remove unnecessary pipenv which causes pip to complain about incompatibilities. Bump setuptools dependency to match the version defined in mobans. .misc/check_moban.sh is no longer needed as the Travis template provides a moban job, and the latest moban handles branches. Related to https://github.com/coala/coala/issues/4708 Related to https://github.com/coala/coala/issues/6053 Related to https://github.com/coala/coala-bears/issues/2940 --- .misc/check_moban.sh | 15 ------- .moban.yaml | 14 +++++-- .travis.yml | 57 +++++++++++++++++---------- coalib/bearlib/abstractions/Linter.py | 24 +++++++---- coalib/processes/Processing.py | 5 +-- coalib/results/Diff.py | 35 ++++++++++++++++ requirements.txt | 3 +- 7 files changed, 103 insertions(+), 50 deletions(-) delete mode 100755 .misc/check_moban.sh diff --git a/.misc/check_moban.sh b/.misc/check_moban.sh deleted file mode 100755 index 293ba7ccdb..0000000000 --- a/.misc/check_moban.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -ex - -: "${MOBAN_BRANCH:=master}" - -if [ ! -d ../coala-mobans ]; then - git clone https://gitlab.com/coala/mobans \ - --branch=${MOBAN_BRANCH} ../coala-mobans || exit 0 -fi - -date - -moban -git diff --exit-code diff --git a/.moban.yaml b/.moban.yaml index 32e381cf1d..efd4434d1d 100644 --- a/.moban.yaml +++ b/.moban.yaml @@ -18,12 +18,17 @@ docs_toc: false test_py36: true allow_generic_no_cover: true test_timeout: 60 +travis_sentinel: true python_versions: - 3.6 - 3.5 - 3.4.4 - 3.4.2 # 3.7 excluded due to https://github.com/coala/coala/issues/6028 and others +test_unsupported_python_versions: + - 2.7 + - 3.3 + - 3.4.1 entry_points: console_scripts: @@ -44,15 +49,18 @@ dependencies: - Pygments~=2.1 - PyPrint~=0.2.6 - requests~=2.12 - - setuptools>=21.0.0 + - setuptools>=21.2.2 - unidiff~=0.5.2 gitignore_extra_rulesets: - node requires: - - https://gitlab.com/coala/mobans.git + - type: git + url: https://gitlab.com/jayvdb/mobans + branch: custom-travis-template - https://github.com/NAStools/homeassistant.git + configuration: template_dir: - .moban.dt/ @@ -82,11 +90,11 @@ targets: - coalib/VERSION: VERSION.jj2 - runtime.txt: runtime.txt - netlify.toml: docs/netlify.toml + - .travis.yml: ci/coala-travis.yml.jj2 - .misc/appveyor.yml: ci/appveyor.yml.jj2 - .misc/run_with_env.cmd: run_with_env.cmd - .misc/check_unsupported.sh: ci/check_unsupported.sh.jj2 copy: - - .misc/check_moban.sh: ci/check_moban.sh - .misc/check_setuptools.py: ci/check_setuptools.py # homeassistant license MIT - coalib/misc/Asyncio.py: async.py diff --git a/.travis.yml b/.travis.yml index c90d782bcb..6686b58108 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,11 @@ sudo: false +dist: xenial language: python +notifications: + email: false python: - - 3.4.4 - 3.5 - - 3.4.2 + - 3.4 stages: - name: sentinel @@ -14,19 +16,27 @@ stages: if: branch = master AND type = push .disable_global: &disable_global + addons: false + cache: false + env: {} + python: false before_install: false - install: true + install: false before_script: false + script: false after_success: false after_failure: false + before_deploy: false + deploy: false -.check_moban: &check_moban +.moban: &moban <<: *disable_global python: 3.6 stage: moban - install: pip install moban - script: .misc/check_moban.sh - if: branch != master OR type = pull_request + install: pip install moban>=0.0.4 + script: + - moban + - git diff --exit-code jobs: include: @@ -54,28 +64,32 @@ jobs: <<: *disable_global stage: unsupported script: .misc/check_unsupported.sh - - python: 3.6 - stage: moban - install: pip install moban - script: .misc/check_moban.sh + - stage: moban + <<: *moban if: branch = master AND type = push - - *check_moban + - *moban allow_failures: - - *check_moban + - *moban + +stage: test env: global: - PATH="$PATH:$TRAVIS_BUILD_DIR/node_modules/.bin" - BEARS_ZIP_URL=https://codeload.github.com/coala/coala-bears/zip + - GRAVIS="https://raw.githubusercontent.com/DanySK/Gravis-CI/master/" + - JDK="adopt@1.8.212-04" cache: pip: true directories: - node_modules - docs/_build + - ~/nltk_data + - ~/.jabba before_install: - - if [[ $TRAVIS_OS_NAME == osx ]]; then TRAVIS_PYTHON_VERSION=3.5; fi + - curl "${GRAVIS}.install-jdk-travis.sh" --output ~/.install-jdk-travis.sh - > if [[ $TRAVIS_OS_NAME == osx ]]; then @@ -84,12 +98,17 @@ before_install: sudo ln -sf $(which pip3) $(which pip 2>/dev/null || echo /usr/bin/pip) fi + # See rationale in .ci/deps.python-packages.ps1 for pre-installing these + - pip install --prefer-binary cffi lxml # Use pip 9, so that setuptools can be downgraded. # https://github.com/coala/coala/issues/4708 # Apart from builds with python 2.7 and 3.5, it installs setuptools # twice. So, it is required to uninstall one version manually. - pip uninstall setuptools --yes - - pip install pip==9.0.1 setuptools==21 + - pip uninstall setuptools --yes || true + - pip uninstall setuptools --yes || true + - pip uninstall pipenv --yes || true + - pip install pip==9.0.1 setuptools==21.2.2 - python .misc/check_setuptools.py # https://github.com/coala/coala/issues/3183 @@ -104,6 +123,7 @@ before_install: # For bears in .coafile requiring npm dependencies - npm install --no-save + - source ~/.install-jdk-travis.sh before_script: # Restore the original requirements.txt @@ -113,9 +133,9 @@ before_script: script: - py.test - python setup.py bdist_wheel - - pip install ./dist/coala-*.whl + - pip install ./dist/*.whl - curl -fsSL -o coala-bears.zip $BEARS_ZIP_URL/master - - pip install coala-bears.zip[alldeps] --pre -U + - pip install coala-bears.zip[alldeps] - coala --non-interactive - python setup.py docs - .ci/check_man.sh @@ -126,9 +146,6 @@ after_success: after_failure: - codecov -notifications: - email: false - branches: exclude: - /^sils\// diff --git a/coalib/bearlib/abstractions/Linter.py b/coalib/bearlib/abstractions/Linter.py index a22b801ad9..3a3c5c5db7 100644 --- a/coalib/bearlib/abstractions/Linter.py +++ b/coalib/bearlib/abstractions/Linter.py @@ -415,14 +415,22 @@ def process_output_corrected(self, An iterator returning results containing patches for the file to correct. """ - return self.process_diff( - Diff.from_string_arrays( - file, - output.splitlines(keepends=True)), - filename, - diff_severity, - result_message, - diff_distance) + if 'XML' not in result_message: + return self.process_diff( + Diff.from_string_arrays( + file, + output.splitlines(keepends=True)), + filename, + diff_severity, + result_message, + diff_distance) + else: + output = output.splitlines(keepends=True) + xml_diff = Diff.from_xml_diff(output, file) + yield Result(self, + result_message, + diffs={filename: xml_diff}, + severity=diff_severity) def process_output_unified_diff(self, output, diff --git a/coalib/processes/Processing.py b/coalib/processes/Processing.py index ee2b44fbab..6a3ddf5d40 100644 --- a/coalib/processes/Processing.py +++ b/coalib/processes/Processing.py @@ -421,8 +421,7 @@ def instantiate_processes(section, # run only for the changed files if caching is enabled. # Start tracking all the files - if cache and (loaded_valid_local_bears_count == loaded_local_bears_count - and not use_raw_files): + if cache and (loaded_valid_local_bears_count == loaded_local_bears_count): cache.track_files(set(complete_filename_list)) changed_files = cache.get_uncached_files( set(filename_list)) if cache else filename_list @@ -806,7 +805,7 @@ def execute_section(section, for runner in processes: runner.start() - + try: return (process_queues(processes, arg_dict['control_queue'], diff --git a/coalib/results/Diff.py b/coalib/results/Diff.py index e115c846a9..afb26133e2 100644 --- a/coalib/results/Diff.py +++ b/coalib/results/Diff.py @@ -1,8 +1,11 @@ +import os + import copy import difflib import logging from unidiff import PatchSet +from xmldiff import main, formatting from coalib.results.LineDiff import LineDiff, ConflictError from coalib.results.SourceRange import SourceRange @@ -155,6 +158,30 @@ def from_unified_diff(cls, unified_diff, original_file): return diff + @classmethod + def from_xml_diff(cls, new_file, original_file): + diff = Diff(original_file) + + #diff.xml_diff(new_file, original_file) + + formatter = formatting.XMLFormatter(normalize=formatting.WS_BOTH, + pretty_print=True) + new_content = "".join(new_file) + original_content = "".join(original_file) + current_dir = os.getcwd() + file1 = os.path.join(current_dir, 'file1') + file2 = os.path.join(current_dir, 'file2') + with open(file1, 'w') as f1: + f1.write(original_content) + with open(file2, 'w') as f2: + f2.write(new_content) + xml_diff = main.diff_files(file1, file2, + formatter=formatter) + diff._changes['diff'] = xml_diff + os.remove(file1) + os.remove(file2) + return diff + def _get_change(self, line_nr, min_line=1): if not isinstance(line_nr, int): raise TypeError('line_nr needs to be an integer.') @@ -284,6 +311,14 @@ def unified_diff(self): return ''.join(self._generate_linebreaks(list_unified_diff)) + @property + def xml_diff(self, filename, output_filename): + formatter = formatting.XMLFormatter(normalize=formatting.WS_BOTH, + pretty_print=True) + diff = main.diff_files('xml_file1.xml', 'xml_file2.xml', + formatter=formatter) + self._changes['diff'] = diff + def __json__(self): """ Override JSON export, using the unified diff is the easiest thing for diff --git a/requirements.txt b/requirements.txt index 1c103aae5f..9e0651691f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,6 @@ packaging>=16.8 Pygments~=2.1 PyPrint~=0.2.6 requests~=2.12 -setuptools>=21.0.0 +setuptools>=21.2.2 unidiff~=0.5.2 +git+https://github.com/Shoobx/xmldiff#egg=xmldiff