diff --git a/cEP-0031.md b/cEP-0031.md new file mode 100644 index 000000000..7426788cf --- /dev/null +++ b/cEP-0031.md @@ -0,0 +1,358 @@ + # Improve Generic Bear Quality + +| Metadata | | +| -------- | ------------------------------------------- | +| cEP | 31 | +| Version | 0.1 | +| Title | Improve Generic Bear Quality | +| Authors | Bhushan Khanale | +| Status | Proposed | +| Type | Feature | + +## Abstract + +This cEP describes the improvement in the generic bears currently available +along with the implementation of the new bears as a part of the +[GSoC 2019 Project][project]. + +## Introduction + +coala has few generic bears which have the potential to perform better +fixing some of the issues in them. Some of these issues include [#2092][2092], +[#644][644], [#502][502], [#1897][1897], [#227][227]. +Along with fixing these issues, new bears including `OutdatedDependencyBear`, +`FileExistsBear`, `RegexLintBear`, `FileModeBear` and `RequirementsCheckBear` +will be implemented. + +This project also introduces changes in order to interlink `PEP8Bear` and +`PycodestyleBear` as a part of the [#551][551] proposal. + +## Implementation of new bears + +### 1. OutdatedDependencyBear + +Issue: [coala/coala-bears#2445][2445] + +New functions to the existing package manager would be the best way of +implementing this. + +For pip requirements, we could use the PyPI JSON API which provides the +version info by specifying the `package_name`. + +```py +class PipRequirement(PackageRequirement): + def get_version_info(self): + # This would get the package info in json format + url = 'https://pypi.python.org/pypi/%s/json' % (self.package,) + + # Get the required data + data = json.load() + versions = data['releases'].keys().sort(key = StrictVersion) + return versions +``` + +For npm dependencies, we can use the npm cli command `npm outdated` which +produces an output as below: + +``` +$ npm outdated +Package Current Wanted Latest Location +glob 5.0.15 5.0.15 6.0.1 test-outdated-output +nothingness 0.0.3 git git test-outdated-output +npm 3.5.1 3.5.2 3.5.1 test-outdated-output +local-dev 0.0.3 linked linked test-outdated-output +once 1.3.2 1.3.3 1.3.3 test-outdated-output +``` + +The output can be directly parsed and can be used to get the list of all +outdated dependencies. + +### 2. FileExistBear + +Issue: [coala/coala-bears#527][527] + +The bear will be responsible to check if the files exist in the project +directory. The name of the files can be passed as a regex to the bear, to +which the bear will check their existence. + +```py +class FileExistBear(LocalBear): + def run(self, required_files: tuple,): + for file in required_files: + if not os.path.is_file(): + message = 'File {} not found.'.format(file) + yield Result.from_values(origin=self, message=message,) +``` + +### 3. RegexLintBear + +Issue: [coala/coala-bears#1532][1532] + +The task of the bear is to check the regex in strings. This can be done using +the AnnotationBear to detect all the regex strings and then check for the valid +regex through the prepared algorithm for each type. + +For Python, the bear can be written as follows: + +```py +def is_regex(code, language): + """ + Returns true if the string is a regex. + """ + +class RegexBear(LocalBear): + def run(self, file, filename, language,) + strings = run_local_bear(AnnotationBear) + for string in strings: + if is_regex(string, language): + try: + re.compile(string) + except re.error as Exception: + message = 'This is not a valid regex.' + yield Result.from_values(origin=self, message=message) +``` + +### 4. FileModeBear + +Issue: [coala/coala-bears#2370][2370] + +The bear will check the permissions on the files provided by the user +and to ensure that the file permissions are the one that is expected. + +The bear will be used as follows: + +``` +[all.mode] +bears = FileModeBear +filemode = rw + +[all.shell.mode] +bears = FileModeBear +filemode = rwx +``` + +The bear will first find the permissions of the files specified by the user and +then if the permissions are not suitable the bear will try to change those +permissions into the expected ones. If the bear doesn't have enough permissions +to do so then the bear will let the user know about this. + +```py +class FileModeBear(LocalBear): + def run(self, file, filename, filemode,): + st = os.stat(filename) + st_read = bool(st.st_mode & stat.S_IRUSR) + st_write = bool(st.st_mode & stat.S_IWUSR) + + mode = {'r': st_read, 'w': st_write} + ch_mode = {'r': stat.S_IRUSR, 'w': stat.S_IWUSR} + + # Similarly, by bitwise operations on the st_mode, the relevant + # information can be found out. + + change_mode = 0 + for char in filemode: + if not mode[char]: + change_mode += ch_mode[char] + if not change_mode: + os.chmod(change_mode) +``` + +### 5. RequirementsCheckBear + +Issue: [coala/coala-bears#1113][1113] + +The bear will be focused on Python only since they are most prone for +conflicting requirements. + +The implementation is based on the recursive check for the requirements of each +package in requirement.txt file of the project. The requirements are checked +through PyPI's JSON API. + +```py +class RequirementsCheckBear(LocalBear): + def collect_requirements(): + """ + This will get the requirements through the requirements.txt file and + through requirements folder and pip files. + """ + packages = get_packages() + + for package in packages: + requirements = get_requirements(package) + packages.append(r for r in requirements) + + return packages + + def get_requirements(package_name): + """ + This will get the requirements of a package through the PyPI's JSON API. + """ + url = 'https://pypi.python.org/pypi/%s/json' % (package_name,) + + data = requests.get(url).json() + + return data['info']['requires_dist'] + + def run(self, file, filaname,): + packages = get_requirements() + conflicts = detect_conflicts() + if conflicts: + for conflict in conflicts: + message = '{} is conflicting package.'.format(conflict) + yield Result.from_values(origin=self, message=message,) +``` + +## Issues to be solved + +### Pycodestyle - PEP8Bear Integration + +Issues covered: + +- [coala/coala-bears#1897][1897] +- [coala/projects#551][551] +- [hhatto/autopep8#227][227] + +Since the autopep8 upstream is not likely to have any major improvements, +we could have the improvements implemented to the PEP8Bear. Since autopep8 uses +Pycodestyle for error detection, we could integrate the Pycodestyle with +PEP8Bear. + +Doing this will involve invoking PEP8Bear after running PycodestyleBear. + +To implement this, we should be able to pass the selection of the issues +through the `--line-range` and `--select` options available in autopep8. +If multiple similar issues are available in the same line-range then we should +implement the `--column-range` option in autopep8 which will allow us to +pass the appropriate issue to generate a fix for. + +This will allow the user to generate a fix for only selected issues. + +```py +import autopep8 +import logging + +from bears.python.PycodestyleBear import PycodestyleBear +from coalib.bearlib.spacing.SpacingHelper import SpacingHelper +from coalib.bears.LocalBear import LocalBear +from coalib.results.Diff import Diff +from coalib.results.Result import Result +from coalib.settings.Section import Section +from coalib.settings.Setting import Setting +from coalib.testing.LocalBearTestHelper import execute_bear +from queue import Queue + + +class PyPEP8Bear(LocalBear): + """ + Integration of PycodestyleBear and PEP8Bear. + """ + LANGUAGES = {'Python', 'Python 2', 'Python 3'} + AUTHORS = {'The coala developers'} + AUTHORS_EMAILS = {'coala-devel@googlegroups.com'} + LICENSE = 'AGPL-3.0' + CAN_FIX = {'Formatting'} + + def run(self, filename, file, + max_line_length: int = 79, + indent_size: int = SpacingHelper.DEFAULT_TAB_WIDTH,): + """ + Detects and fixes PEP8 incompliant code. This bear will not change + functionality of the code in any way. + + :param max_line_length: + Maximum number of characters for a line. + When set to 0 allows infinite line length. + :param indent_size: + Number of spaces per indentation level. + """ + if not max_line_length: + max_line_length = sys.maxsize + + queue = Queue() + section = Section('') + section.append(Setting('max_line_length', max_line_length)) + + with execute_bear(PycodestyleBear(section, queue), + filename, file,) as results: + + for result in results: + start_line = result.affected_code[0].start.line + end_line = result.affected_code[0].end.line + # start_column = result.affected_code[0].start.column + # end_column = result.affected_code[0].end.column + rule = result.origin.split(' ')[1][1:-1] + + options = {'select': (rule,), + 'max_line_length': max_line_length, + 'indent_size': indent_size, + 'line_range': [start_line, end_line + 1]} + + corrected = autopep8.fix_code(''.join(file), + apply_config=False, + options=options).splitlines(True) + + diffs = Diff.from_string_arrays(file, corrected).split_diff() + + logging.debug(corrected) + + for diff in diffs: + yield Result(self, + result.message, + affected_code=(diff.range(filename),), + diffs={filename: diff}) +``` + +The output of the bear correctly displays the error messages: + +``` +test_bear.py +[ 4] print(·'·Hello·world'·) +**** PyPEP8Bear [Section: cli | Severity: NORMAL] **** +! ! E201 whitespace after '(' +[----] /home/bkhanale/gsoc/playground/test_bear.py +[++++] /home/bkhanale/gsoc/playground/test_bear.py +[ 4] print( ' Hello world' ) +[ 4] print(' Hello world' ) +[ ] *0. Do (N)othing +[ ] 1. (O)pen file +[ ] 2. (A)pply patch +[ ] 3. Add (I)gnore comment +[ ] 4. Show Applied (P)atches +[ ] 5. (G)enerate patches +[ ] Enter number (Ctrl-D to exit): +``` + +### MarkdownBear: Show error messages + +MarkdownBear doesn't show the error messages while running it. Solving the issue +will increase the usability of the MarkdownBear. Although remark doesn't give a +give the issues right away along with the patches, we can get them through the +plugins. [preset-lint-markdown-style-guide][guide] is one such guide for this +purpose. + +The output of the reamrk is as follows using the guide: + +``` +somefile.md + 5:5 warning Incorrect list-item indent: remove 1 space list-item-indent remark-lint + 6:1-6:76 warning Marker should be `1`, was `2` ordered-list-marker-value remark-lint + 6:5 warning Incorrect list-item indent: remove 1 space list-item-indent remark-lint + +⚠ 3 warnings +``` + +This can be parsed via a regex and can be provided to the user through the +line-range. + +[guide]: https://github.com/remarkjs/remark-lint/tree/master/packages/remark-preset-lint-markdown-style-guide +[project]: https://summerofcode.withgoogle.com/projects/#4866569388163072 +[227]: https://github.com/hhatto/autopep8/issues/227 +[502]: https://github.com/coala/coala-bears/issues/502 +[527]: https://github.com/coala/coala-bears/issues/527 +[551]: https://github.com/coala/projects/issues/551 +[644]: https://github.com/coala/coala-bears/issues/644 +[1113]: https://github.com/coala/coala-bears/issues/1113 +[1532]: https://github.com/coala/coala-bears/issues/1532 +[1897]: https://github.com/coala/coala-bears/issues/1897 +[2092]: https://github.com/coala/coala-bears/issues/2092 +[2445]: https://github.com/coala/coala-bears/issues/2445