From 805f9a23111d16e7dfe070eedee6931922d2ad52 Mon Sep 17 00:00:00 2001 From: Yeison Vargas Date: Fri, 9 Sep 2022 17:16:53 -0500 Subject: [PATCH] Adding poetry parser and throwing exceptions on parsing --- dparse/dependencies.py | 48 ++++++++++++-- dparse/filetypes.py | 1 + dparse/parser.py | 144 ++++++++++++++++++++++++++++++----------- dparse/updater.py | 27 ++++---- tests/test_parse.py | 95 ++++++++++++++++++++++++++- 5 files changed, 258 insertions(+), 57 deletions(-) diff --git a/dparse/dependencies.py b/dparse/dependencies.py index b450fa9..9e8d8c1 100644 --- a/dparse/dependencies.py +++ b/dparse/dependencies.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals, absolute_import import json +from json import JSONEncoder from . import filetypes, errors @@ -11,7 +12,9 @@ class Dependency(object): """ - def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], line_numbers=None, index_server=None, hashes=(), dependency_type=None, section=None): + def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], + line_numbers=None, index_server=None, hashes=(), + dependency_type=None, section=None): """ :param name: @@ -87,12 +90,25 @@ def full_name(self): return self.name +class DparseJSONEncoder(JSONEncoder): + def default(self, o): + from packaging.specifiers import SpecifierSet + + if isinstance(o, SpecifierSet): + return str(o) + if isinstance(o, set): + return list(o) + + return JSONEncoder.default(self, o) + + class DependencyFile(object): """ """ - def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ()), parser=None): + def __init__(self, content, path=None, sha=None, file_type=None, + marker=((), ()), parser=None, resolve=False): """ :param content: @@ -130,6 +146,8 @@ def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ()) self.parser = parser_class.PipfileLockParser elif file_type == filetypes.setup_cfg: self.parser = parser_class.SetupCfgParser + elif file_type == filetypes.poetry_lock: + self.parser = parser_class.PoetryLockParser elif path is not None: if path.endswith((".txt", ".in")): @@ -144,11 +162,23 @@ def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ()) self.parser = parser_class.PipfileLockParser elif path.endswith("setup.cfg"): self.parser = parser_class.SetupCfgParser + elif path.endswith(filetypes.poetry_lock): + self.parser = parser_class.PoetryLockParser if not hasattr(self, "parser"): raise errors.UnknownDependencyFileError - self.parser = self.parser(self) + self.parser = self.parser(self, resolve=resolve) + + @property + def resolved_dependencies(self): + deps = self.dependencies.copy() + + for d in self.resolved_files: + if isinstance(d, DependencyFile): + deps.extend(d.resolved_dependencies) + + return deps def serialize(self): """ @@ -160,7 +190,9 @@ def serialize(self): "content": self.content, "path": self.path, "sha": self.sha, - "dependencies": [dep.serialize() for dep in self.dependencies] + "dependencies": [dep.serialize() for dep in self.dependencies], + "resolved_dependencies": [dep.serialize() for dep in + self.resolved_dependencies] } @classmethod @@ -170,7 +202,8 @@ def deserialize(cls, d): :param d: :return: """ - dependencies = [Dependency.deserialize(dep) for dep in d.pop("dependencies", [])] + dependencies = [Dependency.deserialize(dep) for dep in + d.pop("dependencies", [])] instance = cls(**d) instance.dependencies = dependencies return instance @@ -180,7 +213,7 @@ def json(self): # pragma: no cover :return: """ - return json.dumps(self.serialize(), indent=2) + return json.dumps(self.serialize(), indent=2, cls=DparseJSONEncoder) def parse(self): """ @@ -192,5 +225,6 @@ def parse(self): return self self.parser.parse() - self.is_valid = len(self.dependencies) > 0 or len(self.resolved_files) > 0 + self.is_valid = len(self.dependencies) > 0 or len( + self.resolved_files) > 0 return self diff --git a/dparse/filetypes.py b/dparse/filetypes.py index df8a913..eadfff4 100644 --- a/dparse/filetypes.py +++ b/dparse/filetypes.py @@ -7,3 +7,4 @@ tox_ini = "tox.ini" pipfile = "Pipfile" pipfile_lock = "Pipfile.lock" +poetry_lock = "poetry.lock" diff --git a/dparse/parser.py b/dparse/parser.py index eb62632..8036978 100644 --- a/dparse/parser.py +++ b/dparse/parser.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, absolute_import + +import os from collections import OrderedDict import re @@ -7,14 +9,16 @@ from configparser import ConfigParser, NoOptionError - +from .errors import MalformedDependencyFileError from .regex import HASH_REGEX from .dependencies import DependencyFile, Dependency -from packaging.requirements import Requirement as PackagingRequirement, InvalidRequirement +from packaging.requirements import Requirement as PackagingRequirement,\ + InvalidRequirement from . import filetypes import toml from packaging.specifiers import SpecifierSet +from packaging.version import Version, InvalidVersion import json @@ -22,23 +26,24 @@ def setuptools_parse_requirements_backport(strs): # pragma: no cover # Copyright (C) 2016 Jason R Coombs # - # Permission is hereby granted, free of charge, to any person obtaining a copy of - # this software and associated documentation files (the "Software"), to deal in - # the Software without restriction, including without limitation the rights to - # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies - # of the Software, and to permit persons to whom the Software is furnished to do - # so, subject to the following conditions: + # Permission is hereby granted, free of charge, to any person obtaining a + # copy of this software and associated documentation files + # (the "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to permit + # persons to whom the Software is furnished to do so, subject to the + # following conditions: # - # The above copyright notice and this permission notice shall be included in all - # copies or substantial portions of the Software. + # The above copyright notice and this permission notice shall be included + # in all copies or substantial portions of the Software. # - # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - # SOFTWARE. + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """Yield ``Requirement`` objects for each specification in `strs` `strs` must be a string, or a (possibly-nested) iterable thereof. @@ -82,9 +87,11 @@ def parse(cls, line): :return: """ try: - # setuptools requires a space before the comment. If this isn't the case, add it. + # setuptools requires a space before the comment. + # If this isn't the case, add it. if "\t#" in line: - parsed, = setuptools_parse_requirements_backport(line.replace("\t#", "\t #")) + parsed, = setuptools_parse_requirements_backport( + line.replace("\t#", "\t #")) else: parsed, = setuptools_parse_requirements_backport(line) except InvalidRequirement: @@ -104,13 +111,14 @@ class Parser(object): """ - def __init__(self, obj): + def __init__(self, obj, resolve=False): """ :param obj: """ self.obj = obj self._lines = None + self.resolve = resolve def iter_lines(self, lineno=0): """ @@ -175,7 +183,7 @@ def parse_index_server(cls, line): :param line: :return: """ - groups = re.split(pattern="[=\s]+", string=line.strip(), maxsplit=100) + groups = re.split(pattern=r"[=\s]+", string=line.strip(), maxsplit=100) if len(groups) >= 2: return groups[1] if groups[1].endswith("/") else groups[1] + "/" @@ -217,17 +225,37 @@ def parse(self): # comments are lines that start with # only continue if line.startswith('-i') or \ - line.startswith('--index-url') or \ - line.startswith('--extra-index-url'): + line.startswith('--index-url') or \ + line.startswith('--extra-index-url'): # this file is using a private index server, try to parse it index_server = self.parse_index_server(line) continue - elif self.obj.path and (line.startswith('-r') or line.startswith('--requirement')): - self.obj.resolved_files.append(self.resolve_file(self.obj.path, line)) + elif self.obj.path and \ + (line.startswith('-r') or + line.startswith('--requirement')): + + req_file_path = self.resolve_file(self.obj.path, line) + + if self.resolve and os.path.exists(req_file_path): + with open(req_file_path, 'r') as f: + content = f.read() + + dep_file = DependencyFile( + content=content, + path=req_file_path, + resolve=True + ) + dep_file.parse() + self.obj.resolved_files.append(dep_file) + else: + self.obj.resolved_files.append(req_file_path) + elif line.startswith('-f') or line.startswith('--find-links') or \ - line.startswith('--no-index') or line.startswith('--allow-external') or \ - line.startswith('--allow-unverified') or line.startswith('-Z') or \ - line.startswith('--always-unzip'): + line.startswith('--no-index') or \ + line.startswith('--allow-external') or \ + line.startswith('--allow-unverified') or \ + line.startswith('-Z') or \ + line.startswith('--always-unzip'): continue elif self.is_marked_line(line): continue @@ -240,7 +268,8 @@ def parse(self): if "\\" in line: parseable_line = line.replace("\\", "") for next_line in self.iter_lines(num + 1): - parseable_line += next_line.strip().replace("\\", "") + parseable_line += next_line.strip().replace("\\", + "") line += "\n" + next_line if "\\" in next_line: continue @@ -251,7 +280,8 @@ def parse(self): hashes = [] if "--hash" in parseable_line: - parseable_line, hashes = Parser.parse_hashes(parseable_line) + parseable_line, hashes = Parser.parse_hashes( + parseable_line) req = RequirementsTXTLineParser.parse(parseable_line) if req: @@ -304,7 +334,8 @@ def parse(self): import yaml try: data = yaml.safe_load(self.obj.content) - if data and 'dependencies' in data and isinstance(data['dependencies'], list): + if data and 'dependencies' in data and \ + isinstance(data['dependencies'], list): for dep in data['dependencies']: if isinstance(dep, dict) and 'pip' in dep: for n, line in enumerate(dep['pip']): @@ -344,7 +375,7 @@ def parse(self): section=package_type ) ) - except (toml.TomlDecodeError, IndexError) as e: + except (toml.TomlDecodeError, IndexError): pass @@ -375,8 +406,8 @@ def parse(self): section=package_type ) ) - except ValueError: - pass + except ValueError as e: + raise MalformedDependencyFileError(info=str(e)) class SetupCfgParser(Parser): @@ -406,7 +437,46 @@ def _parse_content(self, content): self.obj.dependencies.append(req) -def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None): +class PoetryLockParser(Parser): + + def parse(self): + """ + Parse a poetry.lock + """ + try: + data = toml.loads(self.obj.content, _dict=OrderedDict) + pkg_key = 'package' + if data: + try: + dependencies = data[pkg_key] + except KeyError: + raise KeyError( + "Poetry lock file is missing the package section") + + for dep in dependencies: + try: + name = dep['name'] + spec = f"=={Version(dep['version'])}" + section = dep['category'] + except KeyError: + raise KeyError("Malformed poetry lock file") + except InvalidVersion: + continue + + self.obj.dependencies.append( + Dependency( + name=name, specs=SpecifierSet(spec), + dependency_type=filetypes.poetry_lock, + line=''.join([name, spec]), + section=section + ) + ) + except (toml.TomlDecodeError, IndexError) as e: + raise MalformedDependencyFileError(info=str(e)) + + +def parse(content, file_type=None, path=None, sha=None, marker=((), ()), + parser=None, resolve=False): """ :param content: @@ -417,13 +487,15 @@ def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser= :param parser: :return: """ + dep_file = DependencyFile( content=content, path=path, sha=sha, marker=marker, file_type=file_type, - parser=parser + parser=parser, + resolve=resolve ) return dep_file.parse() diff --git a/dparse/updater.py b/dparse/updater.py index 77b5ae6..aa4aeb8 100644 --- a/dparse/updater.py +++ b/dparse/updater.py @@ -8,24 +8,26 @@ class RequirementsTXTUpdater(object): - SUB_REGEX = r"^{}(?=\s*\r?\n?$)" @classmethod def update(cls, content, dependency, version, spec="==", hashes=()): """ - Updates the requirement to the latest version for the given content and adds hashes - if neccessary. + Updates the requirement to the latest version for the given content + and adds hashes if necessary. :param content: str, content :return: str, updated content """ - new_line = "{name}{spec}{version}".format(name=dependency.full_name, spec=spec, version=version) + new_line = "{name}{spec}{version}".format(name=dependency.full_name, + spec=spec, version=version) appendix = '' # leave environment markers intact if ";" in dependency.line: - # condense multiline, split out the env marker, strip comments and --hashes - new_line += ";" + dependency.line.splitlines()[0].split(";", 1)[1] \ - .split("#")[0].split("--hash")[0].rstrip() + # condense multiline, split out the env marker, strip comments + # and --hashes + new_line += ";" + \ + dependency.line.splitlines()[0].split(";", 1)[1] \ + .split("#")[0].split("--hash")[0].rstrip() # add the comment if "#" in dependency.line: # split the line into parts: requirement and comment @@ -40,7 +42,8 @@ def update(cls, content, dependency, version, spec="==", hashes=()): else: break appendix += trailing_whitespace + "#" + comment - # if this is a hashed requirement, add a multiline break before the comment + # if this is a hashed requirement, add a multiline break before the + # comment if dependency.hashes and not new_line.endswith("\\"): new_line += " \\" # if this is a hashed requirement, add the hashes @@ -81,14 +84,16 @@ def update(cls, content, dependency, version, spec="==", hashes=()): for package_type in ['packages', 'dev-packages']: if package_type in data: if dependency.full_name in data[package_type]: - data[package_type][dependency.full_name] = "{spec}{version}".format( + data[package_type][ + dependency.full_name] = "{spec}{version}".format( spec=spec, version=version ) try: from pipenv.project import Project except ImportError: - raise ImportError("Updating a Pipfile requires the pipenv extra to be installed. Install it with " - "pip install dparse[pipenv]") + raise ImportError( + "Updating a Pipfile requires the pipenv extra to be installed." + " Install it with pip install dparse[pipenv]") pipfile = tempfile.NamedTemporaryFile(delete=False) p = Project(chdir=False) p.write_toml(data=data, path=pipfile.name) diff --git a/tests/test_parse.py b/tests/test_parse.py index 6e1e35b..2346a35 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,6 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import unicode_literals + +import sys + +from dparse.errors import MalformedDependencyFileError + """Tests for `dparse.parser`""" from dparse.parser import parse, Parser @@ -305,10 +310,94 @@ def test_pipfile_with_invalid_toml(): def test_pipfile_lock_with_invalid_json(): content = """{ - "_meta": + "_meta": "hash": { "sha256": "8b5635a4f7b069ae6661115b9eaa15466f7cd96794af5d131735a3638be101fb" }, }""" - dep_file = parse(content, file_type=filetypes.pipfile_lock) - assert not dep_file.dependencies + throw = None + + try: + dep_file = parse(content, file_type=filetypes.pipfile_lock) + except Exception as e: + throw = e + + assert isinstance(throw, MalformedDependencyFileError) + + +def test_poetry_lock(): + content = """ + [[package]] + name = "certifi" + version = "2022.6.15" + description = "Python package for providing Mozilla's CA Bundle." + category = "main" + optional = false + python-versions = ">=3.6" + + [[package]] + name = "attrs" + version = "22.1.0" + description = "Classes Without Boilerplate" + category = "dev" + optional = false + python-versions = ">=3.5" + + [package.extras] + dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] + docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] + tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] + tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] + + [metadata] + lock-version = "1.1" + python-versions = "^3.9" + content-hash = "96e49f07dcfd53e21489b9a7f3451d3b76515f33496173d989395e1898ae9a26" + + [metadata.files] + certifi = [] + attrs = [] + """ + + dep_file = parse(content, file_type=filetypes.poetry_lock) + + assert dep_file.dependencies[0].name == 'certifi' + assert dep_file.dependencies[0].specs == SpecifierSet('==2022.6.15') + assert dep_file.dependencies[0].dependency_type == 'poetry.lock' + assert dep_file.dependencies[0].section == 'main' + assert dep_file.dependencies[0].hashes == () + + assert dep_file.dependencies[1].name == 'attrs' + assert dep_file.dependencies[1].specs == SpecifierSet('==22.1.0') + assert dep_file.dependencies[1].dependency_type == 'poetry.lock' + assert dep_file.dependencies[1].section == 'dev' + assert dep_file.dependencies[1].hashes == () + + +def test_poetry_lock_with_invalid_toml(): + content = """ + [[package] + name = "certifi" + version = "2022.6.15" + description = "Python package for providing Mozilla's CA Bundle." + category = "main" + optional = false + python-versions = ">=3.6" + + [metadata] + lock-version = "1.1" + python-versions = "^3.9" + content-hash = "96e49f07dcfd53e21489b9a7f3451d3b76515f33496173d989395e1898ae9a26" + + [metadata.files] + certifi = [] + """ + + throw = None + + try: + dep_file = parse(content, file_type=filetypes.poetry_lock) + except Exception as e: + throw = e + + assert isinstance(throw, MalformedDependencyFileError)