Adding poetry parser and throwing exceptions on parsing

dparse/dependencies.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals, absolute_import
 
 import json
+from json import JSONEncoder
 
 from . import filetypes, errors
 
@@ -11,7 +12,9 @@ class Dependency(object):
 
     """
 
-    def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], line_numbers=None, index_server=None, hashes=(), dependency_type=None, section=None):
+    def __init__(self, name, specs, line, source="pypi", meta={}, extras=[],
+                 line_numbers=None, index_server=None, hashes=(),
+                 dependency_type=None, section=None):
         """
 
         :param name:
@@ -87,12 +90,25 @@ def full_name(self):
         return self.name
 
 
+class DparseJSONEncoder(JSONEncoder):
+    def default(self, o):
+        from packaging.specifiers import SpecifierSet
+
+        if isinstance(o, SpecifierSet):
+            return str(o)
+        if isinstance(o, set):
+            return list(o)
+
+        return JSONEncoder.default(self, o)
+
+
 class DependencyFile(object):
     """
 
     """
 
-    def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ()), parser=None):
+    def __init__(self, content, path=None, sha=None, file_type=None,
+                 marker=((), ()), parser=None, resolve=False):
         """
 
         :param content:
@@ -130,6 +146,8 @@ def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ())
                     self.parser = parser_class.PipfileLockParser
                 elif file_type == filetypes.setup_cfg:
                     self.parser = parser_class.SetupCfgParser
+                elif file_type == filetypes.poetry_lock:
+                    self.parser = parser_class.PoetryLockParser
 
             elif path is not None:
                 if path.endswith((".txt", ".in")):
@@ -144,11 +162,23 @@ def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ())
                     self.parser = parser_class.PipfileLockParser
                 elif path.endswith("setup.cfg"):
                     self.parser = parser_class.SetupCfgParser
+                elif path.endswith(filetypes.poetry_lock):
+                    self.parser = parser_class.PoetryLockParser
 
         if not hasattr(self, "parser"):
             raise errors.UnknownDependencyFileError
 
-        self.parser = self.parser(self)
+        self.parser = self.parser(self, resolve=resolve)
+
+    @property
+    def resolved_dependencies(self):
+        deps = self.dependencies.copy()
+
+        for d in self.resolved_files:
+            if isinstance(d, DependencyFile):
+                deps.extend(d.resolved_dependencies)
+
+        return deps
 
     def serialize(self):
         """
@@ -160,7 +190,9 @@ def serialize(self):
             "content": self.content,
             "path": self.path,
             "sha": self.sha,
-            "dependencies": [dep.serialize() for dep in self.dependencies]
+            "dependencies": [dep.serialize() for dep in self.dependencies],
+            "resolved_dependencies": [dep.serialize() for dep in
+                                      self.resolved_dependencies]
         }
 
     @classmethod
@@ -170,7 +202,8 @@ def deserialize(cls, d):
         :param d:
         :return:
         """
-        dependencies = [Dependency.deserialize(dep) for dep in d.pop("dependencies", [])]
+        dependencies = [Dependency.deserialize(dep) for dep in
+                        d.pop("dependencies", [])]
         instance = cls(**d)
         instance.dependencies = dependencies
         return instance
@@ -180,7 +213,7 @@ def json(self):  # pragma: no cover
 
         :return:
         """
-        return json.dumps(self.serialize(), indent=2)
+        return json.dumps(self.serialize(), indent=2, cls=DparseJSONEncoder)
 
     def parse(self):
         """
@@ -192,5 +225,6 @@ def parse(self):
             return self
         self.parser.parse()
 
-        self.is_valid = len(self.dependencies) > 0 or len(self.resolved_files) > 0
+        self.is_valid = len(self.dependencies) > 0 or len(
+            self.resolved_files) > 0
         return self

dparse/filetypes.py

@@ -7,3 +7,4 @@
 tox_ini = "tox.ini"
 pipfile = "Pipfile"
 pipfile_lock = "Pipfile.lock"
+poetry_lock = "poetry.lock"

dparse/parser.py

@@ -1,44 +1,49 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
+
+import os
 from collections import OrderedDict
 import re
 
 from io import StringIO
 
 from configparser import ConfigParser, NoOptionError
 
-
+from .errors import MalformedDependencyFileError
 from .regex import HASH_REGEX
 
 from .dependencies import DependencyFile, Dependency
-from packaging.requirements import Requirement as PackagingRequirement, InvalidRequirement
+from packaging.requirements import Requirement as PackagingRequirement,\
+    InvalidRequirement
 from . import filetypes
 import toml
 from packaging.specifiers import SpecifierSet
+from packaging.version import Version, InvalidVersion
 import json
 
 
 # this is a backport from setuptools 26.1
 def setuptools_parse_requirements_backport(strs):  # pragma: no cover
     # Copyright (C) 2016 Jason R Coombs <[email protected]>
     #
-    # Permission is hereby granted, free of charge, to any person obtaining a copy of
-    # this software and associated documentation files (the "Software"), to deal in
-    # the Software without restriction, including without limitation the rights to
-    # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-    # of the Software, and to permit persons to whom the Software is furnished to do
-    # so, subject to the following conditions:
+    # Permission is hereby granted, free of charge, to any person obtaining a
+    # copy of this software and associated documentation files
+    # (the "Software"), to deal in the Software without restriction, including
+    # without limitation the rights to use, copy, modify, merge, publish,
+    # distribute, sublicense, and/or sell copies of the Software, and to permit
+    # persons to whom the Software is furnished to do so, subject to the
+    # following conditions:
     #
-    # The above copyright notice and this permission notice shall be included in all
-    # copies or substantial portions of the Software.
+    # The above copyright notice and this permission notice shall be included
+    # in all copies or substantial portions of the Software.
     #
-    # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-    # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-    # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-    # SOFTWARE.
+    # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+    # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+    # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+    # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+    # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+    # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     """Yield ``Requirement`` objects for each specification in `strs`
 
     `strs` must be a string, or a (possibly-nested) iterable thereof.
@@ -82,9 +87,11 @@ def parse(cls, line):
         :return:
         """
         try:
-            # setuptools requires a space before the comment. If this isn't the case, add it.
+            # setuptools requires a space before the comment.
+            # If this isn't the case, add it.
             if "\t#" in line:
-                parsed, = setuptools_parse_requirements_backport(line.replace("\t#", "\t #"))
+                parsed, = setuptools_parse_requirements_backport(
+                    line.replace("\t#", "\t #"))
             else:
                 parsed, = setuptools_parse_requirements_backport(line)
         except InvalidRequirement:
@@ -104,13 +111,14 @@ class Parser(object):
 
     """
 
-    def __init__(self, obj):
+    def __init__(self, obj, resolve=False):
         """
 
         :param obj:
         """
         self.obj = obj
         self._lines = None
+        self.resolve = resolve
 
     def iter_lines(self, lineno=0):
         """
@@ -175,7 +183,7 @@ def parse_index_server(cls, line):
         :param line:
         :return:
         """
-        groups = re.split(pattern="[=\s]+", string=line.strip(), maxsplit=100)
+        groups = re.split(pattern=r"[=\s]+", string=line.strip(), maxsplit=100)
 
         if len(groups) >= 2:
             return groups[1] if groups[1].endswith("/") else groups[1] + "/"
@@ -217,17 +225,37 @@ def parse(self):
                 # comments are lines that start with # only
                 continue
             if line.startswith('-i') or \
-                line.startswith('--index-url') or \
-                line.startswith('--extra-index-url'):
+               line.startswith('--index-url') or \
+               line.startswith('--extra-index-url'):
                 # this file is using a private index server, try to parse it
                 index_server = self.parse_index_server(line)
                 continue
-            elif self.obj.path and (line.startswith('-r') or line.startswith('--requirement')):
-                self.obj.resolved_files.append(self.resolve_file(self.obj.path, line))
+            elif self.obj.path and \
+                    (line.startswith('-r') or
+                     line.startswith('--requirement')):
+
+                req_file_path = self.resolve_file(self.obj.path, line)
+
+                if self.resolve and os.path.exists(req_file_path):
+                    with open(req_file_path, 'r') as f:
+                        content = f.read()
+
+                        dep_file = DependencyFile(
+                            content=content,
+                            path=req_file_path,
+                            resolve=True
+                        )
+                        dep_file.parse()
+                        self.obj.resolved_files.append(dep_file)
+                else:
+                    self.obj.resolved_files.append(req_file_path)
+
             elif line.startswith('-f') or line.startswith('--find-links') or \
-                line.startswith('--no-index') or line.startswith('--allow-external') or \
-                line.startswith('--allow-unverified') or line.startswith('-Z') or \
-                line.startswith('--always-unzip'):
+                line.startswith('--no-index') or \
+                line.startswith('--allow-external') or \
+                line.startswith('--allow-unverified') or \
+                line.startswith('-Z') or \
+                    line.startswith('--always-unzip'):
                 continue
             elif self.is_marked_line(line):
                 continue
@@ -240,7 +268,8 @@ def parse(self):
                     if "\\" in line:
                         parseable_line = line.replace("\\", "")
                         for next_line in self.iter_lines(num + 1):
-                            parseable_line += next_line.strip().replace("\\", "")
+                            parseable_line += next_line.strip().replace("\\",
+                                                                        "")
                             line += "\n" + next_line
                             if "\\" in next_line:
                                 continue
@@ -251,7 +280,8 @@ def parse(self):
 
                     hashes = []
                     if "--hash" in parseable_line:
-                        parseable_line, hashes = Parser.parse_hashes(parseable_line)
+                        parseable_line, hashes = Parser.parse_hashes(
+                            parseable_line)
 
                     req = RequirementsTXTLineParser.parse(parseable_line)
                     if req:
@@ -304,7 +334,8 @@ def parse(self):
         import yaml
         try:
             data = yaml.safe_load(self.obj.content)
-            if data and 'dependencies' in data and isinstance(data['dependencies'], list):
+            if data and 'dependencies' in data and \
+                    isinstance(data['dependencies'], list):
                 for dep in data['dependencies']:
                     if isinstance(dep, dict) and 'pip' in dep:
                         for n, line in enumerate(dep['pip']):
@@ -344,7 +375,7 @@ def parse(self):
                                     section=package_type
                                 )
                             )
-        except (toml.TomlDecodeError, IndexError) as e:
+        except (toml.TomlDecodeError, IndexError):
             pass
 
 
@@ -375,8 +406,8 @@ def parse(self):
                                     section=package_type
                                 )
                             )
-        except ValueError:
-            pass
+        except ValueError as e:
+            raise MalformedDependencyFileError(info=str(e))
 
 
 class SetupCfgParser(Parser):
@@ -406,7 +437,46 @@ def _parse_content(self, content):
                     self.obj.dependencies.append(req)
 
 
-def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None):
+class PoetryLockParser(Parser):
+
+    def parse(self):
+        """
+        Parse a poetry.lock
+        """
+        try:
+            data = toml.loads(self.obj.content, _dict=OrderedDict)
+            pkg_key = 'package'
+            if data:
+                try:
+                    dependencies = data[pkg_key]
+                except KeyError:
+                    raise KeyError(
+                        "Poetry lock file is missing the package section")
+
+                for dep in dependencies:
+                    try:
+                        name = dep['name']
+                        spec = f"=={Version(dep['version'])}"
+                        section = dep['category']
+                    except KeyError:
+                        raise KeyError("Malformed poetry lock file")
+                    except InvalidVersion:
+                        continue
+
+                    self.obj.dependencies.append(
+                        Dependency(
+                            name=name, specs=SpecifierSet(spec),
+                            dependency_type=filetypes.poetry_lock,
+                            line=''.join([name, spec]),
+                            section=section
+                        )
+                    )
+        except (toml.TomlDecodeError, IndexError) as e:
+            raise MalformedDependencyFileError(info=str(e))
+
+
+def parse(content, file_type=None, path=None, sha=None, marker=((), ()),
+          parser=None, resolve=False):
     """
 
     :param content:
@@ -417,13 +487,15 @@ def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=
     :param parser:
     :return:
     """
+
     dep_file = DependencyFile(
         content=content,
         path=path,
         sha=sha,
         marker=marker,
         file_type=file_type,
-        parser=parser
+        parser=parser,
+        resolve=resolve
     )
 
     return dep_file.parse()