Add requirement parsing module.

src/pip/_internal/req/parsing.py

@@ -0,0 +1,183 @@
+import os.path
+from contextlib import contextmanager
+
+from pip._vendor.packaging.requirements import Requirement
+
+from pip._internal.models.link import Link
+from pip._internal.req.constructors import _strip_extras
+from pip._internal.utils.misc import path_to_url
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Optional, Set, Tuple
+
+    from pip._vendor.packaging.markers import Marker
+
+
+__all__ = [
+    'RequirementInfo',
+    'RequirementParsingError',
+    'parse_requirement_text',
+]
+
+
+PATH_MARKER_SEP = ';'
+URL_MARKER_SEP = '; '
+
+
+def convert_extras(extras):
+    # type: (Optional[str]) -> Set[str]
+    if extras:
+        return Requirement("placeholder" + extras.lower()).extras
+    else:
+        return set()
+
+
+def strip_and_convert_extras(text):
+    # type: (str) -> Tuple[str, Set[str]]
+    result, extras = _strip_extras(text)
+    return result, convert_extras(extras)
+
+
+class RequirementParsingError(Exception):
+    def __init__(self, type_tried, cause):
+        # type: (str, Exception) -> None
+        self.type_tried = type_tried
+        self.cause = cause
+
+
+class RequirementInfo(object):
+    def __init__(
+        self,
+        requirement,  # type: Optional[Requirement]
+        link,         # type: Optional[Link]
+        markers,      # type: Optional[Marker]
+        extras,       # type: Set[str]
+    ):
+        self.requirement = requirement
+        self.link = link
+        self.markers = markers
+        self.extras = extras
+
+    @property
+    def is_unnamed(self):
+        return self.requirement is None
+
+    @property
+    def is_name_based(self):
+        return self.link is None
+
+    def __repr__(self):
+        return '<RequirementInfo({!r}, {!r}, {!r}, {!r})>'.format(
+            self.requirement, self.link, self.markers, self.extras,
+        )
+
+
+def requirement_info_from_requirement(text):
+    # type: (str) -> RequirementInfo
+    req = Requirement(text)
+    return RequirementInfo(
+        req,
+        Link(req.url) if req.url else None,
+        req.marker,
+        req.extras,
+    )
+
+
+def requirement_info_from_url(url):
+    # type: (str) -> RequirementInfo
+    try:
+        url, marker_text = url.split(URL_MARKER_SEP)
+    except ValueError:
+        marker = None
+    else:
+        marker = Marker(marker_text)
+
+    # Also works when egg=example[extra1] is at the end.
+    url, extras = strip_and_convert_extras(url)
+
+    link = Link(url)
+
+    egg_fragment = link.egg_fragment
+
+    req = None  # type: Optional[Requirement]
+    if egg_fragment:
+        req = Requirement(egg_fragment)
+        # We prefer fragment extras if present.
+        if req.extras:
+            extras = req.extras
+
+    return RequirementInfo(req, link, marker, extras)
+
+
+def requirement_info_from_path(path):
+    # type: (str) -> RequirementInfo
+    try:
+        path, markers = path.split(PATH_MARKER_SEP)
+    except ValueError:
+        markers = ''
+    else:
+        markers = URL_MARKER_SEP + markers
+
+    path, extras = _strip_extras(path)
+    if extras is None:
+        extras = ''
+
+    url = path_to_url(path)
+
+    return requirement_info_from_url(
+        '{}{}{}'.format(url, extras, markers),
+    )
+
+
+def looks_like_direct_reference(text):
+    try:
+        assert text.index('@') < text.index('://')
+    except (AssertionError, ValueError):
+        return False
+    else:
+        return True
+
+
+def looks_like_url(text):
+    # type: (str) -> bool
+    return '://' in text
+
+
+def looks_like_path(text):
+    # type: (str) -> bool
+    return (
+        os.path.sep in text or
+        os.path.altsep is not None and os.path.altsep in text or
+        text.startswith('.')
+    )
+
+
+@contextmanager
+def try_parse_as(message):
+    try:
+        yield
+    except Exception as e:
+        raise RequirementParsingError(message, e)
+
+
+def parse_requirement_text(text):
+    # type: (str) -> RequirementInfo
+    # Only search before any ';', since marker strings can
+    # contain most kinds of text.
+    search_text = text.split(';', 1)[0]
+
+    if looks_like_direct_reference(search_text):
+        with try_parse_as('direct reference'):
+            return requirement_info_from_requirement(text)
+
+    if looks_like_url(search_text):
+        with try_parse_as('url'):
+            return requirement_info_from_url(text)
+
+    if looks_like_path(search_text):
+        with try_parse_as('path'):
+            return requirement_info_from_path(text)
+
+    with try_parse_as('name-based reference'):
+        return requirement_info_from_requirement(text)

tests/unit/test_req_parsing.py

@@ -0,0 +1,150 @@
+import pytest
+from pip._vendor.packaging.markers import Marker
+from pip._vendor.packaging.requirements import Requirement
+
+from pip._internal.models.link import Link
+from pip._internal.req.parsing import (
+    RequirementInfo,
+    RequirementParsingError,
+    convert_extras,
+    looks_like_direct_reference,
+    looks_like_path,
+    looks_like_url,
+    parse_requirement_text,
+)
+from pip._internal.utils.misc import path_to_url
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Optional, Set, Union
+
+
+@pytest.mark.parametrize('text,expected', [
+    ('[extra]', {'extra'}),
+    ('[extra1,extra2]', {'extra1', 'extra2'}),
+    ('', set()),
+    (None, set()),
+])
+def test_convert_extras(text, expected):
+    assert expected == convert_extras(text)
+
+
+@pytest.mark.parametrize('text,expected', [
+    ('example @ file://.', True),
+    ('example@ https://.', True),
+    ('https://.', False),
+    ('https://git.example.com/repo@commit', False),
+    ('https://user:[email protected]/repo', False),
+])
+def test_looks_like_direct_reference(text, expected):
+    assert expected == looks_like_direct_reference(text)
+
+
+@pytest.mark.parametrize('text,expected', [
+    ('/example/hello', True),
+    ('hello', False),
+    ('/example[extra]', True),
+    ('.', True),
+    ('.[extra]', True),
+])
+def test_looks_like_path(text, expected):
+    assert expected == looks_like_path(text)
+
+
+@pytest.mark.skipif('sys.platform != "win32"')
+@pytest.mark.parametrize('text,expected', [
+    ('C:\\Example', True),
+    ('.\\', True),
+])
+def test_looks_like_path_windows(text, expected):
+    assert expected == looks_like_path(expected)
+
+
+@pytest.mark.parametrize('text,expected', [
+    ('http://example.com/', True),
+    ('./example', False),
+])
+def test_looks_like_url(text, expected):
+    assert expected == looks_like_url(text)
+
+
+def _assert_requirement(expected, test):
+    # type: (Optional[Requirement], Optional[Requirement]) -> None
+    if expected is None or test is None:
+        assert expected == test
+        return
+
+    assert expected.name == test.name
+    assert expected.specifier == test.specifier
+    assert expected.url == test.url
+    assert expected.extras == test.extras
+    assert expected.marker == test.marker
+
+
+def _assert_requirement_info(
+    expected, test,
+):
+    # type: (RequirementInfo, RequirementInfo) -> None
+    _assert_requirement(expected.requirement, test.requirement)
+    assert (expected.link is None) == (test.link is None)
+    if expected.link is not None:
+        assert expected.link.url == test.link.url
+    assert expected.markers == test.markers
+    assert expected.extras == test.extras
+
+
+INPUT = object()
+
+
+def req_info(
+    req=None,  # type: Optional[Union[str, object]]
+    link=None,  # type: Optional[Union[str, object]]
+    markers=None,  # type: Optional[str]
+    extras=None,  # type: Optional[Set[str]]
+):
+    def get_req_info(text):
+        _req = req
+        if _req is INPUT:
+            _req = text
+        _link = link
+        if _link is INPUT:
+            _link = text
+
+        return RequirementInfo(
+            Requirement(_req) if _req else None,
+            Link(_link) if _link else None,
+            Marker(markers) if markers else None,
+            extras,
+        )
+
+    return get_req_info
+
+
+@pytest.mark.parametrize('text,make_expected', [
+    ('.', req_info(link=path_to_url('.'))),
+    ('.[extra1]', req_info(link=path_to_url('.'), extras={'extra1'})),
+    ('path/to/project', req_info(link=path_to_url('path/to/project'))),
+
+    ('pkg[extra1]', req_info(req=INPUT, extras={'extra1'})),
+    ('pkg[extra1]', req_info(req=INPUT, extras={'extra1'})),
+
+    ('git+http://git.example.com/pkgrepo#egg=pkg',
+     req_info(req='pkg', link=INPUT)),
+    ('git+http://git.example.com/pkgrepo#egg=pkg[extra1]',
+     req_info(req='pkg[extra1]', link=INPUT, extras={'extra1'})),
+])
+def test_parse_requirement_text(text, make_expected):
+    _assert_requirement_info(
+        parse_requirement_text(text), make_expected(text),
+    )
+
+
+@pytest.mark.parametrize('text,expected_message', [
+    ('file:.', 'name-based'),
+    ('http://@@%bad:url', 'url'),
+    ('@ http://example', 'direct reference'),
+])
+def test_parse_requirement_text_fail(text, expected_message):
+    with pytest.raises(RequirementParsingError) as e:
+        parse_requirement_text(text)
+    assert expected_message in e.value.type_tried