From baef2e78a632168bddce0a428969b9f6fb32693c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 19 Oct 2018 20:57:20 -0400 Subject: [PATCH 01/13] Upgrade vendored dependencies - Upgrade pythonfinder - Upgrade vistir - Upgrade requirementslib - Vendor backported version of `functools.lru_cache` for performance - Fix editable dependency installation when markers are present - Fix extraneous resource warnings - Fix filesystem output stream encoding issues - Fix pythonfinder non-standard python name issues - Provide full interaction layer to `Pipfile` and `Pipfile.lock` in requirementslib - Fixes #3017 - Fixes #3014 - Fixes #3021 - Fixes #3019 Signed-off-by: Dan Ryan Update vendored dependencies - Update shellingham, tomlkit, requests, urllib3, certifi, vistir and parse - Fixes #2820 - Fixes #2852 Signed-off-by: Dan Ryan Cleanup old vendored dependencies Signed-off-by: Dan Ryan --- pipenv/environments.py | 17 +- pipenv/resolver.py | 1 - .../backports/functools_lru_cache.LICENSE | 7 + .../vendor/backports/functools_lru_cache.py | 184 +++ pipenv/vendor/certifi/__init__.py | 2 +- pipenv/vendor/certifi/cacert.pem | 30 - pipenv/vendor/cursor/LICENSE | 5 + pipenv/vendor/cursor/__init__.py | 4 + pipenv/vendor/cursor/cursor.py | 57 + pipenv/vendor/parse.py | 33 +- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/cli.py | 18 +- pipenv/vendor/pythonfinder/environment.py | 3 + pipenv/vendor/pythonfinder/models/mixins.py | 38 +- pipenv/vendor/pythonfinder/models/path.py | 279 +++-- pipenv/vendor/pythonfinder/models/pyenv.py | 204 +++- pipenv/vendor/pythonfinder/models/python.py | 77 +- pipenv/vendor/pythonfinder/models/windows.py | 29 +- pipenv/vendor/pythonfinder/pythonfinder.py | 34 +- pipenv/vendor/pythonfinder/utils.py | 40 +- pipenv/vendor/requirementslib/__init__.py | 2 +- .../vendor/requirementslib/models/lockfile.py | 160 ++- .../vendor/requirementslib/models/pipfile.py | 146 ++- .../vendor/requirementslib/models/project.py | 241 ++++ .../requirementslib/models/requirements.py | 27 +- pipenv/vendor/requirementslib/models/utils.py | 12 + pipenv/vendor/shellingham/__init__.py | 2 +- pipenv/vendor/shellingham/posix/ps.py | 12 +- pipenv/vendor/toml.LICENSE | 26 - pipenv/vendor/toml.py | 1039 ----------------- pipenv/vendor/tomlkit/__init__.py | 2 +- pipenv/vendor/tomlkit/container.py | 19 +- pipenv/vendor/tomlkit/exceptions.py | 2 +- pipenv/vendor/tomlkit/items.py | 129 +- pipenv/vendor/tomlkit/parser.py | 229 ++-- pipenv/vendor/tomlkit/toml_char.py | 12 + pipenv/vendor/urllib3/__init__.py | 9 +- pipenv/vendor/urllib3/_collections.py | 5 +- pipenv/vendor/urllib3/connection.py | 40 +- pipenv/vendor/urllib3/connectionpool.py | 18 +- pipenv/vendor/urllib3/contrib/__init__.py | 0 .../urllib3/contrib/_appengine_environ.py | 30 + .../contrib/_securetransport/__init__.py | 0 .../contrib/_securetransport/bindings.py | 0 .../contrib/_securetransport/low_level.py | 0 pipenv/vendor/urllib3/contrib/appengine.py | 36 +- pipenv/vendor/urllib3/contrib/ntlmpool.py | 3 +- pipenv/vendor/urllib3/contrib/pyopenssl.py | 25 +- .../vendor/urllib3/contrib/securetransport.py | 0 pipenv/vendor/urllib3/contrib/socks.py | 0 pipenv/vendor/urllib3/exceptions.py | 0 pipenv/vendor/urllib3/fields.py | 0 pipenv/vendor/urllib3/filepost.py | 0 pipenv/vendor/urllib3/packages/__init__.py | 0 .../urllib3/packages/backports/__init__.py | 0 .../urllib3/packages/backports/makefile.py | 2 +- .../vendor/urllib3/packages/ordered_dict.py | 259 ---- pipenv/vendor/urllib3/packages/six.py | 0 .../packages/ssl_match_hostname/__init__.py | 0 .../ssl_match_hostname/_implementation.py | 3 +- pipenv/vendor/urllib3/poolmanager.py | 1 + pipenv/vendor/urllib3/request.py | 2 +- pipenv/vendor/urllib3/response.py | 41 +- pipenv/vendor/urllib3/util/__init__.py | 0 pipenv/vendor/urllib3/util/connection.py | 8 + pipenv/vendor/urllib3/util/queue.py | 0 pipenv/vendor/urllib3/util/request.py | 0 pipenv/vendor/urllib3/util/response.py | 10 +- pipenv/vendor/urllib3/util/retry.py | 2 +- pipenv/vendor/urllib3/util/ssl_.py | 33 +- pipenv/vendor/urllib3/util/timeout.py | 0 pipenv/vendor/urllib3/util/url.py | 0 pipenv/vendor/urllib3/util/wait.py | 5 +- pipenv/vendor/vendor.txt | 18 +- pipenv/vendor/vistir/__init__.py | 10 +- pipenv/vendor/vistir/compat.py | 18 +- pipenv/vendor/vistir/contextmanagers.py | 70 +- pipenv/vendor/vistir/misc.py | 129 +- pipenv/vendor/vistir/path.py | 77 +- pipenv/vendor/vistir/spin.py | 149 +++ pipenv/vendor/vistir/termcolors.py | 116 ++ 81 files changed, 2232 insertions(+), 2011 deletions(-) create mode 100644 pipenv/vendor/backports/functools_lru_cache.LICENSE create mode 100644 pipenv/vendor/backports/functools_lru_cache.py create mode 100644 pipenv/vendor/cursor/LICENSE create mode 100644 pipenv/vendor/cursor/__init__.py create mode 100644 pipenv/vendor/cursor/cursor.py create mode 100644 pipenv/vendor/requirementslib/models/project.py delete mode 100644 pipenv/vendor/toml.LICENSE delete mode 100644 pipenv/vendor/toml.py mode change 100755 => 100644 pipenv/vendor/urllib3/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/_collections.py mode change 100755 => 100644 pipenv/vendor/urllib3/connection.py mode change 100755 => 100644 pipenv/vendor/urllib3/connectionpool.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/__init__.py create mode 100644 pipenv/vendor/urllib3/contrib/_appengine_environ.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/_securetransport/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/_securetransport/bindings.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/_securetransport/low_level.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/appengine.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/ntlmpool.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/pyopenssl.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/securetransport.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/socks.py mode change 100755 => 100644 pipenv/vendor/urllib3/exceptions.py mode change 100755 => 100644 pipenv/vendor/urllib3/fields.py mode change 100755 => 100644 pipenv/vendor/urllib3/filepost.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/backports/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/backports/makefile.py delete mode 100755 pipenv/vendor/urllib3/packages/ordered_dict.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/six.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py mode change 100755 => 100644 pipenv/vendor/urllib3/poolmanager.py mode change 100755 => 100644 pipenv/vendor/urllib3/request.py mode change 100755 => 100644 pipenv/vendor/urllib3/response.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/connection.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/queue.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/request.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/response.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/retry.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/ssl_.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/timeout.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/url.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/wait.py create mode 100644 pipenv/vendor/vistir/spin.py create mode 100644 pipenv/vendor/vistir/termcolors.py diff --git a/pipenv/environments.py b/pipenv/environments.py index 4bfd937d88..e6de4f0d73 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -8,6 +8,8 @@ # I hope I can remove this one day. os.environ["PYTHONDONTWRITEBYTECODE"] = fs_str("1") +PIPENV_IS_CI = bool("CI" in os.environ or "TF_BUILD" in os.environ) + # HACK: Prevent invalid shebangs with Homebrew-installed Python: # https://bugs.python.org/issue22490 os.environ.pop("__PYVENV_LAUNCHER__", None) @@ -68,7 +70,7 @@ Default is to show emojis. This is automatically set on Windows. """ -if os.name == "nt": +if os.name == "nt" or PIPENV_IS_CI: PIPENV_HIDE_EMOJIS = True PIPENV_IGNORE_VIRTUALENVS = bool(os.environ.get("PIPENV_IGNORE_VIRTUALENVS")) @@ -94,7 +96,7 @@ PIPENV_MAX_RETRIES = int(os.environ.get( "PIPENV_MAX_RETRIES", - "1" if "CI" in os.environ else "0", + "1" if PIPENV_IS_CI else "0", )) """Specify how many retries Pipenv should attempt for network requests. @@ -128,9 +130,18 @@ This can make the logs cleaner. Automatically set on Windows, and in CI environments. """ -if os.name == "nt" or "CI" in os.environ: +if PIPENV_IS_CI: PIPENV_NOSPIN = True +PIPENV_SPINNER = "dots" +"""Sets the default spinner type. + +Spinners are identitcal to the node.js spinners and can be found at +https://github.com/sindresorhus/cli-spinners +""" +if os.name == "nt": + PIPENV_SPINNER = "bouncingBar" + PIPENV_PIPFILE = os.environ.get("PIPENV_PIPFILE") """If set, this specifies a custom Pipfile location. diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 6526d99054..8c282b71f8 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -80,7 +80,6 @@ def resolve(packages, pre, project, sources, clear, system): if pypi_mirror_source else project.pipfile_sources ) - print("using sources: %s" % sources) results = resolve( packages, pre=do_pre, diff --git a/pipenv/vendor/backports/functools_lru_cache.LICENSE b/pipenv/vendor/backports/functools_lru_cache.LICENSE new file mode 100644 index 0000000000..5e795a61f3 --- /dev/null +++ b/pipenv/vendor/backports/functools_lru_cache.LICENSE @@ -0,0 +1,7 @@ +Copyright Jason R. Coombs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/vendor/backports/functools_lru_cache.py b/pipenv/vendor/backports/functools_lru_cache.py new file mode 100644 index 0000000000..707c6c766d --- /dev/null +++ b/pipenv/vendor/backports/functools_lru_cache.py @@ -0,0 +1,184 @@ +from __future__ import absolute_import + +import functools +from collections import namedtuple +from threading import RLock + +_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) + + +@functools.wraps(functools.update_wrapper) +def update_wrapper(wrapper, + wrapped, + assigned = functools.WRAPPER_ASSIGNMENTS, + updated = functools.WRAPPER_UPDATES): + """ + Patch two bugs in functools.update_wrapper. + """ + # workaround for http://bugs.python.org/issue3445 + assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr)) + wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated) + # workaround for https://bugs.python.org/issue17482 + wrapper.__wrapped__ = wrapped + return wrapper + + +class _HashedSeq(list): + __slots__ = 'hashvalue' + + def __init__(self, tup, hash=hash): + self[:] = tup + self.hashvalue = hash(tup) + + def __hash__(self): + return self.hashvalue + + +def _make_key(args, kwds, typed, + kwd_mark=(object(),), + fasttypes=set([int, str, frozenset, type(None)]), + sorted=sorted, tuple=tuple, type=type, len=len): + 'Make a cache key from optionally typed positional and keyword arguments' + key = args + if kwds: + sorted_items = sorted(kwds.items()) + key += kwd_mark + for item in sorted_items: + key += item + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for k, v in sorted_items) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + + +def lru_cache(maxsize=100, typed=False): + """Least-recently-used cache decorator. + + If *maxsize* is set to None, the LRU features are disabled and the cache + can grow without bound. + + If *typed* is True, arguments of different types will be cached separately. + For example, f(3.0) and f(3) will be treated as distinct calls with + distinct results. + + Arguments to the cached function must be hashable. + + View the cache statistics named tuple (hits, misses, maxsize, currsize) with + f.cache_info(). Clear the cache and statistics with f.cache_clear(). + Access the underlying function with f.__wrapped__. + + See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used + + """ + + # Users should only access the lru_cache through its public API: + # cache_info, cache_clear, and f.__wrapped__ + # The internals of the lru_cache are encapsulated for thread safety and + # to allow the implementation to change (including a possible C version). + + def decorating_function(user_function): + + cache = dict() + stats = [0, 0] # make statistics updateable non-locally + HITS, MISSES = 0, 1 # names for the stats fields + make_key = _make_key + cache_get = cache.get # bound method to lookup key or return None + _len = len # localize the global len() function + lock = RLock() # because linkedlist updates aren't threadsafe + root = [] # root of the circular doubly linked list + root[:] = [root, root, None, None] # initialize by pointing to self + nonlocal_root = [root] # make updateable non-locally + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + if maxsize == 0: + + def wrapper(*args, **kwds): + # no caching, just do a statistics update after a successful call + result = user_function(*args, **kwds) + stats[MISSES] += 1 + return result + + elif maxsize is None: + + def wrapper(*args, **kwds): + # simple caching without ordering or size limit + key = make_key(args, kwds, typed) + result = cache_get(key, root) # root used here as a unique not-found sentinel + if result is not root: + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + cache[key] = result + stats[MISSES] += 1 + return result + + else: + + def wrapper(*args, **kwds): + # size limited caching that tracks accesses by recency + key = make_key(args, kwds, typed) if kwds or typed else args + with lock: + link = cache_get(key) + if link is not None: + # record recent use of the key by moving it to the front of the list + root, = nonlocal_root + link_prev, link_next, key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + with lock: + root, = nonlocal_root + if key in cache: + # getting here means that this same key was added to the + # cache while the lock was released. since the link + # update is already done, we need only return the + # computed result and update the count of misses. + pass + elif _len(cache) >= maxsize: + # use the old root to store the new key and result + oldroot = root + oldroot[KEY] = key + oldroot[RESULT] = result + # empty the oldest link and make it the new root + root = nonlocal_root[0] = oldroot[NEXT] + oldkey = root[KEY] + root[KEY] = root[RESULT] = None + # now update the cache dictionary for the new links + del cache[oldkey] + cache[key] = oldroot + else: + # put result in a new link at the front of the list + last = root[PREV] + link = [last, root, key, result] + last[NEXT] = root[PREV] = cache[key] = link + stats[MISSES] += 1 + return result + + def cache_info(): + """Report cache statistics""" + with lock: + return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache)) + + def cache_clear(): + """Clear the cache and cache statistics""" + with lock: + cache.clear() + root = nonlocal_root[0] + root[:] = [root, root, None, None] + stats[:] = [0, 0] + + wrapper.__wrapped__ = user_function + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return update_wrapper(wrapper, user_function) + + return decorating_function diff --git a/pipenv/vendor/certifi/__init__.py b/pipenv/vendor/certifi/__init__.py index aa329fbb4b..50f2e1301f 100644 --- a/pipenv/vendor/certifi/__init__.py +++ b/pipenv/vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.08.24" +__version__ = "2018.10.15" diff --git a/pipenv/vendor/certifi/cacert.pem b/pipenv/vendor/certifi/cacert.pem index 85de024e71..e75d85b38a 100644 --- a/pipenv/vendor/certifi/cacert.pem +++ b/pipenv/vendor/certifi/cacert.pem @@ -326,36 +326,6 @@ OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS -----END CERTIFICATE----- -# Issuer: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Subject: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Label: "Visa eCommerce Root" -# Serial: 25952180776285836048024890241505565794 -# MD5 Fingerprint: fc:11:b8:d8:08:93:30:00:6d:23:f9:7e:eb:52:1e:02 -# SHA1 Fingerprint: 70:17:9b:86:8c:00:a4:fa:60:91:52:22:3f:9f:3e:32:bd:e0:05:62 -# SHA256 Fingerprint: 69:fa:c9:bd:55:fb:0a:c7:8d:53:bb:ee:5c:f1:d5:97:98:9f:d0:aa:ab:20:a2:51:51:bd:f1:73:3e:e7:d1:22 ------BEGIN CERTIFICATE----- -MIIDojCCAoqgAwIBAgIQE4Y1TR0/BvLB+WUF1ZAcYjANBgkqhkiG9w0BAQUFADBr -MQswCQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRl -cm5hdGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNv -bW1lcmNlIFJvb3QwHhcNMDIwNjI2MDIxODM2WhcNMjIwNjI0MDAxNjEyWjBrMQsw -CQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRlcm5h -dGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNvbW1l -cmNlIFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvV95WHm6h -2mCxlCfLF9sHP4CFT8icttD0b0/Pmdjh28JIXDqsOTPHH2qLJj0rNfVIsZHBAk4E -lpF7sDPwsRROEW+1QK8bRaVK7362rPKgH1g/EkZgPI2h4H3PVz4zHvtH8aoVlwdV -ZqW1LS7YgFmypw23RuwhY/81q6UCzyr0TP579ZRdhE2o8mCP2w4lPJ9zcc+U30rq -299yOIzzlr3xF7zSujtFWsan9sYXiwGd/BmoKoMWuDpI/k4+oKsGGelT84ATB+0t -vz8KPFUgOSwsAGl0lUq8ILKpeeUYiZGo3BxN77t+Nwtd/jmliFKMAGzsGHxBvfaL -dXe6YJ2E5/4tAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD -AgEGMB0GA1UdDgQWBBQVOIMPPyw/cDMezUb+B4wg4NfDtzANBgkqhkiG9w0BAQUF -AAOCAQEAX/FBfXxcCLkr4NWSR/pnXKUTwwMhmytMiUbPWU3J/qVAtmPN3XEolWcR -zCSs00Rsca4BIGsDoo8Ytyk6feUWYFN4PMCvFYP3j1IzJL1kk5fui/fbGKhtcbP3 -LBfQdCVp9/5rPJS+TUtBjE7ic9DjkCJzQ83z7+pzzkWKsKZJ/0x9nXGIxHYdkFsd -7v3M9+79YKWxehZx0RbQfBI8bGmX265fOZpwLwU8GUYEmSA20GBuYQa7FkKMcPcw -++DbZqMAAb3mLNqRX6BGi01qnD093QVG/na/oAo85ADmJ7f/hC3euiInlhBx6yLt -398znM/jra6O1I7mT1GvFpLgXPYHDw== ------END CERTIFICATE----- - # Issuer: CN=AAA Certificate Services O=Comodo CA Limited # Subject: CN=AAA Certificate Services O=Comodo CA Limited # Label: "Comodo AAA Services root" diff --git a/pipenv/vendor/cursor/LICENSE b/pipenv/vendor/cursor/LICENSE new file mode 100644 index 0000000000..00023c8025 --- /dev/null +++ b/pipenv/vendor/cursor/LICENSE @@ -0,0 +1,5 @@ +This work is licensed under the Creative Commons +Attribution-ShareAlike 2.5 International License. To view a copy of +this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or +send a letter to Creative Commons, PO Box 1866, Mountain View, +CA 94042, USA. diff --git a/pipenv/vendor/cursor/__init__.py b/pipenv/vendor/cursor/__init__.py new file mode 100644 index 0000000000..76a4f671f8 --- /dev/null +++ b/pipenv/vendor/cursor/__init__.py @@ -0,0 +1,4 @@ +from .cursor import hide, show, HiddenCursor + +__all__ = ["hide", "show", "HiddenCursor"] + diff --git a/pipenv/vendor/cursor/cursor.py b/pipenv/vendor/cursor/cursor.py new file mode 100644 index 0000000000..e4407c02c9 --- /dev/null +++ b/pipenv/vendor/cursor/cursor.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +## Author: James Spencer: http://stackoverflow.com/users/1375885/james-spencer +## Packager: Gijs TImmers: https://github.com/GijsTimmers + +## Based on James Spencer's answer on StackOverflow: +## http://stackoverflow.com/questions/5174810/how-to-turn-off-blinking-cursor-in-command-window + +## Licence: CC-BY-SA-2.5 +## http://creativecommons.org/licenses/by-sa/2.5/ + +## This work is licensed under the Creative Commons +## Attribution-ShareAlike 2.5 International License. To view a copy of +## this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or +## send a letter to Creative Commons, PO Box 1866, Mountain View, +## CA 94042, USA. + +import sys +import os + +if os.name == 'nt': + import ctypes + + class _CursorInfo(ctypes.Structure): + _fields_ = [("size", ctypes.c_int), + ("visible", ctypes.c_byte)] + +def hide(stream=sys.stdout): + if os.name == 'nt': + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = False + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == 'posix': + stream.write("\033[?25l") + stream.flush() + +def show(stream=sys.stdout): + if os.name == 'nt': + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = True + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == 'posix': + stream.write("\033[?25h") + stream.flush() + +class HiddenCursor(object): + def __init__(self, stream=sys.stdout): + self._stream = stream + def __enter__(self): + hide(stream=self._stream) + def __exit__(self, type, value, traceback): + show(stream=self._stream) \ No newline at end of file diff --git a/pipenv/vendor/parse.py b/pipenv/vendor/parse.py index ba58decbda..7f9f07860a 100644 --- a/pipenv/vendor/parse.py +++ b/pipenv/vendor/parse.py @@ -186,6 +186,19 @@ Note that the "center" alignment does not test to make sure the value is centered - it just strips leading and trailing whitespace. +Width and precision may be used to restrict the size of matched text +from the input. Width specifies a minimum size and precision specifies +a maximum. For example: + +>>> parse('{:.2}{:.2}', 'look') # specifying precision + +>>> parse('{:4}{:4}', 'look at that') # specifying width + +>>> parse('{:4}{:.4}', 'look at that') # specifying both + +>>> parse('{:2d}{:2d}', '0440') # parsing two contiguous numbers + + Some notes for the date and time types: - the presence of the time part is optional (including ISO 8601, starting @@ -329,6 +342,9 @@ **Version history (in brief)**: +- 1.9.0 We now honor precision and width specifiers when parsing numbers + and strings, allowing parsing of concatenated elements of fixed width + (thanks Julia Signell) - 1.8.4 Add LICENSE file at request of packagers. Correct handling of AM/PM to follow most common interpretation. Correct parsing of hexadecimal that looks like a binary prefix. @@ -389,7 +405,7 @@ ''' from __future__ import absolute_import -__version__ = '1.8.4' +__version__ = '1.9.0' # yes, I now have two problems import re @@ -977,7 +993,11 @@ def f(string, m): self._group_index += 2 self._type_conversions[group] = lambda s, m: float(s) elif type == 'd': - s = r'\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|0[oO][0-7]+' + if format.get('width'): + width = '{1,%s}' % int(format['width']) + else: + width = '+' + s = '\\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) self._type_conversions[group] = int_convert(10) elif type == 'ti': s = r'(\d{4}-\d\d-\d\d)((\s+|T)%s)?(Z|\s*[-+]\d\d:?\d\d)?' % \ @@ -1038,6 +1058,13 @@ def f(string, m): elif type: s = r'\%s+' % type + elif format.get('precision'): + if format.get('width'): + s = '.{%s,%s}?' % (format['width'], format['precision']) + else: + s = '.{1,%s}?' % format['precision'] + elif format.get('width'): + s = '.{%s,}?' % format['width'] else: s = '.+?' @@ -1053,8 +1080,6 @@ def f(string, m): if not fill: fill = '0' s = '%s*' % fill + s - elif format['zero']: - s = '0*' + s # allow numbers to be prefixed with a sign s = r'[-+ ]?' + s diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 672724b4bf..9ac6031ca4 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.2' +__version__ = '1.1.3' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index b5aa7da363..1757c081ad 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -17,7 +17,7 @@ @click.option( "--version", is_flag=True, default=False, help="Display PythonFinder version." ) -@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, help="Ignore unsupported python versions.") +@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", help="Ignore unsupported python versions.") @click.version_option(prog_name='pyfinder', version=__version__) @click.pass_context def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True): @@ -36,7 +36,7 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup for v in versions: py = v.py_version click.secho( - "Python: {py.version!s} ({py.architecture!s}) @ {py.comes_from.path!s}".format( + "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {py.comes_from.path!s}".format( py=py ), fg="yellow", @@ -47,23 +47,21 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup fg="red", ) if find: - if any([find.startswith("{0}".format(n)) for n in range(10)]): - found = finder.find_python_version(find.strip()) - else: - found = finder.system_path.python_executables + click.secho("Searching for python: {0!s}".format(find.strip()), fg="yellow") + found = finder.find_python_version(find.strip()) if found: - click.echo("Found Python Version: {0}".format(found), color="white") + click.secho("Found python at the following locations:", fg="green") sys.exit(0) else: - click.echo("Failed to find matching executable...") + click.secho("Failed to find matching executable...", fg="yellow") sys.exit(1) elif which: found = finder.system_path.which(which.strip()) if found: - click.echo("Found Executable: {0}".format(found), color="white") + click.secho("Found Executable: {0}".format(found), fg="white") sys.exit(0) else: - click.echo("Failed to find matching executable...") + click.secho("Failed to find matching executable...", fg="yellow") sys.exit(1) else: click.echo("Please provide a command", color="red") diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index 2cdb5fd959..7c69b9fc37 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -15,3 +15,6 @@ IS_64BIT_OS = platform.machine() == "AMD64" else: IS_64BIT_OS = False + + +IGNORE_UNSUPPORTED = bool(os.environ.get("PYTHONFINDER_IGNORE_UNSUPPORTED", False)) diff --git a/pipenv/vendor/pythonfinder/models/mixins.py b/pipenv/vendor/pythonfinder/models/mixins.py index 8cbd45dfeb..7d4065484c 100644 --- a/pipenv/vendor/pythonfinder/models/mixins.py +++ b/pipenv/vendor/pythonfinder/models/mixins.py @@ -2,12 +2,14 @@ from __future__ import absolute_import, unicode_literals import abc +import attr import operator import six -from ..utils import KNOWN_EXTS, unnest +from ..utils import ensure_path, KNOWN_EXTS, unnest +@attr.s class BasePath(object): def which(self, name): """Search in this path for an executable. @@ -33,7 +35,14 @@ def which(self, name): return found def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. Return all copies @@ -44,6 +53,7 @@ def find_all_python_versions( :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ @@ -52,7 +62,14 @@ def find_all_python_versions( "find_all_python_versions" if self.is_dir else "find_python_version" ) sub_finder = operator.methodcaller( - call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + call_method, + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, ) if not self.is_dir: return sub_finder(self) @@ -61,7 +78,14 @@ def find_all_python_versions( return [c for c in sorted(path_filter, key=version_sort, reverse=True)] def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search or self for the specified Python version and return the first match. @@ -72,6 +96,7 @@ def find_python_version( :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ @@ -83,12 +108,13 @@ def find_python_version( pre=pre, dev=dev, arch=arch, + name=name, ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and self.as_python and version_matcher(self.as_python): - return self + if self.is_python and self.as_python and version_matcher(self.py_version): + return attr.evolve(self) return finder = ( (child, child.as_python) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index c90d9be37c..d54393b6b3 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -10,6 +10,7 @@ from itertools import chain import attr +import six from cached_property import cached_property @@ -19,8 +20,12 @@ from ..environment import PYENV_INSTALLED, PYENV_ROOT from ..exceptions import InvalidPythonVersion from ..utils import ( - ensure_path, filter_pythons, looks_like_python, optional_instance_of, - path_is_known_executable, unnest + ensure_path, + filter_pythons, + looks_like_python, + optional_instance_of, + path_is_known_executable, + unnest, ) from .python import PythonVersion @@ -75,7 +80,7 @@ def version_dict(self): if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue - if isinstance(entry, VersionPath): + if type(entry).__name__ == "VersionPath": for path in entry.paths.values(): if path not in self._version_dict[version] and path.is_python: self._version_dict[version].append(path) @@ -130,17 +135,16 @@ def _setup_pyenv(self): pyenv_index = self.path_order.index(last_pyenv) except ValueError: return - self.pyenv_finder = PyenvFinder.create(root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported) - # paths = (v.paths.values() for v in self.pyenv_finder.versions.values()) - root_paths = ( - p for path in self.pyenv_finder.expanded_paths for p in path if p.is_root + self.pyenv_finder = PyenvFinder.create( + root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported ) + root_paths = [p for p in self.pyenv_finder.roots] before_path = self.path_order[: pyenv_index + 1] after_path = self.path_order[pyenv_index + 2 :] self.path_order = ( - before_path + [p.path.as_posix() for p in root_paths] + after_path + before_path + [p.as_posix() for p in root_paths] + after_path ) - self.paths.update({p.path: p for p in root_paths}) + self.paths.update(self.pyenv_finder.roots) self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): @@ -155,7 +159,9 @@ def _setup_windows(self): def get_path(self, path): path = ensure_path(path) - _path = self.paths.get(path.as_posix()) + _path = self.paths.get(path) + if not _path: + _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: _path = PathEntry.create( path=path.absolute(), is_root=True, only_python=self.only_python @@ -163,6 +169,14 @@ def get_path(self, path): self.paths[path.as_posix()] = _path return _path + def _get_paths(self): + return (self.get_path(k) for k in self.path_order) + + @cached_property + def path_entries(self): + paths = self._get_paths() + return paths + def find_all(self, executable): """Search the path for an executable. Return all copies. @@ -171,8 +185,8 @@ def find_all(self, executable): :returns: List[PathEntry] """ sub_which = operator.methodcaller("which", name=executable) - filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) - return [f for f in filtered] + filtered = (sub_which(self.get_path(k)) for k in self.path_order) + return list(filtered) def which(self, executable): """Search for an executable on the path. @@ -182,11 +196,39 @@ def which(self, executable): :returns: :class:`~pythonfinder.models.PathEntry` object. """ sub_which = operator.methodcaller("which", name=executable) - filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) - return next((f for f in filtered), None) + filtered = (sub_which(self.get_path(k)) for k in self.path_order) + return next(iter(f for f in filtered if f is not None), None) + + def _filter_paths(self, finder): + return ( + pth for pth in unnest(finder(p) for p in self.path_entries if p is not None) + if pth is not None + ) + + def _get_all_pythons(self, finder): + paths = {p.path.as_posix(): p for p in self._filter_paths(finder)} + paths.update(self.python_executables) + return (p for p in paths.values() if p is not None) + + def get_pythons(self, finder): + sort_key = operator.attrgetter("as_python.version_sort") + return ( + k for k in sorted( + (p for p in self._filter_paths(finder) if p.is_python), + key=sort_key, + reverse=True + ) if k is not None + ) def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. Return all copies @@ -197,32 +239,46 @@ def find_all_python_versions( :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ sub_finder = operator.methodcaller( "find_all_python_versions", - major, + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, + name=name, ) + alternate_sub_finder = None + if major and not (minor or patch or pre or dev or arch or name): + alternate_sub_finder = operator.methodcaller( + "find_all_python_versions", + major=None, + name=major + ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version - paths = (self.get_path(k) for k in self.path_order) - path_filter = filter( - None, unnest((sub_finder(p) for p in paths if p is not None)) - ) - version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + values = list(self.get_pythons(sub_finder)) + if not values and alternate_sub_finder is not None: + values = list(self.get_pythons(alternate_sub_finder)) + return values def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. @@ -233,10 +289,24 @@ def find_python_version( :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ + if isinstance(major, six.string_types) and not minor and not patch: + # Only proceed if this is in the format "x.y.z" or similar + if major.count(".") > 0 and major[0].isdigit(): + version = major.split(".", 2) + if len(version) > 3: + major, minor, patch, rest = version + elif len(version) == 3: + major, minor, patch = version + else: + major, minor = version + else: + name = "{0!s}".format(major) + major = None sub_finder = operator.methodcaller( "find_python_version", major, @@ -245,7 +315,15 @@ def find_python_version( pre=pre, dev=dev, arch=arch, + name=name, ) + alternate_sub_finder = None + if major and not (minor or patch or pre or dev or arch or name): + alternate_sub_finder = operator.methodcaller( + "find_all_python_versions", + major=None, + name=major + ) if major and minor and patch: _tuple_pre = pre if pre is not None else False _tuple_dev = dev if dev is not None else False @@ -255,37 +333,41 @@ def find_python_version( windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version - paths = (self.get_path(k) for k in self.path_order) - path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) - version_sort = operator.attrgetter("as_python.version_sort") - ver = next( - (c for c in sorted(path_filter, key=version_sort, reverse=True)), None - ) + ver = next(iter(self.get_pythons(sub_finder)), None) + if not ver and alternate_sub_finder is not None: + ver = next(iter(self.get_pythons(alternate_sub_finder)), None) if ver: if ver.as_python.version_tuple[:5] in self.python_version_dict: self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) else: self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver] + print(ver) return ver @classmethod - def create(cls, path=None, system=False, only_python=False, global_search=True, ignore_unsupported=False): + def create( + cls, + path=None, + system=False, + only_python=False, + global_search=True, + ignore_unsupported=True, + ): """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None :param path: str, optional - :param system: Whether to use the running python by default instead of searching, defaults to False - :param system: bool, optional - :param only_python: Whether to search only for python executables, defaults to False - :param only_python: bool, optional - :param ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True - :param ignore_unsupported: bool, optional + :param bool system: Whether to use the running python by default instead of searching, defaults to False + :param bool only_python: Whether to search only for python executables, defaults to False + :param bool ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True :return: A new :class:`pythonfinder.models.SystemPath` instance. :rtype: :class:`pythonfinder.models.SystemPath` """ path_entries = defaultdict(PathEntry) paths = [] + if ignore_unsupported: + os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") if global_search: paths = os.environ.get("PATH").split(os.pathsep) if path: @@ -316,7 +398,8 @@ class PathEntry(BasePath): _children = attr.ib(default=attr.Factory(dict)) is_root = attr.ib(default=True) only_python = attr.ib(default=False) - py_version = attr.ib(default=None) + name = attr.ib() + py_version = attr.ib() pythons = attr.ib() def __str__(self): @@ -329,17 +412,46 @@ def _filter_children(self): children = self.path.iterdir() return children + def _gen_children(self): + pass_name = self.name != self.path.name + pass_args = {"is_root": False, "only_python": self.only_python} + if pass_name: + pass_args["name"] = self.name + + if not self.is_dir: + yield (self.path.as_posix(), copy.deepcopy(self)) + elif self.is_root: + for child in self._filter_children(): + yield (child.as_posix(), PathEntry.create(path=child, **pass_args)) + return + @cached_property def children(self): - if not self._children and self.is_dir and self.is_root: - self._children = { - child.as_posix(): PathEntry.create(path=child, is_root=False) - for child in self._filter_children() - } - elif not self.is_dir: - self._children = {self.path.as_posix(): self} + if not self._children: + children = {} + for child_key, child_val in self._gen_children(): + children[child_key] = child_val + self._children = children return self._children + @name.default + def get_name(self): + return self.path.name + + @py_version.default + def get_py_version(self): + from ..environment import IGNORE_UNSUPPORTED + if self.is_dir: + return None + if self.is_python: + from .python import PythonVersion + try: + py_version = PythonVersion.from_path(path=self, name=self.name) + except InvalidPythonVersion: + py_version = None + return py_version + return + @pythons.default def get_pythons(self): pythons = defaultdict() @@ -351,56 +463,62 @@ def get_pythons(self): else: if self.is_python: _path = ensure_path(self.path) - pythons[_path.as_posix()] = copy.deepcopy(self) + pythons[_path.as_posix()] = self return pythons @cached_property def as_python(self): + py_version = None + if self.py_version: + return self.py_version if not self.is_dir and self.is_python: - if not self.py_version: - try: - from .python import PythonVersion - - self.py_version = PythonVersion.from_path(self.path) - except (ValueError, InvalidPythonVersion): - self.py_version = None - return self.py_version + try: + from .python import PythonVersion + py_version = PythonVersion.from_path(path=attr.evolve(self), name=self.name) + except (ValueError, InvalidPythonVersion): + py_version = None + return py_version @classmethod - def create(cls, path, is_root=False, only_python=False, pythons=None): + def create(cls, path, is_root=False, only_python=False, pythons=None, name=None): """Helper method for creating new :class:`pythonfinder.models.PathEntry` instances. - :param path: Path to the specified location. - :type path: str - :param is_root: Whether this is a root from the environment PATH variable, defaults to False - :param is_root: bool, optional - :param only_python: Whether to search only for python executables, defaults to False - :param only_python: bool, optional - :param pythons: A dictionary of existing python objects (usually from a finder), defaults to None - :param pythons: dict, optional + :param str path: Path to the specified location. + :param bool is_root: Whether this is a root from the environment PATH variable, defaults to False + :param bool only_python: Whether to search only for python executables, defaults to False + :param dict pythons: A dictionary of existing python objects (usually from a finder), defaults to None + :param str name: Name of the python version, e.g. ``anaconda3-5.3.0`` :return: A new instance of the class. :rtype: :class:`pythonfinder.models.PathEntry` """ target = ensure_path(path) - creation_args = {"path": target, "is_root": is_root, "only_python": only_python} + guessed_name = False + if not name: + guessed_name = True + name = target.name + creation_args = {"path": target, "is_root": is_root, "only_python": only_python, "name": name} if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) if pythons and only_python: children = {} + child_creation_args = { + "is_root": False, + "py_version": python, + "only_python": only_python + } + if not guessed_name: + child_creation_args["name"] = name for pth, python in pythons.items(): pth = ensure_path(pth) children[pth.as_posix()] = PathEntry( - path=pth, is_root=False, only_python=only_python, py_version=python + path=pth, + **child_creation_args ) _new._children = children return _new - @cached_property - def name(self): - return self.path.name - @cached_property def is_dir(self): try: @@ -416,28 +534,5 @@ def is_executable(self): @cached_property def is_python(self): return self.is_executable and ( - self.py_version or looks_like_python(self.path.name) - ) - - -@attr.s -class VersionPath(SystemPath): - base = attr.ib(default=None, validator=optional_instance_of(Path)) - - @classmethod - def create(cls, path, only_python=True, pythons=None): - """Accepts a path to a base python version directory. - - Generates the pyenv version listings for it""" - path = ensure_path(path) - path_entries = defaultdict(PathEntry) - if not path.name.lower() in ["scripts", "bin"]: - bin_name = "Scripts" if os.name == "nt" else "bin" - bin_dir = path / bin_name - else: - bin_dir = path - current_entry = PathEntry.create( - bin_dir, is_root=True, only_python=True, pythons=pythons + looks_like_python(self.path.name) ) - path_entries[bin_dir.as_posix()] = current_entry - return cls(base=bin_dir, paths=path_entries) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 527c5f0af7..1595a963a7 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function import logging +import operator from collections import defaultdict @@ -10,9 +11,15 @@ from vistir.compat import Path -from ..utils import ensure_path, optional_instance_of, get_python_version, filter_pythons -from .mixins import BaseFinder -from .path import VersionPath +from ..utils import ( + ensure_path, + optional_instance_of, + get_python_version, + filter_pythons, + unnest, +) +from .mixins import BaseFinder, BasePath +from .path import SystemPath, PathEntry from .python import PythonVersion @@ -20,51 +27,66 @@ @attr.s -class PyenvFinder(BaseFinder): +class PyenvFinder(BaseFinder, BasePath): root = attr.ib(default=None, validator=optional_instance_of(Path)) - # ignore_unsupported should come before versions, because its value is used - # in versions's default initializer. - ignore_unsupported = attr.ib(default=False) + #: ignore_unsupported should come before versions, because its value is used + #: in versions's default initializer. + ignore_unsupported = attr.ib(default=True) + paths = attr.ib(default=attr.Factory(list)) + roots = attr.ib(default=attr.Factory(defaultdict)) versions = attr.ib() pythons = attr.ib() + @property + def expanded_paths(self): + return ( + path for path in unnest(p for p in self.versions.values()) + if path is not None + ) + @classmethod - def version_from_bin_dir(cls, base_dir): - pythons = [py for py in filter_pythons(base_dir)] + def version_from_bin_dir(cls, base_dir, name=None): py_version = None - for py in pythons: - version = get_python_version(py.as_posix()) - try: - py_version = PythonVersion.parse(version) - except Exception: - continue - if py_version: - return py_version - return + version_path = PathEntry.create( + path=base_dir.absolute().as_posix(), + only_python=True, + name=base_dir.parent.name, + ) + py_version = next(iter(version_path.find_all_python_versions()), None) + return py_version @versions.default def get_versions(self): - versions = defaultdict(VersionPath) + versions = defaultdict() bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] for p in self.root.glob("versions/*"): - if p.parent.name == "envs": + if p.parent.name == "envs" or p.name == "envs": continue + bin_dir = Path(bin_.format(base=p.as_posix())) + version_path = None + if bin_dir.exists(): + version_path = PathEntry.create( + path=bin_dir.absolute().as_posix(), + only_python=False, + name=p.name, + is_root=True, + ) + version = None try: version = PythonVersion.parse(p.name) except ValueError: - bin_dir = Path(bin_.format(base=p.as_posix())) - if bin_dir.exists(): - version = self.version_from_bin_dir(bin_dir) - if not version: - if not self.ignore_unsupported: - raise - continue + entry = next(iter(version_path.find_all_python_versions()), None) + if not entry: + if self.ignore_unsupported: + continue + raise + else: + version = entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise logger.warning( - 'Unsupported Python version %r, ignoring...', - p.name, exc_info=True + "Unsupported Python version %r, ignoring...", p.name, exc_info=True ) continue if not version: @@ -75,24 +97,128 @@ def get_versions(self): version.get("patch"), version.get("is_prerelease"), version.get("is_devrelease"), - version.get("is_debug") - ) - versions[version_tuple] = VersionPath.create( - path=p.resolve(), only_python=True + version.get("is_debug"), ) + self.roots[p] = version_path + versions[version_tuple] = version_path + self.paths.append(version_path) return versions @pythons.default def get_pythons(self): pythons = defaultdict() - for v in self.versions.values(): - for p in v.paths.values(): - _path = ensure_path(p.path) - if p.is_python: - pythons[_path] = p + for p in self.paths: + pythons.update(p.pythons) return pythons @classmethod - def create(cls, root, ignore_unsupported=False): + def create(cls, root, ignore_unsupported=True): root = ensure_path(root) return cls(root=root, ignore_unsupported=ignore_unsupported) + + def find_all_python_versions( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search for a specific python version on the path. Return all copies + + :param major: Major python version to search for. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. + :rtype: List[:class:`~pythonfinder.models.PathEntry`] + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + py = operator.attrgetter("as_python") + pythons = ( + py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) + if py_ver is not None + ) + # pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return sorted(matching_versions, key=version_sort, reverse=True) + + def find_python_version( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search or self for the specified Python version and return the first match. + + :param major: Major version number. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) + + +@attr.s +class VersionPath(SystemPath): + base = attr.ib(default=None, validator=optional_instance_of(Path)) + name = attr.ib(default=None) + + @classmethod + def create(cls, path, only_python=True, pythons=None, name=None): + """Accepts a path to a base python version directory. + + Generates the pyenv version listings for it""" + path = ensure_path(path) + path_entries = defaultdict(PathEntry) + bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] + if path.as_posix().endswith(Path(bin_).name): + path = path.parent + bin_dir = ensure_path(bin_.format(base=path.as_posix())) + if not name: + name = path.name + current_entry = PathEntry.create( + bin_dir, is_root=True, only_python=True, pythons=pythons, name=name + ) + path_entries[bin_dir.as_posix()] = current_entry + return cls(name=name, base=bin_dir, paths=path_entries) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index c71b9d9bf7..ec99afe731 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -13,7 +13,11 @@ from ..environment import SYSTEM_ARCH from ..utils import ( - _filter_none, ensure_path, get_python_version, optional_instance_of + _filter_none, + ensure_path, + get_python_version, + optional_instance_of, + ensure_path, ) @@ -30,6 +34,7 @@ class PythonVersion(object): architecture = attr.ib(default=None) comes_from = attr.ib(default=None) executable = attr.ib(default=None) + name = attr.ib(default=None) @property def version_sort(self): @@ -65,22 +70,37 @@ def version_tuple(self): self.patch, self.is_prerelease, self.is_devrelease, - self.is_debug + self.is_debug, ) def matches( - self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None, debug=False + self, + major=None, + minor=None, + patch=None, + pre=False, + dev=False, + arch=None, + debug=False, + name=None, ): - if arch and arch.isdigit(): - arch = "{0}bit".format(arch) + if arch: + own_arch = self.get_architecture() + if arch.isdigit(): + arch = "{0}bit".format(arch) return ( (major is None or self.major == major) and (minor is None or self.minor == minor) and (patch is None or self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) - and (arch is None or self.architecture == arch) + and (arch is None or own_arch == arch) and (debug is None or self.is_debug == debug) + and ( + name is None + or (name and self.name) + and (self.name == name or self.name.startswith(name)) + ) ) def as_major(self): @@ -93,6 +113,18 @@ def as_minor(self): self_dict.update({"patch": None}) return self.create(**self_dict) + def as_dict(self): + return { + "major": self.major, + "minor": self.minor, + "patch": self.patch, + "is_prerelease": self.is_prerelease, + "is_postrelease": self.is_postrelease, + "is_devrelease": self.is_devrelease, + "is_debug": self.is_debug, + "version": self.version, + } + @classmethod def parse(cls, version): """Parse a valid version string into a dictionary @@ -138,8 +170,15 @@ def parse(cls, version): "version": version, } + def get_architecture(self): + if self.architecture: + return self.architecture + arch, _ = platform.architecture(path.path.as_posix()) + self.architecture = arch + return self.architecture + @classmethod - def from_path(cls, path): + def from_path(cls, path, name=None): """Parses a python version from a system path. Raises: @@ -147,29 +186,33 @@ def from_path(cls, path): :param path: A string or :class:`~pythonfinder.models.path.PathEntry` :type path: str or :class:`~pythonfinder.models.path.PathEntry` instance - :param launcher_entry: A python launcher environment object. + :param str name: Name of the python distribution in question :return: An instance of a PythonVersion. :rtype: :class:`~pythonfinder.models.python.PythonVersion` """ from .path import PathEntry + from ..environment import IGNORE_UNSUPPORTED if not isinstance(path, PathEntry): - path = PathEntry.create(path, is_root=False, only_python=True) - if not path.is_python: + path = PathEntry.create(path, is_root=False, only_python=True, name=name) + if not path.is_python and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return - py_version = get_python_version(str(path.path)) + py_version = get_python_version(path.path.as_posix()) instance_dict = cls.parse(py_version) - if not isinstance(instance_dict.get("version"), Version): + if not isinstance(instance_dict.get("version"), Version) and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return - architecture, _ = platform.architecture(path.path.as_posix()) - instance_dict.update({"comes_from": path, "architecture": architecture}) + if not name: + name = path.name + instance_dict.update( + {"comes_from": path, "name": name} + ) return cls(**instance_dict) @classmethod - def from_windows_launcher(cls, launcher_entry): + def from_windows_launcher(cls, launcher_entry, name=None): """Create a new PythonVersion instance from a Windows Launcher Entry :param launcher_entry: A python launcher environment object. @@ -193,12 +236,14 @@ def from_windows_launcher(cls, launcher_entry): launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, + "name": name } ) py_version = cls.create(**creation_dict) - comes_from = PathEntry.create(exe_path, only_python=True) + comes_from = PathEntry.create(exe_path, only_python=True, name=name) comes_from.py_version = copy.deepcopy(py_version) py_version.comes_from = comes_from + py_version.name = comes_from.name return py_version @classmethod diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index fcb4d42a53..e47bcc2c0b 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -22,7 +22,14 @@ class WindowsFinder(BaseFinder): pythons = attr.ib() def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): version_matcher = operator.methodcaller( "matches", @@ -32,6 +39,7 @@ def find_all_python_versions( pre=pre, dev=dev, arch=arch, + name=name, ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -40,13 +48,26 @@ def find_all_python_versions( return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): return next( ( v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=None, ) ), None, @@ -60,7 +81,7 @@ def get_versions(self): env_versions = pep514env.findall() path = None for version_object in env_versions: - install_path = getattr(version_object.info, 'install_path', None) + install_path = getattr(version_object.info, "install_path", None) if install_path is None: continue path = ensure_path(install_path.__getattr__("")) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index e965bb511b..19a52e0a3b 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -7,13 +7,13 @@ class Finder(object): - def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=False): + def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): """Finder A cross-platform Finder for locating python and other executables. Searches for python and other specified binaries starting in `path`, if supplied, but searching the bin path of `sys.executable` if `system=True`, and then searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` - is `False`, this search operation is restricted to the allowed locations of + is `False`, this search operation is restricted to the allowed locations of `path` and `system`. :param path: A bin-directory search location, defaults to None @@ -57,7 +57,7 @@ def which(self, exe): return self.system_path.which(exe) def find_python_version( - self, major, minor=None, patch=None, pre=None, dev=None, arch=None + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): from .models import PythonVersion @@ -69,12 +69,24 @@ def find_python_version( and patch is None ): if arch is None and "-" in major: - major, arch = major.rsplit("-", 1) - if not arch.isdigit(): - major = "{0}-{1}".format(major, arch) + orig_string = "{0!s}".format(major) + major, _, arch = major.rpartition("-") + if arch.startswith("x"): + arch = arch.lstrip("x") + if arch.lower().endswith("bit"): + arch = arch.lower().replace("bit", "") + if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0): + major = orig_string + arch = None else: arch = "{0}bit".format(arch) - version_dict = PythonVersion.parse(major) + try: + version_dict = PythonVersion.parse(major) + except ValueError: + if name is None: + name = "{0!s}".format(major) + major = None + version_dict = {} major = version_dict.get("major", major) minor = version_dict.get("minor", minor) patch = version_dict.get("patch", patch) @@ -83,16 +95,16 @@ def find_python_version( arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if match: return match return self.system_path.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, "python_version_dict") @@ -109,7 +121,7 @@ def find_all_python_versions( paths = sorted(paths, key=version_sort, reverse=True) return paths versions = self.system_path.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if not isinstance(versions, list): versions = [versions] diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index dced9eabcb..2f5a860da1 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -16,10 +16,15 @@ from .exceptions import InvalidPythonVersion +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + PYTHON_IMPLEMENTATIONS = ( "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", - "stackless", "activepython" + "stackless", "activepython", "micropython" ) RULES_BASE = ["*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m"] RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] @@ -29,7 +34,17 @@ filter(None, os.environ.get("PATHEXT", "").split(os.pathsep)) ) +MATCH_RULES = [] +for rule in RULES: + MATCH_RULES.extend( + [ + "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) + for ext in KNOWN_EXTS + ] + ) + +@lru_cache(maxsize=128) def get_python_version(path): """Get python version string using subprocess from a given path.""" version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] @@ -54,6 +69,7 @@ def path_is_executable(path): return os.access(str(path), os.X_OK) +@lru_cache(maxsize=1024) def path_is_known_executable(path): return ( path_is_executable(path) @@ -62,24 +78,19 @@ def path_is_known_executable(path): ) +@lru_cache(maxsize=1024) def looks_like_python(name): - match_rules = [] - for rule in RULES: - match_rules.extend( - [ - "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) - for ext in KNOWN_EXTS - ] - ) if not any(name.lower().startswith(py_name) for py_name in PYTHON_IMPLEMENTATIONS): return False - return any(fnmatch(name, rule) for rule in match_rules) + return any(fnmatch(name, rule) for rule in MATCH_RULES) +@lru_cache(maxsize=128) def path_is_python(path): return path_is_executable(path) and looks_like_python(path.name) +@lru_cache(maxsize=1024) def ensure_path(path): """Given a path (either a string or a Path object), expand variables and return a Path object. @@ -90,13 +101,9 @@ def ensure_path(path): """ if isinstance(path, vistir.compat.Path): - path = path.as_posix() + return path path = vistir.compat.Path(os.path.expandvars(path)) - try: - path = path.resolve() - except OSError: - path = path.absolute() - return path + return path.absolute() def _filter_none(k, v): @@ -105,6 +112,7 @@ def _filter_none(k, v): return False +@lru_cache(maxsize=128) def filter_pythons(path): """Return all valid pythons in a given path""" if not isinstance(path, vistir.compat.Path): diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 1f3a2fcb78..e0bc6746a8 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.9' +__version__ = '1.1.10' from .exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index f9ca97b83e..bd76ca0177 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -1,17 +1,19 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import json +import copy import os +import attr import plette.lockfiles import six -from vistir.compat import Path -from vistir.contextmanagers import atomic_open_for_write +from vistir.compat import Path, FileNotFoundError +from .project import ProjectFile from .requirements import Requirement +from .utils import optional_instance_of DEFAULT_NEWLINES = u"\n" @@ -22,47 +24,125 @@ def preferred_newlines(f): return DEFAULT_NEWLINES -class Lockfile(plette.lockfiles.Lockfile): - def __init__(self, *args, **kwargs): - path = kwargs.pop("path", None) - self._requirements = kwargs.pop("requirements", []) - self._dev_requirements = kwargs.pop("dev_requirements", []) - self.path = Path(path) if path else None - self.newlines = u"\n" - super(Lockfile, self).__init__(*args, **kwargs) +is_lockfile = optional_instance_of(plette.lockfiles.Lockfile) +is_projectfile = optional_instance_of(ProjectFile) + + +@attr.s(slots=True) +class Lockfile(object): + path = attr.ib(validator=optional_instance_of(Path), type=Path) + _requirements = attr.ib(default=attr.Factory(list), type=list) + _dev_requirements = attr.ib(default=attr.Factory(list), type=list) + projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) + _lockfile = attr.ib(validator=is_lockfile, type=plette.lockfiles.Lockfile) + newlines = attr.ib(default=DEFAULT_NEWLINES, type=six.text_type) + + @path.default + def _get_path(self): + return Path(os.curdir).absolute() + + @projectfile.default + def _get_projectfile(self): + return self.load_projectfile(self.path) + + @_lockfile.default + def _get_lockfile(self): + return self.projectfile.lockfile + + def __getattr__(self, k, *args, **kwargs): + retval = None + lockfile = super(Lockfile, self).__getattribute__("_lockfile") + try: + return super(Lockfile, self).__getattribute__(k) + except AttributeError: + retval = getattr(lockfile, k, None) + if not retval: + retval = super(Lockfile, self).__getattribute__(k, *args, **kwargs) + return retval + + @classmethod + def read_projectfile(cls, path): + """Read the specified project file and provide an interface for writing/updating. + + :param str path: Path to the target file. + :return: A project file with the model and location for interaction + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + + pf = ProjectFile.read( + path, + plette.lockfiles.Lockfile, + invalid_ok=True + ) + return pf @classmethod - def load(cls, path): + def load_projectfile(cls, path, create=True): + """Given a path, load or create the necessary lockfile. + + :param str path: Path to the project root or lockfile + :param bool create: Whether to create the lockfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the lockfile doesn't exist and ``create=False`` + :return: A project file instance for the supplied project + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + if not path: path = os.curdir path = Path(path).absolute() - if path.is_dir(): - path = path / "Pipfile.lock" - elif path.name == "Pipfile": - path = path.parent / "Pipfile.lock" - if not path.exists(): - raise OSError("Path does not exist: %s" % path) - return cls.create(path.parent, lockfile_name=path.name) + project_path = path if path.is_dir() else path.parent + lockfile_path = project_path / "Pipfile.lock" + if not project_path.exists(): + raise OSError("Project does not exist: %s" % project_path.as_posix()) + elif not lockfile_path.exists() and not create: + raise FileNotFoundError("Lockfile does not exist: %s" % lockfile_path.as_posix()) + projectfile = cls.read_projectfile(lockfile_path.as_posix()) + return projectfile @classmethod - def create(cls, project_path, lockfile_name="Pipfile.lock"): - """Create a new lockfile instance + def load(cls, path, create=True): + """Create a new lockfile instance. :param project_path: Path to project root - :type project_path: str or :class:`~pathlib.Path` - :returns: List[:class:`~requirementslib.Requirement`] objects + :type project_path: str or :class:`pathlib.Path` + :param str lockfile_name: Name of the lockfile in the project root directory + :param pipfile_path: Path to the project pipfile + :type pipfile_path: :class:`pathlib.Path` + :returns: A new lockfile representing the supplied project paths + :rtype: :class:`~requirementslib.models.lockfile.Lockfile` """ - if not isinstance(project_path, Path): - project_path = Path(project_path) - lockfile_path = project_path / lockfile_name - with lockfile_path.open(encoding="utf-8") as f: - lockfile = super(Lockfile, cls).load(f) - lockfile.newlines = preferred_newlines(f) - lockfile.path = lockfile_path - return lockfile + projectfile = cls.load_projectfile(path, create=create) + lockfile_path = Path(projectfile.location) + creation_args = { + "projectfile": projectfile, + "lockfile": projectfile.model, + "newlines": projectfile.line_ending, + "path": lockfile_path + } + return cls(**creation_args) + + @classmethod + def create(cls, path, create=True): + return cls.load(path, create=create) + + @property + def develop(self): + return self._lockfile.develop + + @property + def default(self): + return self._lockfile.default def get_requirements(self, dev=False): + """Produces a generator which generates requirements from the desired section. + + :param bool dev: Indicates whether to use dev requirements, defaults to False + :return: Requirements from the relevant the relevant pipfile + :rtype: :class:`~requirementslib.models.requirements.Requirement` + """ + section = self.develop if dev else self.default for k in section.keys(): yield Requirement.from_pipfile(k, section[k]._data) @@ -81,24 +161,26 @@ def requirements(self): @property def dev_requirements_list(self): - return [{name: entry._data} for name, entry in self.develop.items()] + return [{name: entry._data} for name, entry in self._lockfile.develop.items()] @property def requirements_list(self): - return [{name: entry._data} for name, entry in self.develop.items()] + return [{name: entry._data} for name, entry in self._lockfile.default.items()] def write(self): - open_kwargs = {"newline": self.newlines} - with atomic_open_for_write(self.path.as_posix(), **open_kwargs) as f: - super(Lockfile, self).dump(f, encoding="utf-8") + self.projectfile.model = copy.deepcopy(self._lockfile) + self.projectfile.write() def as_requirements(self, include_hashes=False, dev=False): """Returns a list of requirements in pip-style format""" lines = [] section = self.dev_requirements if dev else self.requirements for req in section: - r = req.as_line() - if not include_hashes: - r = r.split("--hash", 1)[0] + kwargs = { + "include_hashes": include_hashes, + } + if req.editable: + kwargs["include_markers"] = False + r = req.as_line(**kwargs) lines.append(r.strip()) return lines diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 3a6f5b1ee8..94e9a2a1f9 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -1,64 +1,144 @@ # -*- coding: utf-8 -*- -from vistir.compat import Path + +from __future__ import absolute_import, unicode_literals, print_function + +import attr +import copy +import os + +from vistir.compat import Path, FileNotFoundError from .requirements import Requirement +from .project import ProjectFile +from .utils import optional_instance_of from ..exceptions import RequirementError import plette.pipfiles -class Pipfile(plette.pipfiles.Pipfile): +is_pipfile = optional_instance_of(plette.pipfiles.Pipfile) +is_path = optional_instance_of(Path) +is_projectfile = optional_instance_of(ProjectFile) + + +@attr.s(slots=True) +class Pipfile(object): + path = attr.ib(validator=is_path, type=Path) + projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) + _pipfile = attr.ib(type=plette.pipfiles.Pipfile) + requirements = attr.ib(default=attr.Factory(list), type=list) + dev_requirements = attr.ib(default=attr.Factory(list), type=list) + + @path.default + def _get_path(self): + return Path(os.curdir).absolute() + + @projectfile.default + def _get_projectfile(self): + return self.load_projectfile(os.curdir, create=False) + + @_pipfile.default + def _get_pipfile(self): + return self.projectfile.model + + def __getattr__(self, k, *args, **kwargs): + retval = None + pipfile = super(Pipfile, self).__getattribute__("_pipfile") + try: + return super(Pipfile, self).__getattribute__(k) + except AttributeError: + retval = getattr(pipfile, k, None) + if not retval: + retval = super(Pipfile, self).__getattribute__(k, *args, **kwargs) + return retval @property def requires_python(self): - return self.requires.requires_python + return self._pipfile.requires.requires_python @property def allow_prereleases(self): - return self.get("pipenv", {}).get("allow_prereleases", False) + return self._pipfile.get("pipenv", {}).get("allow_prereleases", False) @classmethod - def load(cls, path): + def read_projectfile(cls, path): + """Read the specified project file and provide an interface for writing/updating. + + :param str path: Path to the target file. + :return: A project file with the model and location for interaction + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + pf = ProjectFile.read( + path, + plette.pipfiles.Pipfile, + invalid_ok=True + ) + return pf + + @classmethod + def load_projectfile(cls, path, create=False): + """Given a path, load or create the necessary pipfile. + + :param str path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A project file instance for the supplied project + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + if not path: + raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): path = Path(path) - pipfile_path = path / "Pipfile" - if not path.exists(): + pipfile_path = path if path.name == "Pipfile" else path.joinpath("Pipfile") + project_path = pipfile_path.parent + if not project_path.exists(): raise FileNotFoundError("%s is not a valid project path!" % path) elif not pipfile_path.exists() or not pipfile_path.is_file(): - raise RequirementError("%s is not a valid Pipfile" % pipfile_path) - with pipfile_path.open(encoding="utf-8") as fp: - pipfile = super(Pipfile, cls).load(fp) - pipfile.dev_requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.get("dev-packages", {}).items() + if not create: + raise RequirementError("%s is not a valid Pipfile" % pipfile_path) + return cls.read_projectfile(pipfile_path.as_posix()) + + @classmethod + def load(cls, path, create=False): + """Given a path, load or create the necessary pipfile. + + :param str path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A pipfile instance pointing at the supplied project + :rtype:: class:`~requirementslib.models.pipfile.Pipfile` + """ + + projectfile = cls.load_projectfile(path, create=create) + pipfile = projectfile.model + dev_requirements = [ + Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("dev-packages", {}).items() ] - pipfile.requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.get("packages", {}).items() + requirements = [ + Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("packages", {}).items() ] - pipfile.path = pipfile_path - return pipfile - - # def resolve(self): - # It would be nice to still use this api someday - # option_sources = [s.expanded for s in self.sources] - # pip_args = [] - # if self.pipenv.allow_prereleases: - # pip_args.append('--pre') - # pip_options = get_pip_options(pip_args, sources=option_sources) - # finder = get_finder(sources=option_sources, pip_options=pip_options) - # resolver = DependencyResolver.create(finder=finder, allow_prereleases=self.pipenv.allow_prereleases) - # pkg_dict = {} - # for pkg in self.dev_packages.requirements + self.packages.requirements: - # pkg_dict[pkg.name] = pkg - # resolver.resolve(list(pkg_dict.values())) - # return resolver + creation_args = { + "projectfile": projectfile, + "pipfile": pipfile, + "dev_requirements": dev_requirements, + "requirements": requirements, + "path": Path(projectfile.location) + } + return cls(**creation_args) + + def write(self): + self.projectfile.model = copy.deepcopy(self._pipfile) + self.projectfile.write() @property def dev_packages(self, as_requirements=True): if as_requirements: return self.dev_requirements - return self.get('dev-packages', {}) + return self._pipfile.get('dev-packages', {}) @property def packages(self, as_requirements=True): if as_requirements: return self.requirements - return self.get('packages', {}) + return self._pipfile.get('packages', {}) diff --git a/pipenv/vendor/requirementslib/models/project.py b/pipenv/vendor/requirementslib/models/project.py new file mode 100644 index 0000000000..f6e037d651 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/project.py @@ -0,0 +1,241 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, unicode_literals + +import collections +import io +import os + +import attr +import packaging.markers +import packaging.utils +import plette +import plette.models +import six +import tomlkit + + +SectionDifference = collections.namedtuple("SectionDifference", [ + "inthis", "inthat", +]) +FileDifference = collections.namedtuple("FileDifference", [ + "default", "develop", +]) + + +def _are_pipfile_entries_equal(a, b): + a = {k: v for k, v in a.items() if k not in ("markers", "hashes", "hash")} + b = {k: v for k, v in b.items() if k not in ("markers", "hashes", "hash")} + if a != b: + return False + try: + marker_eval_a = packaging.markers.Marker(a["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_a = True + try: + marker_eval_b = packaging.markers.Marker(b["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_b = True + return marker_eval_a == marker_eval_b + + +DEFAULT_NEWLINES = "\n" + + +def preferred_newlines(f): + if isinstance(f.newlines, six.text_type): + return f.newlines + return DEFAULT_NEWLINES + + +@attr.s +class ProjectFile(object): + """A file in the Pipfile project. + """ + location = attr.ib() + line_ending = attr.ib() + model = attr.ib() + + @classmethod + def read(cls, location, model_cls, invalid_ok=False): + try: + with io.open(location, encoding="utf-8") as f: + model = model_cls.load(f) + line_ending = preferred_newlines(f) + except Exception: + if not invalid_ok: + raise + model = None + line_ending = DEFAULT_NEWLINES + return cls(location=location, line_ending=line_ending, model=model) + + def write(self): + kwargs = {"encoding": "utf-8", "newline": self.line_ending} + with io.open(self.location, "w", **kwargs) as f: + self.model.dump(f) + + def dumps(self): + strio = six.StringIO() + self.model.dump(strio) + return strio.getvalue() + + +@attr.s +class Project(object): + + root = attr.ib() + _p = attr.ib(init=False) + _l = attr.ib(init=False) + + def __attrs_post_init__(self): + self.root = root = os.path.abspath(self.root) + self._p = ProjectFile.read( + os.path.join(root, "Pipfile"), + plette.Pipfile, + ) + self._l = ProjectFile.read( + os.path.join(root, "Pipfile.lock"), + plette.Lockfile, + invalid_ok=True, + ) + + @property + def pipfile(self): + return self._p.model + + @property + def pipfile_location(self): + return self._p.location + + @property + def lockfile(self): + return self._l.model + + @property + def lockfile_location(self): + return self._l.location + + @lockfile.setter + def lockfile(self, new): + self._l.model = new + + def is_synced(self): + return self.lockfile and self.lockfile.is_up_to_date(self.pipfile) + + def _get_pipfile_section(self, develop, insert=True): + name = "dev-packages" if develop else "packages" + try: + section = self.pipfile[name] + except KeyError: + section = plette.models.PackageCollection(tomlkit.table()) + if insert: + self.pipfile[name] = section + return section + + def contains_key_in_pipfile(self, key): + sections = [ + self._get_pipfile_section(develop=False, insert=False), + self._get_pipfile_section(develop=True, insert=False), + ] + return any( + (packaging.utils.canonicalize_name(name) == + packaging.utils.canonicalize_name(key)) + for section in sections + for name in section + ) + + def add_line_to_pipfile(self, line, develop): + from requirementslib import Requirement + requirement = Requirement.from_line(line) + section = self._get_pipfile_section(develop=develop) + key = requirement.normalized_name + entry = next(iter(requirement.as_pipfile().values())) + if isinstance(entry, dict): + # HACK: TOMLKit prefers to expand tables by default, but we + # always want inline tables here. Also tomlkit.inline_table + # does not have `update()`. + table = tomlkit.inline_table() + for k, v in entry.items(): + table[k] = v + entry = table + section[key] = entry + + def remove_keys_from_pipfile(self, keys, default, develop): + keys = {packaging.utils.canonicalize_name(key) for key in keys} + sections = [] + if default: + sections.append(self._get_pipfile_section( + develop=False, insert=False, + )) + if develop: + sections.append(self._get_pipfile_section( + develop=True, insert=False, + )) + for section in sections: + removals = set() + for name in section: + if packaging.utils.canonicalize_name(name) in keys: + removals.add(name) + for key in removals: + del section._data[key] + + def remove_keys_from_lockfile(self, keys): + keys = {packaging.utils.canonicalize_name(key) for key in keys} + removed = False + for section_name in ("default", "develop"): + try: + section = self.lockfile[section_name] + except KeyError: + continue + removals = set() + for name in section: + if packaging.utils.canonicalize_name(name) in keys: + removals.add(name) + removed = removed or bool(removals) + for key in removals: + del section._data[key] + + if removed: + # HACK: The lock file no longer represents the Pipfile at this + # point. Set the hash to an arbitrary invalid value. + self.lockfile.meta.hash = plette.models.Hash({"__invalid__": ""}) + + def difference_lockfile(self, lockfile): + """Generate a difference between the current and given lockfiles. + + Returns a 2-tuple containing differences in default in develop + sections. + + Each element is a 2-tuple of dicts. The first, `inthis`, contains + entries only present in the current lockfile; the second, `inthat`, + contains entries only present in the given one. + + If a key exists in both this and that, but the values differ, the key + is present in both dicts, pointing to values from each file. + """ + diff_data = { + "default": SectionDifference({}, {}), + "develop": SectionDifference({}, {}), + } + for section_name, section_diff in diff_data.items(): + try: + this = self.lockfile[section_name]._data + except (KeyError, TypeError): + this = {} + try: + that = lockfile[section_name]._data + except (KeyError, TypeError): + that = {} + for key, this_value in this.items(): + try: + that_value = that[key] + except KeyError: + section_diff.inthis[key] = this_value + continue + if not _are_pipfile_entries_equal(this_value, that_value): + section_diff.inthis[key] = this_value + section_diff.inthat[key] = that_value + for key, that_value in that.items(): + if key not in this: + section_diff.inthat[key] = that_value + return FileDifference(**diff_data) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 248ca77786..c2768417c3 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- + from __future__ import absolute_import -import atexit import collections import hashlib import os @@ -9,6 +9,7 @@ from contextlib import contextmanager import attr +import six from first import first from packaging.markers import Marker @@ -42,7 +43,8 @@ is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, specs_to_string, split_markers_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs + validate_specifiers, validate_vcs, normalize_name, + Requirement as PkgResourcesRequirement ) from .vcs import VCSRepository @@ -99,7 +101,7 @@ def line_part(self): # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index - return "{0}".format(self.name.lower().replace("_", "-")) + return "{0}".format(normalize_name(self.name)) @property def pipfile_part(self): @@ -123,12 +125,12 @@ class FileRequirement(BaseRequirement): setup_path = attr.ib(default=None) path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) # : path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - editable = attr.ib(default=None) - extras = attr.ib(default=attr.Factory(list)) - uri = attr.ib() - link = attr.ib() - name = attr.ib() - req = attr.ib() + editable = attr.ib(default=False, type=bool) + extras = attr.ib(default=attr.Factory(list), type=list) + uri = attr.ib(type=six.string_types) + link = attr.ib(type=Link) + name = attr.ib(type=six.string_types) + req = attr.ib(type=PkgResourcesRequirement) _has_hashed_name = False _uri_scheme = attr.ib(default=None) @@ -297,7 +299,7 @@ def get_link(self): @req.default def get_requirement(self): - req = init_requirement(canonicalize_name(self.name)) + req = init_requirement(normalize_name(self.name)) req.editable = False req.line = self.link.url_without_fragment if self.path and self.link and self.link.scheme.startswith("file"): @@ -948,7 +950,8 @@ def from_pipfile(cls, name, pipfile): cls_inst.req.req.line = cls_inst.as_line() return cls_inst - def as_line(self, sources=None, include_hashes=True, include_extras=True, as_list=False): + def as_line(self, sources=None, include_hashes=True, include_extras=True, + include_markers=True, as_list=False): """Format this requirement as a line in requirements.txt. If ``sources`` provided, it should be an sequence of mappings, containing @@ -967,7 +970,7 @@ def as_line(self, sources=None, include_hashes=True, include_extras=True, as_lis self.req.line_part, self.extras_as_pip if include_extras else "", self.specifiers if include_specifiers else "", - self.markers_as_pip, + self.markers_as_pip if include_markers else "", ] if as_list: # This is used for passing to a subprocess call diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index cba63295ac..7350d0ac08 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -508,3 +508,15 @@ def fix_requires_python_marker(requires_python): ]) marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker return marker_to_add + + +def normalize_name(pkg): + """Given a package name, return its normalized, non-canonicalized form. + + :param str pkg: The name of a package + :return: A normalized package name + :rtype: str + """ + + assert isinstance(pkg, six.string_types) + return pkg.replace("_", "-").lower() diff --git a/pipenv/vendor/shellingham/__init__.py b/pipenv/vendor/shellingham/__init__.py index 90c00abbdc..576c422496 100644 --- a/pipenv/vendor/shellingham/__init__.py +++ b/pipenv/vendor/shellingham/__init__.py @@ -4,7 +4,7 @@ from ._core import ShellDetectionFailure -__version__ = '1.2.6' +__version__ = '1.2.7' def detect_shell(pid=None, max_depth=6): diff --git a/pipenv/vendor/shellingham/posix/ps.py b/pipenv/vendor/shellingham/posix/ps.py index ab4c2a9efa..4a155ed5cb 100644 --- a/pipenv/vendor/shellingham/posix/ps.py +++ b/pipenv/vendor/shellingham/posix/ps.py @@ -21,6 +21,12 @@ def get_process_mapping(): if e.errno != errno.ENOENT: raise raise PsNotAvailable('ps not found') + except subprocess.CalledProcessError as e: + # `ps` can return 1 if the process list is completely empty. + # (sarugaku/shellingham#15) + if not e.output.strip(): + return {} + raise if not isinstance(output, str): encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() output = output.decode(encoding) @@ -28,9 +34,9 @@ def get_process_mapping(): for line in output.split('\n'): try: pid, ppid, args = line.strip().split(None, 2) + processes[pid] = Process( + args=tuple(shlex.split(args)), pid=pid, ppid=ppid, + ) except ValueError: continue - processes[pid] = Process( - args=tuple(shlex.split(args)), pid=pid, ppid=ppid, - ) return processes diff --git a/pipenv/vendor/toml.LICENSE b/pipenv/vendor/toml.LICENSE deleted file mode 100644 index d8b406c90a..0000000000 --- a/pipenv/vendor/toml.LICENSE +++ /dev/null @@ -1,26 +0,0 @@ -The MIT License - -Copyright 2013-2017 Uiri Noyb -Copyright 2015-2016 Julien Enselme -Copyright 2016 Google Inc. -Copyright 2017 Samuel Vasko -Copyright 2017 Nate Prewitt -Copyright 2017 Jack Evans - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file diff --git a/pipenv/vendor/toml.py b/pipenv/vendor/toml.py deleted file mode 100644 index dac398837b..0000000000 --- a/pipenv/vendor/toml.py +++ /dev/null @@ -1,1039 +0,0 @@ -"""Python module which parses and emits TOML. - -Released under the MIT license. -""" -import re -import io -import datetime -from os import linesep -import sys - -__version__ = "0.9.6" -_spec_ = "0.4.0" - - -class TomlDecodeError(Exception): - """Base toml Exception / Error.""" - pass - - -class TomlTz(datetime.tzinfo): - def __init__(self, toml_offset): - if toml_offset == "Z": - self._raw_offset = "+00:00" - else: - self._raw_offset = toml_offset - self._sign = -1 if self._raw_offset[0] == '-' else 1 - self._hours = int(self._raw_offset[1:3]) - self._minutes = int(self._raw_offset[4:6]) - - def tzname(self, dt): - return "UTC" + self._raw_offset - - def utcoffset(self, dt): - return self._sign * datetime.timedelta(hours=self._hours, - minutes=self._minutes) - - def dst(self, dt): - return datetime.timedelta(0) - - -class InlineTableDict(object): - """Sentinel subclass of dict for inline tables.""" - - -def _get_empty_inline_table(_dict): - class DynamicInlineTableDict(_dict, InlineTableDict): - """Concrete sentinel subclass for inline tables. - It is a subclass of _dict which is passed in dynamically at load time - It is also a subclass of InlineTableDict - """ - - return DynamicInlineTableDict() - - -try: - _range = xrange -except NameError: - unicode = str - _range = range - basestring = str - unichr = chr - -try: - FNFError = FileNotFoundError -except NameError: - FNFError = IOError - - -def load(f, _dict=dict): - """Parses named file or files as toml and returns a dictionary - - Args: - f: Path to the file to open, array of files to read into single dict - or a file descriptor - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError -- When f is invalid type - TomlDecodeError: Error while decoding toml - IOError / FileNotFoundError -- When an array with no valid (existing) - (Python 2 / Python 3) file paths is passed - """ - - if isinstance(f, basestring): - with io.open(f, encoding='utf-8') as ffile: - return loads(ffile.read(), _dict) - elif isinstance(f, list): - from os import path as op - from warnings import warn - if not [path for path in f if op.exists(path)]: - error_msg = "Load expects a list to contain filenames only." - error_msg += linesep - error_msg += ("The list needs to contain the path of at least one " - "existing file.") - raise FNFError(error_msg) - d = _dict() - for l in f: - if op.exists(l): - d.update(load(l)) - else: - warn("Non-existent filename in list with at least one valid " - "filename") - return d - else: - try: - return loads(f.read(), _dict) - except AttributeError: - raise TypeError("You can only load a file descriptor, filename or " - "list") - - -_groupname_re = re.compile(r'^[A-Za-z0-9_-]+$') - - -def loads(s, _dict=dict): - """Parses string as toml - - Args: - s: String to be parsed - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError: When a non-string is passed - TomlDecodeError: Error while decoding toml - """ - - implicitgroups = [] - retval = _dict() - currentlevel = retval - if not isinstance(s, basestring): - raise TypeError("Expecting something like a string") - - if not isinstance(s, unicode): - s = s.decode('utf8') - - sl = list(s) - openarr = 0 - openstring = False - openstrchar = "" - multilinestr = False - arrayoftables = False - beginline = True - keygroup = False - keyname = 0 - for i, item in enumerate(sl): - if item == '\r' and sl[i + 1] == '\n': - sl[i] = ' ' - continue - if keyname: - if item == '\n': - raise TomlDecodeError("Key name found without value." - " Reached end of line.") - if openstring: - if item == openstrchar: - keyname = 2 - openstring = False - openstrchar = "" - continue - elif keyname == 1: - if item.isspace(): - keyname = 2 - continue - elif item.isalnum() or item == '_' or item == '-': - continue - elif keyname == 2 and item.isspace(): - continue - if item == '=': - keyname = 0 - else: - raise TomlDecodeError("Found invalid character in key name: '" + - item + "'. Try quoting the key name.") - if item == "'" and openstrchar != '"': - k = 1 - try: - while sl[i - k] == "'": - k += 1 - if k == 3: - break - except IndexError: - pass - if k == 3: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = "'" - else: - openstrchar = "" - if item == '"' and openstrchar != "'": - oddbackslash = False - k = 1 - tripquote = False - try: - while sl[i - k] == '"': - k += 1 - if k == 3: - tripquote = True - break - if k == 1 or (k == 3 and tripquote): - while sl[i - k] == '\\': - oddbackslash = not oddbackslash - k += 1 - except IndexError: - pass - if not oddbackslash: - if tripquote: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = '"' - else: - openstrchar = "" - if item == '#' and (not openstring and not keygroup and - not arrayoftables): - j = i - try: - while sl[j] != '\n': - sl[j] = ' ' - j += 1 - except IndexError: - break - if item == '[' and (not openstring and not keygroup and - not arrayoftables): - if beginline: - if len(sl) > i + 1 and sl[i + 1] == '[': - arrayoftables = True - else: - keygroup = True - else: - openarr += 1 - if item == ']' and not openstring: - if keygroup: - keygroup = False - elif arrayoftables: - if sl[i - 1] == ']': - arrayoftables = False - else: - openarr -= 1 - if item == '\n': - if openstring or multilinestr: - if not multilinestr: - raise TomlDecodeError("Unbalanced quotes") - if ((sl[i - 1] == "'" or sl[i - 1] == '"') and ( - sl[i - 2] == sl[i - 1])): - sl[i] = sl[i - 1] - if sl[i - 3] == sl[i - 1]: - sl[i - 3] = ' ' - elif openarr: - sl[i] = ' ' - else: - beginline = True - elif beginline and sl[i] != ' ' and sl[i] != '\t': - beginline = False - if not keygroup and not arrayoftables: - if sl[i] == '=': - raise TomlDecodeError("Found empty keyname. ") - keyname = 1 - s = ''.join(sl) - s = s.split('\n') - multikey = None - multilinestr = "" - multibackslash = False - for line in s: - if not multilinestr or multibackslash or '\n' not in multilinestr: - line = line.strip() - if line == "" and (not multikey or multibackslash): - continue - if multikey: - if multibackslash: - multilinestr += line - else: - multilinestr += line - multibackslash = False - if len(line) > 2 and (line[-1] == multilinestr[0] and - line[-2] == multilinestr[0] and - line[-3] == multilinestr[0]): - try: - value, vtype = _load_value(multilinestr, _dict) - except ValueError as err: - raise TomlDecodeError(str(err)) - currentlevel[multikey] = value - multikey = None - multilinestr = "" - else: - k = len(multilinestr) - 1 - while k > -1 and multilinestr[k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = multilinestr[:-1] - else: - multilinestr += "\n" - continue - if line[0] == '[': - arrayoftables = False - if len(line) == 1: - raise TomlDecodeError("Opening key group bracket on line by " - "itself.") - if line[1] == '[': - arrayoftables = True - line = line[2:] - splitstr = ']]' - else: - line = line[1:] - splitstr = ']' - i = 1 - quotesplits = _get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and splitstr in quotesplit: - break - i += quotesplit.count(splitstr) - quoted = not quoted - line = line.split(splitstr, i) - if len(line) < i + 1 or line[-1].strip() != "": - raise TomlDecodeError("Key group not on a line by itself.") - groups = splitstr.join(line[:-1]).split('.') - i = 0 - while i < len(groups): - groups[i] = groups[i].strip() - if len(groups[i]) > 0 and (groups[i][0] == '"' or - groups[i][0] == "'"): - groupstr = groups[i] - j = i + 1 - while not groupstr[0] == groupstr[-1]: - j += 1 - if j > len(groups) + 2: - raise TomlDecodeError("Invalid group name '" + - groupstr + "' Something " + - "went wrong.") - groupstr = '.'.join(groups[i:j]).strip() - groups[i] = groupstr[1:-1] - groups[i + 1:j] = [] - else: - if not _groupname_re.match(groups[i]): - raise TomlDecodeError("Invalid group name '" + - groups[i] + "'. Try quoting it.") - i += 1 - currentlevel = retval - for i in _range(len(groups)): - group = groups[i] - if group == "": - raise TomlDecodeError("Can't have a keygroup with an empty " - "name") - try: - currentlevel[group] - if i == len(groups) - 1: - if group in implicitgroups: - implicitgroups.remove(group) - if arrayoftables: - raise TomlDecodeError("An implicitly defined " - "table can't be an array") - elif arrayoftables: - currentlevel[group].append(_dict()) - else: - raise TomlDecodeError("What? " + group + - " already exists?" + - str(currentlevel)) - except TypeError: - currentlevel = currentlevel[-1] - try: - currentlevel[group] - except KeyError: - currentlevel[group] = _dict() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [_dict()] - except KeyError: - if i != len(groups) - 1: - implicitgroups.append(group) - currentlevel[group] = _dict() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [_dict()] - currentlevel = currentlevel[group] - if arrayoftables: - try: - currentlevel = currentlevel[-1] - except KeyError: - pass - elif line[0] == "{": - if line[-1] != "}": - raise TomlDecodeError("Line breaks are not allowed in inline" - "objects") - try: - _load_inline_object(line, currentlevel, _dict, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err)) - elif "=" in line: - try: - ret = _load_line(line, currentlevel, _dict, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err)) - if ret is not None: - multikey, multilinestr, multibackslash = ret - return retval - - -def _load_inline_object(line, currentlevel, _dict, multikey=False, - multibackslash=False): - candidate_groups = line[1:-1].split(",") - groups = [] - if len(candidate_groups) == 1 and not candidate_groups[0].strip(): - candidate_groups.pop() - while len(candidate_groups) > 0: - candidate_group = candidate_groups.pop(0) - try: - _, value = candidate_group.split('=', 1) - except ValueError: - raise ValueError("Invalid inline table encountered") - value = value.strip() - if ((value[0] == value[-1] and value[0] in ('"', "'")) or ( - value[0] in '-0123456789' or - value in ('true', 'false') or - (value[0] == "[" and value[-1] == "]") or - (value[0] == '{' and value[-1] == '}'))): - groups.append(candidate_group) - elif len(candidate_groups) > 0: - candidate_groups[0] = candidate_group + "," + candidate_groups[0] - else: - raise ValueError("Invalid inline table value encountered") - for group in groups: - status = _load_line(group, currentlevel, _dict, multikey, - multibackslash) - if status is not None: - break - - -# Matches a TOML number, which allows underscores for readability -_number_with_underscores = re.compile('([0-9])(_([0-9]))*') - - -def _strictly_valid_num(n): - n = n.strip() - if not n: - return False - if n[0] == '_': - return False - if n[-1] == '_': - return False - if "_." in n or "._" in n: - return False - if len(n) == 1: - return True - if n[0] == '0' and n[1] != '.': - return False - if n[0] == '+' or n[0] == '-': - n = n[1:] - if n[0] == '0' and n[1] != '.': - return False - if '__' in n: - return False - return True - - -def _get_split_on_quotes(line): - doublequotesplits = line.split('"') - quoted = False - quotesplits = [] - if len(doublequotesplits) > 1 and "'" in doublequotesplits[0]: - singlequotesplits = doublequotesplits[0].split("'") - doublequotesplits = doublequotesplits[1:] - while len(singlequotesplits) % 2 == 0 and len(doublequotesplits): - singlequotesplits[-1] += '"' + doublequotesplits[0] - doublequotesplits = doublequotesplits[1:] - if "'" in singlequotesplits[-1]: - singlequotesplits = (singlequotesplits[:-1] + - singlequotesplits[-1].split("'")) - quotesplits += singlequotesplits - for doublequotesplit in doublequotesplits: - if quoted: - quotesplits.append(doublequotesplit) - else: - quotesplits += doublequotesplit.split("'") - quoted = not quoted - return quotesplits - - -def _load_line(line, currentlevel, _dict, multikey, multibackslash): - i = 1 - quotesplits = _get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and '=' in quotesplit: - break - i += quotesplit.count('=') - quoted = not quoted - pair = line.split('=', i) - strictly_valid = _strictly_valid_num(pair[-1]) - if _number_with_underscores.match(pair[-1]): - pair[-1] = pair[-1].replace('_', '') - while len(pair[-1]) and (pair[-1][0] != ' ' and pair[-1][0] != '\t' and - pair[-1][0] != "'" and pair[-1][0] != '"' and - pair[-1][0] != '[' and pair[-1][0] != '{' and - pair[-1] != 'true' and pair[-1] != 'false'): - try: - float(pair[-1]) - break - except ValueError: - pass - if _load_date(pair[-1]) is not None: - break - i += 1 - prev_val = pair[-1] - pair = line.split('=', i) - if prev_val == pair[-1]: - raise ValueError("Invalid date or number") - if strictly_valid: - strictly_valid = _strictly_valid_num(pair[-1]) - pair = ['='.join(pair[:-1]).strip(), pair[-1].strip()] - if (pair[0][0] == '"' or pair[0][0] == "'") and \ - (pair[0][-1] == '"' or pair[0][-1] == "'"): - pair[0] = pair[0][1:-1] - if len(pair[1]) > 2 and ((pair[1][0] == '"' or pair[1][0] == "'") and - pair[1][1] == pair[1][0] and - pair[1][2] == pair[1][0] and - not (len(pair[1]) > 5 and - pair[1][-1] == pair[1][0] and - pair[1][-2] == pair[1][0] and - pair[1][-3] == pair[1][0])): - k = len(pair[1]) - 1 - while k > -1 and pair[1][k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = pair[1][:-1] - else: - multilinestr = pair[1] + "\n" - multikey = pair[0] - else: - value, vtype = _load_value(pair[1], _dict, strictly_valid) - try: - currentlevel[pair[0]] - raise ValueError("Duplicate keys!") - except KeyError: - if multikey: - return multikey, multilinestr, multibackslash - else: - currentlevel[pair[0]] = value - - -def _load_date(val): - microsecond = 0 - tz = None - try: - if len(val) > 19: - if val[19] == '.': - if val[-1].upper() == 'Z': - subsecondval = val[20:-1] - tzval = "Z" - else: - subsecondvalandtz = val[20:] - if '+' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('+') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - elif '-' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('-') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - tz = TomlTz(tzval) - microsecond = int(int(subsecondval) * - (10 ** (6 - len(subsecondval)))) - else: - tz = TomlTz(val[19:]) - except ValueError: - tz = None - if "-" not in val[1:]: - return None - try: - d = datetime.datetime( - int(val[:4]), int(val[5:7]), - int(val[8:10]), int(val[11:13]), - int(val[14:16]), int(val[17:19]), microsecond, tz) - except ValueError: - return None - return d - - -def _load_unicode_escapes(v, hexbytes, prefix): - skip = False - i = len(v) - 1 - while i > -1 and v[i] == '\\': - skip = not skip - i -= 1 - for hx in hexbytes: - if skip: - skip = False - i = len(hx) - 1 - while i > -1 and hx[i] == '\\': - skip = not skip - i -= 1 - v += prefix - v += hx - continue - hxb = "" - i = 0 - hxblen = 4 - if prefix == "\\U": - hxblen = 8 - hxb = ''.join(hx[i:i + hxblen]).lower() - if hxb.strip('0123456789abcdef'): - raise ValueError("Invalid escape sequence: " + hxb) - if hxb[0] == "d" and hxb[1].strip('01234567'): - raise ValueError("Invalid escape sequence: " + hxb + - ". Only scalar unicode points are allowed.") - v += unichr(int(hxb, 16)) - v += unicode(hx[len(hxb):]) - return v - - -# Unescape TOML string values. - -# content after the \ -_escapes = ['0', 'b', 'f', 'n', 'r', 't', '"'] -# What it should be replaced by -_escapedchars = ['\0', '\b', '\f', '\n', '\r', '\t', '\"'] -# Used for substitution -_escape_to_escapedchars = dict(zip(_escapes, _escapedchars)) - - -def _unescape(v): - """Unescape characters in a TOML string.""" - i = 0 - backslash = False - while i < len(v): - if backslash: - backslash = False - if v[i] in _escapes: - v = v[:i - 1] + _escape_to_escapedchars[v[i]] + v[i + 1:] - elif v[i] == '\\': - v = v[:i - 1] + v[i:] - elif v[i] == 'u' or v[i] == 'U': - i += 1 - else: - raise ValueError("Reserved escape sequence used") - continue - elif v[i] == '\\': - backslash = True - i += 1 - return v - - -def _load_value(v, _dict, strictly_valid=True): - if not v: - raise ValueError("Empty value is invalid") - if v == 'true': - return (True, "bool") - elif v == 'false': - return (False, "bool") - elif v[0] == '"': - testv = v[1:].split('"') - triplequote = False - triplequotecount = 0 - if len(testv) > 1 and testv[0] == '' and testv[1] == '': - testv = testv[2:] - triplequote = True - closed = False - for tv in testv: - if tv == '': - if triplequote: - triplequotecount += 1 - else: - closed = True - else: - oddbackslash = False - try: - i = -1 - j = tv[i] - while j == '\\': - oddbackslash = not oddbackslash - i -= 1 - j = tv[i] - except IndexError: - pass - if not oddbackslash: - if closed: - raise ValueError("Stuff after closed string. WTF?") - else: - if not triplequote or triplequotecount > 1: - closed = True - else: - triplequotecount = 0 - escapeseqs = v.split('\\')[1:] - backslash = False - for i in escapeseqs: - if i == '': - backslash = not backslash - else: - if i[0] not in _escapes and (i[0] != 'u' and i[0] != 'U' and - not backslash): - raise ValueError("Reserved escape sequence used") - if backslash: - backslash = False - for prefix in ["\\u", "\\U"]: - if prefix in v: - hexbytes = v.split(prefix) - v = _load_unicode_escapes(hexbytes[0], hexbytes[1:], prefix) - v = _unescape(v) - if len(v) > 1 and v[1] == '"' and (len(v) < 3 or v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == "'": - if v[1] == "'" and (len(v) < 3 or v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == '[': - return (_load_array(v, _dict), "array") - elif v[0] == '{': - inline_object = _get_empty_inline_table(_dict) - _load_inline_object(v, inline_object, _dict) - return (inline_object, "inline_object") - else: - parsed_date = _load_date(v) - if parsed_date is not None: - return (parsed_date, "date") - if not strictly_valid: - raise ValueError("Weirdness with leading zeroes or " - "underscores in your number.") - itype = "int" - neg = False - if v[0] == '-': - neg = True - v = v[1:] - elif v[0] == '+': - v = v[1:] - v = v.replace('_', '') - if '.' in v or 'e' in v or 'E' in v: - if '.' in v and v.split('.', 1)[1] == '': - raise ValueError("This float is missing digits after " - "the point") - if v[0] not in '0123456789': - raise ValueError("This float doesn't have a leading digit") - v = float(v) - itype = "float" - else: - v = int(v) - if neg: - return (0 - v, itype) - return (v, itype) - - -def _bounded_string(s): - if len(s) == 0: - return True - if s[-1] != s[0]: - return False - i = -2 - backslash = False - while len(s) + i > 0: - if s[i] == "\\": - backslash = not backslash - i -= 1 - else: - break - return not backslash - - -def _load_array(a, _dict): - atype = None - retval = [] - a = a.strip() - if '[' not in a[1:-1] or "" != a[1:-1].split('[')[0].strip(): - strarray = False - tmpa = a[1:-1].strip() - if tmpa != '' and (tmpa[0] == '"' or tmpa[0] == "'"): - strarray = True - if not a[1:-1].strip().startswith('{'): - a = a[1:-1].split(',') - else: - # a is an inline object, we must find the matching parenthesis - # to define groups - new_a = [] - start_group_index = 1 - end_group_index = 2 - in_str = False - while end_group_index < len(a[1:]): - if a[end_group_index] == '"' or a[end_group_index] == "'": - if in_str: - backslash_index = end_group_index - 1 - while (backslash_index > -1 and - a[backslash_index] == '\\'): - in_str = not in_str - backslash_index -= 1 - in_str = not in_str - if in_str or a[end_group_index] != '}': - end_group_index += 1 - continue - - # Increase end_group_index by 1 to get the closing bracket - end_group_index += 1 - new_a.append(a[start_group_index:end_group_index]) - - # The next start index is at least after the closing bracket, a - # closing bracket can be followed by a comma since we are in - # an array. - start_group_index = end_group_index + 1 - while (start_group_index < len(a[1:]) and - a[start_group_index] != '{'): - start_group_index += 1 - end_group_index = start_group_index + 1 - a = new_a - b = 0 - if strarray: - while b < len(a) - 1: - ab = a[b].strip() - while (not _bounded_string(ab) or - (len(ab) > 2 and - ab[0] == ab[1] == ab[2] and - ab[-2] != ab[0] and - ab[-3] != ab[0])): - a[b] = a[b] + ',' + a[b + 1] - ab = a[b].strip() - if b < len(a) - 2: - a = a[:b + 1] + a[b + 2:] - else: - a = a[:b + 1] - b += 1 - else: - al = list(a[1:-1]) - a = [] - openarr = 0 - j = 0 - for i in _range(len(al)): - if al[i] == '[': - openarr += 1 - elif al[i] == ']': - openarr -= 1 - elif al[i] == ',' and not openarr: - a.append(''.join(al[j:i])) - j = i + 1 - a.append(''.join(al[j:])) - for i in _range(len(a)): - a[i] = a[i].strip() - if a[i] != '': - nval, ntype = _load_value(a[i], _dict) - if atype: - if ntype != atype: - raise ValueError("Not a homogeneous array") - else: - atype = ntype - retval.append(nval) - return retval - - -def dump(o, f): - """Writes out dict as toml to a file - - Args: - o: Object to dump into toml - f: File descriptor where the toml should be stored - - Returns: - String containing the toml corresponding to dictionary - - Raises: - TypeError: When anything other than file descriptor is passed - """ - - if not f.write: - raise TypeError("You can only dump an object to a file descriptor") - d = dumps(o) - f.write(d) - return d - - -def dumps(o, preserve=False): - """Stringifies input dict as toml - - Args: - o: Object to dump into toml - - preserve: Boolean parameter. If true, preserve inline tables. - - Returns: - String containing the toml corresponding to dict - """ - - retval = "" - addtoretval, sections = _dump_sections(o, "") - retval += addtoretval - while sections != {}: - newsections = {} - for section in sections: - addtoretval, addtosections = _dump_sections(sections[section], - section, preserve) - if addtoretval or (not addtoretval and not addtosections): - if retval and retval[-2:] != "\n\n": - retval += "\n" - retval += "[" + section + "]\n" - if addtoretval: - retval += addtoretval - for s in addtosections: - newsections[section + "." + s] = addtosections[s] - sections = newsections - return retval - - -def _dump_sections(o, sup, preserve=False): - retstr = "" - if sup != "" and sup[-1] != ".": - sup += '.' - retdict = o.__class__() - arraystr = "" - for section in o: - section = unicode(section) - qsection = section - if not re.match(r'^[A-Za-z0-9_-]+$', section): - if '"' in section: - qsection = "'" + section + "'" - else: - qsection = '"' + section + '"' - if not isinstance(o[section], dict): - arrayoftables = False - if isinstance(o[section], list): - for a in o[section]: - if isinstance(a, dict): - arrayoftables = True - if arrayoftables: - for a in o[section]: - arraytabstr = "\n" - arraystr += "[[" + sup + qsection + "]]\n" - s, d = _dump_sections(a, sup + qsection) - if s: - if s[0] == "[": - arraytabstr += s - else: - arraystr += s - while d != {}: - newd = {} - for dsec in d: - s1, d1 = _dump_sections(d[dsec], sup + qsection + - "." + dsec) - if s1: - arraytabstr += ("[" + sup + qsection + "." + - dsec + "]\n") - arraytabstr += s1 - for s1 in d1: - newd[dsec + "." + s1] = d1[s1] - d = newd - arraystr += arraytabstr - else: - if o[section] is not None: - retstr += (qsection + " = " + - unicode(_dump_value(o[section])) + '\n') - elif preserve and isinstance(o[section], InlineTableDict): - retstr += (qsection + " = " + _dump_inline_table(o[section])) - else: - retdict[qsection] = o[section] - retstr += arraystr - return (retstr, retdict) - - -def _dump_inline_table(section): - """Preserve inline table in its compact syntax instead of expanding - into subsection. - - https://github.com/toml-lang/toml#user-content-inline-table - """ - retval = "" - if isinstance(section, dict): - val_list = [] - for k, v in section.items(): - val = _dump_inline_table(v) - val_list.append(k + " = " + val) - retval += "{ " + ", ".join(val_list) + " }\n" - return retval - else: - return unicode(_dump_value(section)) - - -def _dump_value(v): - dump_funcs = { - str: _dump_str, - unicode: _dump_str, - list: _dump_list, - int: lambda v: v, - bool: lambda v: unicode(v).lower(), - float: _dump_float, - datetime.datetime: lambda v: v.isoformat().replace('+00:00', 'Z'), - } - # Lookup function corresponding to v's type - dump_fn = dump_funcs.get(type(v)) - if dump_fn is None and hasattr(v, '__iter__'): - dump_fn = dump_funcs[list] - # Evaluate function (if it exists) else return v - return dump_fn(v) if dump_fn is not None else dump_funcs[str](v) - - -def _dump_str(v): - if sys.version_info < (3,) and hasattr(v, 'decode') and isinstance(v, str): - v = v.decode('utf-8') - v = "%r" % v - if v[0] == 'u': - v = v[1:] - singlequote = v.startswith("'") - if singlequote or v.startswith('"'): - v = v[1:-1] - if singlequote: - v = v.replace("\\'", "'") - v = v.replace('"', '\\"') - v = v.split("\\x") - while len(v) > 1: - i = -1 - if not v[0]: - v = v[1:] - v[0] = v[0].replace("\\\\", "\\") - # No, I don't know why != works and == breaks - joinx = v[0][i] != "\\" - while v[0][:i] and v[0][i] == "\\": - joinx = not joinx - i -= 1 - if joinx: - joiner = "x" - else: - joiner = "u00" - v = [v[0] + joiner + v[1]] + v[2:] - return unicode('"' + v[0] + '"') - - -def _dump_list(v): - retval = "[" - for u in v: - retval += " " + unicode(_dump_value(u)) + "," - retval += "]" - return retval - - -def _dump_float(v): - return "{0:.16}".format(v).replace("e+0", "e+").replace("e-0", "e-") diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index 23d4ef7461..89e4cf595e 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import ws -__version__ = "0.4.4" +__version__ = "0.4.6" diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index bb3696d984..3701492162 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -4,7 +4,6 @@ from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey from .items import AoT -from .items import Bool from .items import Comment from .items import Item from .items import Key @@ -525,3 +524,21 @@ def __eq__(self, other): # type: (Dict) -> bool return NotImplemented return self.value == other + + def _getstate(self, protocol): + return (self._parsed,) + + def __reduce__(self): + return self.__reduce_ex__(2) + + def __reduce_ex__(self, protocol): + return ( + self.__class__, + self._getstate(protocol), + (self._map, self._body, self._parsed), + ) + + def __setstate__(self, state): + self._map = state[0] + self._body = state[1] + self._parsed = state[2] diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py index d889a924ae..46ee938b45 100644 --- a/pipenv/vendor/tomlkit/exceptions.py +++ b/pipenv/vendor/tomlkit/exceptions.py @@ -85,7 +85,7 @@ class InvalidCharInStringError(ParseError): """ def __init__(self, line, col, char): # type: (int, int, str) -> None - message = "Invalid character '{}' in string".format(char) + message = "Invalid character {} in string".format(repr(char)) super(InvalidCharInStringError, self).__init__(line, col, message=message) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 26f24701d0..abece0200c 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -17,6 +17,11 @@ from ._compat import unicode from ._utils import escape_string +if PY2: + from functools32 import lru_cache +else: + from functools import lru_cache + def item(value, _parent=None): from .container import Container @@ -75,18 +80,45 @@ def item(value, _parent=None): class StringType(Enum): - + # Single Line Basic SLB = '"' + # Multi Line Basic MLB = '"""' + # Single Line Literal SLL = "'" + # Multi Line Literal MLL = "'''" + @property + @lru_cache(maxsize=None) + def unit(self): # type: () -> str + return self.value[0] + + @lru_cache(maxsize=None) + def is_basic(self): # type: () -> bool + return self in {StringType.SLB, StringType.MLB} + + @lru_cache(maxsize=None) def is_literal(self): # type: () -> bool return self in {StringType.SLL, StringType.MLL} + @lru_cache(maxsize=None) + def is_singleline(self): # type: () -> bool + return self in {StringType.SLB, StringType.SLL} + + @lru_cache(maxsize=None) def is_multiline(self): # type: () -> bool return self in {StringType.MLB, StringType.MLL} + @lru_cache(maxsize=None) + def toggle(self): # type: () -> StringType + return { + StringType.SLB: StringType.MLB, + StringType.MLB: StringType.SLB, + StringType.SLL: StringType.MLL, + StringType.MLL: StringType.SLL, + }[self] + class Trivia: """ @@ -158,7 +190,10 @@ def __hash__(self): # type: () -> int return hash(self.key) def __eq__(self, other): # type: (Key) -> bool - return self.key == other.key + if isinstance(other, Key): + return self.key == other.key + + return self.key == other def __str__(self): # type: () -> str return self.as_string() @@ -205,6 +240,15 @@ def indent(self, indent): # type: (int) -> Item return self + def _getstate(self, protocol=3): + return (self._trivia,) + + def __reduce__(self): + return self.__reduce_ex__(2) + + def __reduce_ex__(self, protocol): + return self.__class__, self._getstate(protocol) + class Whitespace(Item): """ @@ -240,6 +284,9 @@ def as_string(self): # type: () -> str def __repr__(self): # type: () -> str return "<{} {}>".format(self.__class__.__name__, repr(self._s)) + def _getstate(self, protocol=3): + return self._s, self._fixed + class Comment(Item): """ @@ -273,7 +320,7 @@ def __init__(self, _, trivia, raw): # type: (int, Trivia, str) -> None self._raw = raw self._sign = False - if re.match("^[+\-]\d+$", raw): + if re.match(r"^[+\-]\d+$", raw): self._sign = True @property @@ -322,6 +369,9 @@ def _new(self, result): return Integer(result, self._trivia, raw) + def _getstate(self, protocol=3): + return int(self), self._trivia, self._raw + class Float(float, Item): """ @@ -337,7 +387,7 @@ def __init__(self, _, trivia, raw): # type: (float, Trivia, str) -> None self._raw = raw self._sign = False - if re.match("^[+\-].+$", raw): + if re.match(r"^[+\-].+$", raw): self._sign = True @property @@ -386,6 +436,9 @@ def _new(self, result): return Float(result, self._trivia, raw) + def _getstate(self, protocol=3): + return float(self), self._trivia, self._raw + class Bool(Item): """ @@ -408,13 +461,16 @@ def value(self): # type: () -> bool def as_string(self): # type: () -> str return str(self._value).lower() + def _getstate(self, protocol=3): + return self._value, self._trivia -class DateTime(datetime, Item): + +class DateTime(Item, datetime): """ A datetime literal. """ - def __new__(cls, value, *_): # type: (datetime, ...) -> datetime + def __new__(cls, value, *_): # type: (..., datetime, ...) -> datetime return datetime.__new__( cls, value.year, @@ -458,13 +514,29 @@ def _new(self, result): return DateTime(result, self._trivia, raw) + def _getstate(self, protocol=3): + return ( + datetime( + self.year, + self.month, + self.day, + self.hour, + self.minute, + self.second, + self.microsecond, + self.tzinfo, + ), + self._trivia, + self._raw, + ) + -class Date(date, Item): +class Date(Item, date): """ A date literal. """ - def __new__(cls, value, *_): # type: (date, ...) -> date + def __new__(cls, value, *_): # type: (..., date, ...) -> date return date.__new__(cls, value.year, value.month, value.day) def __init__(self, _, trivia, raw): # type: (date, Trivia, str) -> None @@ -498,8 +570,11 @@ def _new(self, result): return Date(result, self._trivia, raw) + def _getstate(self, protocol=3): + return (datetime(self.year, self.month, self.day), self._trivia, self._raw) + -class Time(time, Item): +class Time(Item, time): """ A time literal. """ @@ -525,6 +600,13 @@ def value(self): # type: () -> time def as_string(self): # type: () -> str return self._raw + def _getstate(self, protocol=3): + return ( + time(self.hour, self.minute, self.second, self.microsecond, self.tzinfo), + self._trivia, + self._raw, + ) + class Array(Item, list): """ @@ -623,6 +705,9 @@ def __str__(self): def __repr__(self): return str(self) + def _getstate(self, protocol=3): + return self._value, self._trivia + class Table(Item, dict): """ @@ -637,7 +722,7 @@ def __init__( is_super_table=False, name=None, display_name=None, - ): # type: (tomlkit.container.Container, Trivia, bool) -> None + ): # type: (tomlkit.container.Container, Trivia, bool, ...) -> None super(Table, self).__init__(trivia) self.name = name @@ -790,6 +875,16 @@ def __delitem__(self, key): # type: (Union[Key, str]) -> None def __repr__(self): return super(Table, self).__repr__() + def _getstate(self, protocol=3): + return ( + self._value, + self._trivia, + self._is_aot_element, + self._is_super_table, + self.name, + self.display_name, + ) + class InlineTable(Item, dict): """ @@ -924,6 +1019,9 @@ def __delitem__(self, key): # type: (Union[Key, str]) -> None def __repr__(self): return super(InlineTable, self).__repr__() + def _getstate(self, protocol=3): + return (self._value, self._trivia) + class String(unicode, Item): """ @@ -965,6 +1063,9 @@ def __sub__(self, other): def _new(self, result): return String(self._t, result, result, self._trivia) + def _getstate(self, protocol=3): + return self._t, unicode(self), self._original, self._trivia + class AoT(Item, list): """ @@ -974,7 +1075,7 @@ class AoT(Item, list): def __init__( self, body, name=None, parsed=False ): # type: (List[Table], Optional[str]) -> None - self.name = None + self.name = name self._body = [] self._parsed = parsed @@ -1025,6 +1126,9 @@ def as_string(self): # type: () -> str def __repr__(self): # type: () -> str return "".format(self.value) + def _getstate(self, protocol=3): + return self._body, self.name, self._parsed + class Null(Item): """ @@ -1044,3 +1148,6 @@ def value(self): # type: () -> None def as_string(self): # type: () -> str return "" + + def _getstate(self, protocol=3): + return tuple() diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 45c8ee8cbb..7971d9a27c 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py @@ -75,7 +75,7 @@ def extract(self): # type: () -> str else: return self._src[self._marker : self._idx] - def inc(self): # type: () -> bool + def inc(self, exception=None): # type: () -> bool """ Increments the parser if the end of the input has not been reached. Returns whether or not it was able to advance. @@ -88,15 +88,17 @@ def inc(self): # type: () -> bool self._idx = len(self._src) self._current = TOMLChar("\0") - return False + if not exception: + return False + raise exception - def inc_n(self, n): # type: (int) -> bool + def inc_n(self, n, exception=None): # type: (int) -> bool """ Increments the parser by n characters if the end of the input has not been reached. """ for _ in range(n): - if not self.inc(): + if not self.inc(exception=exception): return False return True @@ -336,7 +338,7 @@ def _parse_comment_trail(self): # type: () -> Tuple[str, str, str] self.mark() break - elif c in " \t\r,": + elif c in " \t\r": self.inc() else: raise self.parse_error(UnexpectedCharError, (c)) @@ -669,140 +671,143 @@ def _parse_number(self, raw, trivia): # type: (str, Trivia) -> Optional[Item] return def _parse_literal_string(self): # type: () -> Item - return self._parse_string("'") + return self._parse_string(StringType.SLL) def _parse_basic_string(self): # type: () -> Item - return self._parse_string('"') + return self._parse_string(StringType.SLB) + + def _parse_escaped_char(self, multiline): + if multiline and self._current.is_ws(): + # When the last non-whitespace character on a line is + # a \, it will be trimmed along with all whitespace + # (including newlines) up to the next non-whitespace + # character or closing delimiter. + # """\ + # hello \ + # world""" + tmp = "" + while self._current.is_ws(): + tmp += self._current + # consume the whitespace, EOF here is an issue + # (middle of string) + self.inc(exception=UnexpectedEofError) + continue - def _parse_string(self, delim): # type: (str) -> Item - multiline = False - value = "" + # the escape followed by whitespace must have a newline + # before any other chars + if "\n" not in tmp: + raise self.parse_error(InvalidCharInStringError, (self._current,)) - if delim == "'": - str_type = StringType.SLL - else: - str_type = StringType.SLB + return "" - # Skip opening delim - if not self.inc(): - return self.parse_error(UnexpectedEofError) + if self._current in _escaped: + c = _escaped[self._current] - if self._current == delim: - self.inc() + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) - if self._current == delim: - multiline = True - if delim == "'": - str_type = StringType.MLL - else: - str_type = StringType.MLB + return c - if not self.inc(): - return self.parse_error(UnexpectedEofError) - else: - # Empty string - return String(str_type, "", "", Trivia()) + if self._current in {"u", "U"}: + # this needs to be a unicode + u, ue = self._peek_unicode(self._current == "U") + if u is not None: + # consume the U char and the unicode value + self.inc_n(len(ue) + 1) - self.mark() - if self._current == "\n": - # The first new line should be discarded - self.inc() + return u - previous = None - escaped = False - while True: - if ( - previous != "\\" - or previous == "\\" - and (escaped or str_type.is_literal()) - ) and self._current == delim: - val = self.extract() - - if multiline: - stop = True - for _ in range(3): - if self._current != delim: - # Not a triple quote, leave in result as-is. - stop = False + raise self.parse_error(InvalidCharInStringError, (self._current,)) - # Adding back the quote character - value += delim - break + def _parse_string(self, delim): # type: (str) -> Item + delim = StringType(delim) + assert delim.is_singleline() - self.inc() # TODO: Handle EOF + # only keep parsing for string if the current character matches the delim + if self._current != delim.unit: + raise ValueError("Expecting a {!r} character".format(delim)) - if not stop: - continue - else: - self.inc() + # consume the opening/first delim, EOF here is an issue + # (middle of string or middle of delim) + self.inc(exception=UnexpectedEofError) - return String(str_type, value, val, Trivia()) - else: - if previous == "\\" and self._current.is_ws() and multiline: - while self._current.is_ws(): - previous = self._current + if self._current == delim.unit: + # consume the closing/second delim, we do not care if EOF occurs as + # that would simply imply an empty single line string + if not self.inc() or self._current != delim.unit: + # Empty string + return String(delim, "", "", Trivia()) - self.inc() - continue + # consume the third delim, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) - if self._current == delim: - continue + delim = delim.toggle() # convert delim to multi delim - if previous == "\\": - if self._current == "\\" and not escaped: - if not str_type.is_literal(): - escaped = True - else: - value += self._current + self.mark() # to extract the original string with whitespace and all + value = "" - previous = self._current + # A newline immediately following the opening delimiter will be trimmed. + if delim.is_multiline() and self._current == "\n": + # consume the newline, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) - if not self.inc(): - raise self.parse_error(UnexpectedEofError) + escaped = False # whether the previous key was ESCAPE + while True: + if delim.is_singleline() and self._current.is_nl(): + # single line cannot have actual newline characters + raise self.parse_error(InvalidCharInStringError, (self._current,)) + elif not escaped and self._current == delim.unit: + # try to process current as a closing delim + original = self.extract() + + close = "" + if delim.is_multiline(): + # try consuming three delims as this would mean the end of + # the string + for last in [False, False, True]: + if self._current != delim.unit: + # Not a triple quote, leave in result as-is. + # Adding back the characters we already consumed + value += close + close = "" # clear the close + break - continue - elif self._current in _escaped and not escaped: - if not str_type.is_literal(): - value = value[:-1] - value += _escaped[self._current] - else: - value += self._current - elif self._current in {"u", "U"} and not escaped: - # Maybe unicode - u, ue = self._peek_unicode(self._current == "U") - if u is not None: - value = value[:-1] - value += u - self.inc_n(len(ue)) - else: - if not escaped and not str_type.is_literal(): - raise self.parse_error( - InvalidCharInStringError, (self._current,) - ) - - value += self._current - else: - if not escaped and not str_type.is_literal(): - raise self.parse_error( - InvalidCharInStringError, (self._current,) - ) + close += delim.unit - value += self._current + # consume this delim, EOF here is only an issue if this + # is not the third (last) delim character + self.inc(exception=UnexpectedEofError if not last else None) - if self._current.is_ws() and multiline and not escaped: + if not close: # if there is no close characters, keep parsing continue else: - value += self._current + close = delim.unit - if escaped: - escaped = False + # consume the closing delim, we do not care if EOF occurs as + # that would simply imply the end of self._src + self.inc() - previous = self._current - if not self.inc(): - raise self.parse_error(UnexpectedEofError) + return String(delim, value, original, Trivia()) + elif delim.is_basic() and escaped: + # attempt to parse the current char as an escaped value, an exception + # is raised if this fails + value += self._parse_escaped_char(delim.is_multiline()) + + # no longer escaped + escaped = False + elif delim.is_basic() and self._current == "\\": + # the next char is being escaped + escaped = True + + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) + else: + # this is either a literal string where we keep everything as is, + # or this is not a special escaped char in a basic string + value += self._current - if previous == "\\" and self._current.is_ws() and multiline: - value = value[:-1] + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) def _parse_table( self, parent_name=None diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 8a3bf9e1ca..92f1a9c92c 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -1,7 +1,13 @@ import string +from ._compat import PY2 from ._compat import unicode +if PY2: + from functools32 import lru_cache +else: + from functools import lru_cache + class TOMLChar(unicode): def __init__(self, c): @@ -10,36 +16,42 @@ def __init__(self, c): if len(self) > 1: raise ValueError("A TOML character must be of length 1") + @lru_cache(maxsize=None) def is_bare_key_char(self): # type: () -> bool """ Whether the character is a valid bare key name or not. """ return self in string.ascii_letters + string.digits + "-" + "_" + @lru_cache(maxsize=None) def is_kv_sep(self): # type: () -> bool """ Whether the character is a valid key/value separator ot not. """ return self in "= \t" + @lru_cache(maxsize=None) def is_int_float_char(self): # type: () -> bool """ Whether the character if a valid integer or float value character or not. """ return self in string.digits + "+" + "-" + "_" + "." + "e" + @lru_cache(maxsize=None) def is_ws(self): # type: () -> bool """ Whether the character is a whitespace character or not. """ return self in " \t\r\n" + @lru_cache(maxsize=None) def is_nl(self): # type: () -> bool """ Whether the character is a new line character or not. """ return self in "\n\r" + @lru_cache(maxsize=None) def is_spaces(self): # type: () -> bool """ Whether the character is a space or not diff --git a/pipenv/vendor/urllib3/__init__.py b/pipenv/vendor/urllib3/__init__.py old mode 100755 new mode 100644 index 4bd533b5b4..75725167e0 --- a/pipenv/vendor/urllib3/__init__.py +++ b/pipenv/vendor/urllib3/__init__.py @@ -23,16 +23,11 @@ # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.23' +__version__ = '1.24' __all__ = ( 'HTTPConnectionPool', diff --git a/pipenv/vendor/urllib3/_collections.py b/pipenv/vendor/urllib3/_collections.py old mode 100755 new mode 100644 index 6e36b84e59..34f23811c6 --- a/pipenv/vendor/urllib3/_collections.py +++ b/pipenv/vendor/urllib3/_collections.py @@ -14,10 +14,7 @@ def __exit__(self, exc_type, exc_value, traceback): pass -try: # Python 2.7+ - from collections import OrderedDict -except ImportError: - from .packages.ordered_dict import OrderedDict +from collections import OrderedDict from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 diff --git a/pipenv/vendor/urllib3/connection.py b/pipenv/vendor/urllib3/connection.py old mode 100755 new mode 100644 index a03b573f01..02b36654bd --- a/pipenv/vendor/urllib3/connection.py +++ b/pipenv/vendor/urllib3/connection.py @@ -2,7 +2,6 @@ import datetime import logging import os -import sys import socket from socket import error as SocketError, timeout as SocketTimeout import warnings @@ -78,9 +77,6 @@ class HTTPConnection(_HTTPConnection, object): - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` - ``source_address``: Set the source address for the current connection. - - .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x - - ``socket_options``: Set specific options on the underlying socket. If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy. @@ -108,21 +104,13 @@ def __init__(self, *args, **kw): if six.PY3: # Python 3 kw.pop('strict', None) - # Pre-set source_address in case we have an older Python like 2.6. + # Pre-set source_address. self.source_address = kw.get('source_address') - if sys.version_info < (2, 7): # Python 2.6 - # _HTTPConnection on Python 2.6 will balk at this keyword arg, but - # not newer versions. We can still use it when creating a - # connection though, so we pop it *after* we have saved it as - # self.source_address. - kw.pop('source_address', None) - #: The socket options provided by the user. If no options are #: provided, we use the default options. self.socket_options = kw.pop('socket_options', self.default_socket_options) - # Superclass also sets self.source_address in Python 2.7+. _HTTPConnection.__init__(self, *args, **kw) @property @@ -183,10 +171,7 @@ def _new_conn(self): def _prepare_conn(self, conn): self.sock = conn - # the _tunnel_host attribute was added in python 2.6.3 (via - # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do - # not have them. - if getattr(self, '_tunnel_host', None): + if self._tunnel_host: # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -217,13 +202,13 @@ def request_chunked(self, method, url, body=None, headers=None): self.endheaders() if body is not None: - stringish_types = six.string_types + (six.binary_type,) + stringish_types = six.string_types + (bytes,) if isinstance(body, stringish_types): body = (body,) for chunk in body: if not chunk: continue - if not isinstance(chunk, six.binary_type): + if not isinstance(chunk, bytes): chunk = chunk.encode('utf8') len_str = hex(len(chunk))[2:] self.send(len_str.encode('utf-8')) @@ -242,7 +227,7 @@ class HTTPSConnection(HTTPConnection): def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, **kw): + ssl_context=None, server_hostname=None, **kw): HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) @@ -250,6 +235,7 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, self.key_file = key_file self.cert_file = cert_file self.ssl_context = ssl_context + self.server_hostname = server_hostname # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) @@ -270,6 +256,7 @@ def connect(self): keyfile=self.key_file, certfile=self.cert_file, ssl_context=self.ssl_context, + server_hostname=self.server_hostname ) @@ -312,12 +299,9 @@ def set_cert(self, key_file=None, cert_file=None, def connect(self): # Add certificate verification conn = self._new_conn() - hostname = self.host - if getattr(self, '_tunnel_host', None): - # _tunnel_host was added in Python 2.6.3 - # (See: http://hg.python.org/cpython/rev/0f57b30a152f) + if self._tunnel_host: self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -328,6 +312,10 @@ def connect(self): # Override the host with the one we're requesting data from. hostname = self._tunnel_host + server_hostname = hostname + if self.server_hostname is not None: + server_hostname = self.server_hostname + is_time_off = datetime.date.today() < RECENT_DATE if is_time_off: warnings.warn(( @@ -352,7 +340,7 @@ def connect(self): certfile=self.cert_file, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, - server_hostname=hostname, + server_hostname=server_hostname, ssl_context=context) if self.assert_fingerprint: @@ -373,7 +361,7 @@ def connect(self): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - _match_hostname(cert, self.assert_hostname or hostname) + _match_hostname(cert, self.assert_hostname or server_hostname) self.is_verified = ( context.verify_mode == ssl.CERT_REQUIRED or diff --git a/pipenv/vendor/urllib3/connectionpool.py b/pipenv/vendor/urllib3/connectionpool.py old mode 100755 new mode 100644 index 8fcb0bce79..f7a8f193d1 --- a/pipenv/vendor/urllib3/connectionpool.py +++ b/pipenv/vendor/urllib3/connectionpool.py @@ -89,7 +89,7 @@ def close(self): # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 -_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) +_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} class HTTPConnectionPool(ConnectionPool, RequestMethods): @@ -313,7 +313,7 @@ def _raise_timeout(self, err, url, timeout_value): # Catch possible read timeouts thrown as SSL errors. If not the # case, rethrow the original. We need to do this because of: # http://bugs.python.org/issue10272 - if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 + if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4 raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) def _make_request(self, conn, method, url, timeout=_Default, chunked=False, @@ -375,7 +375,7 @@ def _make_request(self, conn, method, url, timeout=_Default, chunked=False, try: try: # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older, Python 3 + except TypeError: # Python 3 try: httplib_response = conn.getresponse() except Exception as e: @@ -801,17 +801,7 @@ def _prepare_proxy(self, conn): Establish tunnel connection early, because otherwise httplib would improperly set Host: header to proxy's IP:port. """ - # Python 2.7+ - try: - set_tunnel = conn.set_tunnel - except AttributeError: # Platform-specific: Python 2.6 - set_tunnel = conn._set_tunnel - - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older - set_tunnel(self._proxy_host, self.port) - else: - set_tunnel(self._proxy_host, self.port, self.proxy_headers) - + conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) conn.connect() def _new_conn(self): diff --git a/pipenv/vendor/urllib3/contrib/__init__.py b/pipenv/vendor/urllib3/contrib/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_appengine_environ.py b/pipenv/vendor/urllib3/contrib/_appengine_environ.py new file mode 100644 index 0000000000..f3e00942cb --- /dev/null +++ b/pipenv/vendor/urllib3/contrib/_appengine_environ.py @@ -0,0 +1,30 @@ +""" +This module provides means to detect the App Engine environment. +""" + +import os + + +def is_appengine(): + return (is_local_appengine() or + is_prod_appengine() or + is_prod_appengine_mvms()) + + +def is_appengine_sandbox(): + return is_appengine() and not is_prod_appengine_mvms() + + +def is_local_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Development/' in os.environ['SERVER_SOFTWARE']) + + +def is_prod_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and + not is_prod_appengine_mvms()) + + +def is_prod_appengine_mvms(): + return os.environ.get('GAE_VM', False) == 'true' diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/__init__.py b/pipenv/vendor/urllib3/contrib/_securetransport/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py b/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py b/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/appengine.py b/pipenv/vendor/urllib3/contrib/appengine.py old mode 100755 new mode 100644 index 66922e06aa..2952f114df --- a/pipenv/vendor/urllib3/contrib/appengine.py +++ b/pipenv/vendor/urllib3/contrib/appengine.py @@ -39,8 +39,8 @@ """ from __future__ import absolute_import +import io import logging -import os import warnings from ..packages.six.moves.urllib.parse import urljoin @@ -53,11 +53,11 @@ SSLError ) -from ..packages.six import BytesIO from ..request import RequestMethods from ..response import HTTPResponse from ..util.timeout import Timeout from ..util.retry import Retry +from . import _appengine_environ try: from google.appengine.api import urlfetch @@ -239,7 +239,7 @@ def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): original_response = HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), msg=urlfetch_resp.header_msg, headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, @@ -247,7 +247,7 @@ def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): ) return HTTPResponse( - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, original_response=original_response, @@ -280,26 +280,10 @@ def _get_retries(self, retries, redirect): return retries -def is_appengine(): - return (is_local_appengine() or - is_prod_appengine() or - is_prod_appengine_mvms()) +# Alias methods from _appengine_environ to maintain public API interface. - -def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_mvms() - - -def is_local_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) - - -def is_prod_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_mvms()) - - -def is_prod_appengine_mvms(): - return os.environ.get('GAE_VM', False) == 'true' +is_appengine = _appengine_environ.is_appengine +is_appengine_sandbox = _appengine_environ.is_appengine_sandbox +is_local_appengine = _appengine_environ.is_local_appengine +is_prod_appengine = _appengine_environ.is_prod_appengine +is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms diff --git a/pipenv/vendor/urllib3/contrib/ntlmpool.py b/pipenv/vendor/urllib3/contrib/ntlmpool.py old mode 100755 new mode 100644 index 642e99ed2d..8ea127c583 --- a/pipenv/vendor/urllib3/contrib/ntlmpool.py +++ b/pipenv/vendor/urllib3/contrib/ntlmpool.py @@ -43,8 +43,7 @@ def _new_conn(self): log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', self.num_connections, self.host, self.authurl) - headers = {} - headers['Connection'] = 'Keep-Alive' + headers = {'Connection': 'Keep-Alive'} req_header = 'Authorization' resp_header = 'www-authenticate' diff --git a/pipenv/vendor/urllib3/contrib/pyopenssl.py b/pipenv/vendor/urllib3/contrib/pyopenssl.py old mode 100755 new mode 100644 index 4d4b1aff97..7c0e9465d9 --- a/pipenv/vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/vendor/urllib3/contrib/pyopenssl.py @@ -163,6 +163,9 @@ def _dnsname_to_stdlib(name): from ASCII bytes. We need to idna-encode that string to get it back, and then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + + If the name cannot be idna-encoded then we return None signalling that + the name given should be skipped. """ def idna_encode(name): """ @@ -172,14 +175,19 @@ def idna_encode(name): """ import idna - for prefix in [u'*.', u'.']: - if name.startswith(prefix): - name = name[len(prefix):] - return prefix.encode('ascii') + idna.encode(name) - return idna.encode(name) + try: + for prefix in [u'*.', u'.']: + if name.startswith(prefix): + name = name[len(prefix):] + return prefix.encode('ascii') + idna.encode(name) + return idna.encode(name) + except idna.core.IDNAError: + return None name = idna_encode(name) - if sys.version_info >= (3, 0): + if name is None: + return None + elif sys.version_info >= (3, 0): name = name.decode('utf-8') return name @@ -223,9 +231,10 @@ def get_subj_alt_name(peer_cert): # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 # decoded. This is pretty frustrating, but that's what the standard library # does with certificates, and so we need to attempt to do the same. + # We also want to skip over names which cannot be idna encoded. names = [ - ('DNS', _dnsname_to_stdlib(name)) - for name in ext.get_values_for_type(x509.DNSName) + ('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) + if name is not None ] names.extend( ('IP Address', str(name)) diff --git a/pipenv/vendor/urllib3/contrib/securetransport.py b/pipenv/vendor/urllib3/contrib/securetransport.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/socks.py b/pipenv/vendor/urllib3/contrib/socks.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/exceptions.py b/pipenv/vendor/urllib3/exceptions.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/fields.py b/pipenv/vendor/urllib3/fields.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/filepost.py b/pipenv/vendor/urllib3/filepost.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/__init__.py b/pipenv/vendor/urllib3/packages/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/backports/__init__.py b/pipenv/vendor/urllib3/packages/backports/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/backports/makefile.py b/pipenv/vendor/urllib3/packages/backports/makefile.py old mode 100755 new mode 100644 index 75b80dcf84..740db377d9 --- a/pipenv/vendor/urllib3/packages/backports/makefile.py +++ b/pipenv/vendor/urllib3/packages/backports/makefile.py @@ -16,7 +16,7 @@ def backport_makefile(self, mode="r", buffering=None, encoding=None, """ Backport of ``socket.makefile`` from Python 3.5. """ - if not set(mode) <= set(["r", "w", "b"]): + if not set(mode) <= {"r", "w", "b"}: raise ValueError( "invalid mode %r (only r, w, b allowed)" % (mode,) ) diff --git a/pipenv/vendor/urllib3/packages/ordered_dict.py b/pipenv/vendor/urllib3/packages/ordered_dict.py deleted file mode 100755 index 4479363cc4..0000000000 --- a/pipenv/vendor/urllib3/packages/ordered_dict.py +++ /dev/null @@ -1,259 +0,0 @@ -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. -# Copyright 2009 Raymond Hettinger, released under the MIT License. -# http://code.activestate.com/recipes/576693/ -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/pipenv/vendor/urllib3/packages/six.py b/pipenv/vendor/urllib3/packages/six.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py old mode 100755 new mode 100644 index 1fd42f38ae..d6e66c0196 --- a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py @@ -9,8 +9,7 @@ # ipaddress has been backported to 2.6+ in pypi. If it is installed on the # system, use it to handle IPAddress ServerAltnames (this was added in # python-3.5) otherwise only do DNS matching. This allows -# backports.ssl_match_hostname to continue to be used all the way back to -# python-2.4. +# backports.ssl_match_hostname to continue to be used in Python 2.7. try: import ipaddress except ImportError: diff --git a/pipenv/vendor/urllib3/poolmanager.py b/pipenv/vendor/urllib3/poolmanager.py old mode 100755 new mode 100644 index 506a3c9b87..fe5491cfda --- a/pipenv/vendor/urllib3/poolmanager.py +++ b/pipenv/vendor/urllib3/poolmanager.py @@ -47,6 +47,7 @@ 'key__socks_options', # dict 'key_assert_hostname', # bool or string 'key_assert_fingerprint', # str + 'key_server_hostname', #str ) #: The namedtuple class used to construct keys for the connection pool. diff --git a/pipenv/vendor/urllib3/request.py b/pipenv/vendor/urllib3/request.py old mode 100755 new mode 100644 index 1be3334113..8f2f44bb21 --- a/pipenv/vendor/urllib3/request.py +++ b/pipenv/vendor/urllib3/request.py @@ -36,7 +36,7 @@ class RequestMethods(object): explicitly. """ - _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + _encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'} def __init__(self, headers=None): self.headers = headers or {} diff --git a/pipenv/vendor/urllib3/response.py b/pipenv/vendor/urllib3/response.py old mode 100755 new mode 100644 index 9873cb9423..f0cfbb5499 --- a/pipenv/vendor/urllib3/response.py +++ b/pipenv/vendor/urllib3/response.py @@ -11,7 +11,7 @@ BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked, IncompleteRead, InvalidHeader ) -from .packages.six import string_types as basestring, binary_type, PY3 +from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head @@ -23,7 +23,7 @@ class DeflateDecoder(object): def __init__(self): self._first_try = True - self._data = binary_type() + self._data = b'' self._obj = zlib.decompressobj() def __getattr__(self, name): @@ -69,7 +69,7 @@ def __getattr__(self, name): return getattr(self._obj, name) def decompress(self, data): - ret = binary_type() + ret = b'' if self._state == GzipDecoderState.SWALLOW_DATA or not data: return ret while True: @@ -90,7 +90,31 @@ def decompress(self, data): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) +class MultiDecoder(object): + """ + From RFC7231: + If one or more encodings have been applied to a representation, the + sender that applied the encodings MUST generate a Content-Encoding + header field that lists the content codings in the order in which + they were applied. + """ + + def __init__(self, modes): + self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')] + + def flush(self): + return self._decoders[0].flush() + + def decompress(self, data): + for d in reversed(self._decoders): + data = d.decompress(data) + return data + + def _get_decoder(mode): + if ',' in mode: + return MultiDecoder(mode) + if mode == 'gzip': return GzipDecoder() @@ -159,7 +183,7 @@ def __init__(self, body='', headers=None, status=0, version=0, reason=None, self.msg = msg self._request_url = request_url - if body and isinstance(body, (basestring, binary_type)): + if body and isinstance(body, (basestring, bytes)): self._body = body self._pool = pool @@ -283,8 +307,13 @@ def _init_decoder(self): # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None and content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + elif ',' in content_encoding: + encodings = [e.strip() for e in content_encoding.split(',') if e.strip() in self.CONTENT_DECODERS] + if len(encodings): + self._decoder = _get_decoder(content_encoding) def _decode(self, data, decode_content, flush_decoder): """ diff --git a/pipenv/vendor/urllib3/util/__init__.py b/pipenv/vendor/urllib3/util/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/connection.py b/pipenv/vendor/urllib3/util/connection.py old mode 100755 new mode 100644 index 5cf488f4b5..5ad70b2f1c --- a/pipenv/vendor/urllib3/util/connection.py +++ b/pipenv/vendor/urllib3/util/connection.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import socket from .wait import NoWayToWaitForSocketError, wait_for_read +from ..contrib import _appengine_environ def is_connection_dropped(conn): # Platform-specific @@ -105,6 +106,13 @@ def _has_ipv6(host): sock = None has_ipv6 = False + # App Engine doesn't support IPV6 sockets and actually has a quota on the + # number of sockets that can be used, so just early out here instead of + # creating a socket needlessly. + # See https://github.com/urllib3/urllib3/issues/1446 + if _appengine_environ.is_appengine_sandbox(): + return False + if socket.has_ipv6: # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To diff --git a/pipenv/vendor/urllib3/util/queue.py b/pipenv/vendor/urllib3/util/queue.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/request.py b/pipenv/vendor/urllib3/util/request.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/response.py b/pipenv/vendor/urllib3/util/response.py old mode 100755 new mode 100644 index 67cf730ab0..3d5486485a --- a/pipenv/vendor/urllib3/util/response.py +++ b/pipenv/vendor/urllib3/util/response.py @@ -59,8 +59,14 @@ def assert_header_parsing(headers): get_payload = getattr(headers, 'get_payload', None) unparsed_data = None - if get_payload: # Platform-specific: Python 3. - unparsed_data = get_payload() + if get_payload: + # get_payload is actually email.message.Message.get_payload; + # we're only interested in the result if it's not a multipart message + if not headers.is_multipart(): + payload = get_payload() + + if isinstance(payload, (bytes, str)): + unparsed_data = payload if defects or unparsed_data: raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) diff --git a/pipenv/vendor/urllib3/util/retry.py b/pipenv/vendor/urllib3/util/retry.py old mode 100755 new mode 100644 index 7ad3dc6608..e7d0abd610 --- a/pipenv/vendor/urllib3/util/retry.py +++ b/pipenv/vendor/urllib3/util/retry.py @@ -115,7 +115,7 @@ class Retry(object): (most errors are resolved immediately by a second try without a delay). urllib3 will sleep for:: - {backoff factor} * (2 ^ ({number of total retries} - 1)) + {backoff factor} * (2 ** ({number of total retries} - 1)) seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer diff --git a/pipenv/vendor/urllib3/util/ssl_.py b/pipenv/vendor/urllib3/util/ssl_.py old mode 100755 new mode 100644 index 2893752a3d..24ee26d632 --- a/pipenv/vendor/urllib3/util/ssl_.py +++ b/pipenv/vendor/urllib3/util/ssl_.py @@ -56,9 +56,8 @@ def _const_compare_digest_backport(a, b): OP_NO_COMPRESSION = 0x20000 -# Python 2.7 and earlier didn't have inet_pton on non-Linux -# so we fallback on inet_aton in those cases. This means that -# we can only detect IPv4 addresses in this case. +# Python 2.7 doesn't have inet_pton on non-Linux so we fallback on inet_aton in +# those cases. This means that we can only detect IPv4 addresses in this case. if hasattr(socket, 'inet_pton'): inet_pton = socket.inet_pton else: @@ -67,7 +66,7 @@ def _const_compare_digest_backport(a, b): import ipaddress def inet_pton(_, host): - if isinstance(host, six.binary_type): + if isinstance(host, bytes): host = host.decode('ascii') return ipaddress.ip_address(host) @@ -115,10 +114,7 @@ def inet_pton(_, host): except ImportError: import sys - class SSLContext(object): # Platform-specific: Python 2 & 3.1 - supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or - (3, 2) <= sys.version_info) - + class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): self.protocol = protocol_version # Use default values from a real SSLContext @@ -141,12 +137,6 @@ def load_verify_locations(self, cafile=None, capath=None): raise SSLError("CA directories not supported in older Pythons") def set_ciphers(self, cipher_suite): - if not self.supports_set_ciphers: - raise TypeError( - 'Your version of Python does not support setting ' - 'a custom cipher suite. Please upgrade to Python ' - '2.7, 3.2, or later if you need this functionality.' - ) self.ciphers = cipher_suite def wrap_socket(self, socket, server_hostname=None, server_side=False): @@ -167,10 +157,7 @@ def wrap_socket(self, socket, server_hostname=None, server_side=False): 'ssl_version': self.protocol, 'server_side': server_side, } - if self.supports_set_ciphers: # Platform-specific: Python 2.7+ - return wrap_socket(socket, ciphers=self.ciphers, **kwargs) - else: # Platform-specific: Python 2.6 - return wrap_socket(socket, **kwargs) + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) def assert_fingerprint(cert, fingerprint): @@ -291,9 +278,6 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, context.options |= options - if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 - context.set_ciphers(ciphers or DEFAULT_CIPHERS) - context.verify_mode = cert_reqs if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 # We do our own verification, including fingerprints and alternative @@ -316,8 +300,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, A pre-made :class:`SSLContext` object. If none is provided, one will be created using :func:`create_urllib3_context`. :param ciphers: - A string of ciphers we wish the client to support. This is not - supported on Python 2.6 as the ssl module does not support it. + A string of ciphers we wish the client to support. :param ca_cert_dir: A directory containing CA certificates in multiple separate files, as supported by OpenSSL's -CApath flag or the capath argument to @@ -334,7 +317,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if ca_certs or ca_cert_dir: try: context.load_verify_locations(ca_certs, ca_cert_dir) - except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + except IOError as e: # Platform-specific: Python 2.7 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError # These are not equivalent unless we check the errno attribute @@ -378,7 +361,7 @@ def is_ipaddress(hostname): :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if six.PY3 and isinstance(hostname, six.binary_type): + if six.PY3 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. hostname = hostname.decode('ascii') diff --git a/pipenv/vendor/urllib3/util/timeout.py b/pipenv/vendor/urllib3/util/timeout.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/url.py b/pipenv/vendor/urllib3/util/url.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/wait.py b/pipenv/vendor/urllib3/util/wait.py old mode 100755 new mode 100644 index fa686eff48..4db71bafd8 --- a/pipenv/vendor/urllib3/util/wait.py +++ b/pipenv/vendor/urllib3/util/wait.py @@ -43,9 +43,6 @@ def _retry_on_intr(fn, timeout): else: # Old and broken Pythons. def _retry_on_intr(fn, timeout): - if timeout is not None and timeout <= 0: - return fn(timeout) - if timeout is None: deadline = float("inf") else: @@ -117,7 +114,7 @@ def _have_working_poll(): # from libraries like eventlet/greenlet. try: poll_obj = select.poll() - poll_obj.poll(0) + _retry_on_intr(poll_obj.poll, 0) except (AttributeError, OSError): return False else: diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 7a574b9f12..8812d2ae79 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -14,28 +14,28 @@ first==2.0.1 iso8601==0.1.12 jinja2==2.10 markupsafe==1.0 -parse==1.8.4 +parse==1.9.0 pathlib2==2.3.2 scandir==1.9 pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.2 -requests==2.19.1 +pythonfinder==1.1.3 +requests==2.20.0 chardet==3.0.4 idna==2.7 - urllib3==1.23 - certifi==2018.8.24 -requirementslib==1.1.9 + urllib3==1.24 + certifi==2018.10.15 +requirementslib==1.1.10 attrs==18.2.0 distlib==0.2.8 packaging==18.0 pyparsing==2.2.2 pytoml==0.1.19 plette==0.2.2 - tomlkit==0.4.4 -shellingham==1.2.6 + tomlkit==0.4.6 +shellingham==1.2.7 six==1.11.0 semver==2.8.1 shutilwhich==1.1.0 @@ -48,3 +48,5 @@ enum34==1.1.6 yaspin==0.14.0 cerberus==1.2 git+https://github.com/sarugaku/passa.git@master#egg=passa +cursor==1.2.0 +backports.functools_lru_cache==1.5 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index fe6f884c74..881985d2a0 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -8,12 +8,14 @@ open_file, temp_environ, temp_path, + spinner, ) from .misc import load_path, partialclass, run, shell_escape -from .path import mkdir_p, rmtree +from .path import mkdir_p, rmtree, create_tracked_tempdir +from .spin import VistirSpinner, create_spinner -__version__ = '0.1.7' +__version__ = '0.1.8' __all__ = [ @@ -31,4 +33,8 @@ "TemporaryDirectory", "NamedTemporaryFile", "partialmethod", + "spinner", + "VistirSpinner", + "create_spinner", + "create_tracked_tempdir" ] diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 0c865fe668..eab879088d 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import errno import os import sys import warnings @@ -16,18 +17,21 @@ "finalize", "partialmethod", "JSONDecodeError", + "FileNotFoundError", "ResourceWarning", "FileNotFoundError", "fs_str", + "lru_cache", "TemporaryDirectory", "NamedTemporaryFile", ] if sys.version_info >= (3, 5): from pathlib import Path - + from functools import lru_cache else: from pathlib2 import Path + from pipenv.vendor.backports.functools_lru_cache import lru_cache if sys.version_info < (3, 3): from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size @@ -57,16 +61,18 @@ class ResourceWarning(Warning): pass class FileNotFoundError(IOError): - pass + """No such file or directory""" + + def __init__(self, *args, **kwargs): + self.errno = errno.ENOENT + super(FileNotFoundError, self).__init__(*args, **kwargs) else: from builtins import ResourceWarning, FileNotFoundError - class ResourceWarning(ResourceWarning): - pass - class FileNotFoundError(FileNotFoundError): - pass +if not sys.warnoptions: + warnings.simplefilter("default", ResourceWarning) class TemporaryDirectory(object): diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 80f1f897d3..70f95c59b5 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import io import os import stat import sys @@ -13,7 +14,9 @@ from .path import is_file_url, is_valid_url, path_to_url, url_to_path -__all__ = ["temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file"] +__all__ = [ + "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner" +] # Borrowed from Pew. @@ -77,6 +80,66 @@ def cd(path): os.chdir(prev_cwd) +@contextmanager +def dummy_spinner(spin_type, text, **kwargs): + class FakeClass(object): + def __init__(self, text=""): + self.text = text + + def fail(self, exitcode=1, text=None): + if text: + print(text) + raise SystemExit(exitcode, text) + + def ok(self, text): + print(text) + return 0 + + def write(self, text): + print(text) + + myobj = FakeClass(text) + yield myobj + + +@contextmanager +def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): + """Get a spinner object or a dummy spinner to wrap a context. + + :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) + :param str start_text: Text to start off the spinner with (default: {None}) + :param dict handler_map: Handler map for signals to be handled gracefully (default: {None}) + :param bool nospin: If true, use the dummy spinner (default: {False}) + :return: A spinner object which can be manipulated while alive + :rtype: :class:`~vistir.spin.VistirSpinner` + + Raises: + RuntimeError -- Raised if the spinner extra is not installed + """ + + from .spin import create_spinner + spinner_func = create_spinner + if nospin is False: + try: + import yaspin + except ImportError: + raise RuntimeError( + "Failed to import spinner! Reinstall vistir with command:" + " pip install --upgrade vistir[spinner]" + ) + else: + spinner_name = None + if not start_text: + start_text = "Running..." + with spinner_func( + spinner_name=spinner_name, + text=start_text, + handler_map=handler_map, + nospin=nospin, + ) as _spinner: + yield _spinner + + @contextmanager def atomic_open_for_write(target, binary=False, newline=None, encoding=None): """Atomically open `target` for writing. @@ -192,8 +255,11 @@ def open_file(link, session=None): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - with open(local_path, "rb") as local_file: + try: + local_file = io.open(local_path, "rb") yield local_file + finally: + local_file.close() else: # Remote URL headers = {"Accept-Encoding": "identity"} diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 44607a9825..f42b4ad165 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -2,19 +2,24 @@ from __future__ import absolute_import, unicode_literals import json +import logging import locale import os import subprocess import sys from collections import OrderedDict -from contextlib import contextmanager from functools import partial import six from .cmdparse import Script from .compat import Path, fs_str, partialmethod +from .contextmanagers import spinner as spinner + +if os.name != "nt": + class WindowsError(OSError): + pass __all__ = [ @@ -30,6 +35,22 @@ ] +def _get_logger(name=None, level="ERROR"): + if not name: + name = __name__ + if isinstance(level, six.string_types): + level = getattr(logging, level.upper()) + logger = logging.getLogger(name) + logger.setLevel(level) + formatter = logging.Formatter( + "%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s" + ) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + def shell_escape(cmd): """Escape strings for use in :func:`~subprocess.Popen` and :func:`run`. @@ -75,9 +96,11 @@ def dedup(iterable): return iter(OrderedDict.fromkeys(iterable)) -def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True): +def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=True): from distutils.spawn import find_executable + if not env: + env = {} command = find_executable(script.command) options = { "env": env, @@ -102,7 +125,7 @@ def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True) try: return subprocess.Popen(cmd, **options) except WindowsError as e: - if e.winerror != 193: + if getattr(e, "winerror", 9999) != 193: raise options["shell"] = True # Try shell mode to use Windows's file association for file launch. @@ -111,7 +134,7 @@ def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True) def _create_subprocess( cmd, - env={}, + env=None, block=True, return_object=False, cwd=os.curdir, @@ -120,6 +143,8 @@ def _create_subprocess( combine_stderr=False, display_limit=200 ): + if not env: + env = {} try: c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr) @@ -128,11 +153,13 @@ def _create_subprocess( raise if not block: c.stdin.close() + log_level = "DEBUG" if verbose else "WARN" + logger = _get_logger(cmd._parts[0], level=log_level) output = [] err = [] spinner_orig_text = "" if spinner: - spinner_orig_text = spinner.text + spinner_orig_text = getattr(spinner, "text", "") streams = { "stdout": c.stdout, "stderr": c.stderr @@ -147,7 +174,7 @@ def _create_subprocess( line = to_text(stream.readline()) if not line: continue - line = line.rstrip() + line = to_text("{0}".format(line.rstrip())) if outstream == "stderr": stderr_line = line else: @@ -155,15 +182,24 @@ def _create_subprocess( if not (stdout_line or stderr_line): break if stderr_line: - err.append(line) + err.append(stderr_line) + if verbose: + if spinner: + spinner.write_err(fs_str(stderr_line)) + else: + logger.error(stderr_line) if stdout_line: output.append(stdout_line) display_line = stdout_line if len(stdout_line) > display_limit: display_line = "{0}...".format(stdout_line[:display_limit]) if verbose: - spinner.write(display_line) - spinner.text = "{0} {1}".format(spinner_orig_text, display_line) + if spinner: + spinner.write(fs_str(display_line)) + else: + logger.debug(display_line) + if spinner: + spinner.text = fs_str("{0} {1}".format(spinner_orig_text, display_line)) continue try: c.wait() @@ -175,18 +211,22 @@ def _create_subprocess( if spinner: if c.returncode > 0: spinner.fail("Failed...cleaning up...") - spinner.text = "Complete!" - spinner.ok("✔") + else: + spinner.text = "Complete!" + if not os.name == "nt": + spinner.ok("✔") + else: + spinner.ok() c.out = "\n".join(output) c.err = "\n".join(err) if err else "" else: c.out, c.err = c.communicate() + if not block: + c.wait() + c.out = fs_str("{0}".format(c.out)) if c.out else fs_str("") + c.err = fs_str("{0}".format(c.err)) if c.err else fs_str("") if not return_object: - if not block: - c.wait() - out = c.out if c.out else "" - err = c.err if c.err else "" - return out.strip(), err.strip() + return c.out.strip(), c.err.strip() return c @@ -198,7 +238,7 @@ def run( cwd=None, verbose=False, nospin=False, - spinner=None, + spinner_name=None, combine_stderr=True, display_limit=200 ): @@ -211,7 +251,7 @@ def run( :param str cwd: Current working directory contect to use for spawning the subprocess. :param bool verbose: Whether to print stdout in real time when non-blocking. :param bool nospin: Whether to disable the cli spinner. - :param str spinner: The name of the spinner to use if enabled, defaults to bouncingBar + :param str spinner_name: The name of the spinner to use if enabled, defaults to bouncingBar :param bool combine_stderr: Optionally merge stdout and stderr in the subprocess, false if nonblocking. :param int dispay_limit: The max width of output lines to display when using a spinner. :returns: A 2-tuple of (output, error) or a :class:`subprocess.Popen` object. @@ -221,8 +261,10 @@ def run( this functionality. """ - if not env: - env = os.environ.copy() + _env = os.environ.copy() + if env: + _env.update(env) + env = _env if six.PY2: fs_encode = partial(to_bytes, encoding=locale_encoding) _env = {fs_encode(k): fs_encode(v) for k, v in os.environ.items()} @@ -230,8 +272,8 @@ def run( _env[fs_encode(key)] = fs_encode(val) else: _env = {k: fs_str(v) for k, v in os.environ.items()} - if not spinner: - spinner = "bouncingBar" + if not spinner_name: + spinner_name = "bouncingBar" if six.PY2: if isinstance(cmd, six.string_types): cmd = cmd.encode("utf-8") @@ -241,48 +283,7 @@ def run( cmd = Script.parse(cmd) if block or not return_object: combine_stderr = False - sigmap = {} - if nospin is False: - try: - import signal - from yaspin import yaspin - from yaspin import spinners - from yaspin.signal_handlers import fancy_handler - except ImportError: - raise RuntimeError( - "Failed to import spinner! Reinstall vistir with command:" - " pip install --upgrade vistir[spinner]" - ) - else: - animation = getattr(spinners.Spinners, spinner) - sigmap = { - signal.SIGINT: fancy_handler - } - if os.name == "nt": - sigmap.update({ - signal.CTRL_C_EVENT: fancy_handler, - signal.CTRL_BREAK_EVENT: fancy_handler - }) - spinner_func = yaspin - else: - - @contextmanager - def spinner_func(spin_type, text, **kwargs): - class FakeClass(object): - def __init__(self, text=""): - self.text = text - - def ok(self, text): - return - - def write(self, text): - print(text) - - myobj = FakeClass(text) - yield myobj - - animation = None - with spinner_func(animation, sigmap=sigmap, text="Running...") as sp: + with spinner(spinner_name=spinner_name, start_text="Running...", nospin=nospin) as sp: return _create_subprocess( cmd, env=_env, diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 166282e864..ce7ecee055 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -15,8 +15,7 @@ from six.moves import urllib_parse from six.moves.urllib import request as urllib_request -from .compat import Path, _fs_encoding, TemporaryDirectory -from .misc import locale_encoding, to_bytes, to_text +from .compat import Path, _fs_encoding, TemporaryDirectory, ResourceWarning __all__ = [ @@ -74,6 +73,7 @@ def normalize_drive(path): identified with either upper or lower cased drive names. The case is always converted to uppercase because it seems to be preferred. """ + from .misc import to_text if os.name != "nt" or not isinstance(path, six.string_types): return path @@ -95,6 +95,7 @@ def path_to_url(path): >>> path_to_url("/home/user/code/myrepo/myfile.zip") 'file:///home/user/code/myrepo/myfile.zip' """ + from .misc import to_text, to_bytes if not path: return path @@ -108,6 +109,7 @@ def url_to_path(url): Follows logic taken from pip's equivalent function """ + from .misc import to_bytes assert is_file_url(url), "Only file: urls can be converted to local paths" _, netloc, path, _, _ = urllib_parse.urlsplit(url) # Netlocs are UNC paths @@ -120,14 +122,16 @@ def url_to_path(url): def is_valid_url(url): """Checks if a given string is an url""" + from .misc import to_text if not url: return url - pieces = urllib_parse.urlparse(url) + pieces = urllib_parse.urlparse(to_text(url)) return all([pieces.scheme, pieces.netloc]) def is_file_url(url): """Returns true if the given url is a file url""" + from .misc import to_text if not url: return False if not isinstance(url, six.string_types): @@ -144,6 +148,7 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ + from .misc import to_bytes fn = to_bytes(fn, encoding="utf-8") if os.path.exists(fn): return bool(os.stat(fn).st_mode & stat.S_IREAD) and not os.access(fn, os.W_OK) @@ -158,7 +163,8 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - newdir = abspathu(to_bytes(newdir, "utf-8")) + from .misc import to_bytes, to_text + newdir = to_bytes(newdir, "utf-8") if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( @@ -166,17 +172,17 @@ def mkdir_p(newdir, mode=0o777): newdir ) ) - pass else: - head, tail = os.path.split(newdir) + head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) # Make sure the tail doesn't point to the asame place as the head - tail_and_head_match = os.path.relpath(tail, start=os.path.basename(head)) == "." + curdir = to_bytes(".", encoding="utf-8") + tail_and_head_match = os.path.relpath(tail, start=os.path.basename(head)) == curdir if tail and not tail_and_head_match and not os.path.isdir(newdir): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( "A file with the same name as the desired dir, '{0}', already exists.".format( - newdir + to_text(newdir, encoding="utf-8") ) ) os.makedirs(os.path.join(head, tail), mode) @@ -210,9 +216,11 @@ def create_tracked_tempdir(*args, **kwargs): The return value is the path to the created directory. """ + tempdir = TemporaryDirectory(*args, **kwargs) TRACKED_TEMPORARY_DIRECTORIES.append(tempdir) atexit.register(tempdir.cleanup) + warnings.simplefilter("default", ResourceWarning) return tempdir.name @@ -223,6 +231,7 @@ def set_write_bit(fn): :param str fn: The target filename or path """ + from .misc import to_bytes, locale_encoding fn = to_bytes(fn, encoding=locale_encoding) if not os.path.exists(fn): return @@ -243,10 +252,17 @@ def rmtree(directory, ignore_errors=False): Setting `ignore_errors=True` may cause this to silently fail to delete the path """ + from .misc import locale_encoding, to_bytes directory = to_bytes(directory, encoding=locale_encoding) - shutil.rmtree( - directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly - ) + try: + shutil.rmtree( + directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly + ) + except (IOError, OSError) as exc: + # Ignore removal failures where the file doesn't exist + if exc.errno == errno.ENOENT: + pass + raise def handle_remove_readonly(func, path, exc): @@ -263,35 +279,41 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ResourceWarning + if six.PY2: + from .compat import ResourceWarning + from .misc import to_bytes + PERM_ERRORS = (errno.EACCES, errno.EPERM) default_warning_message = ( "Unable to remove file due to permissions restriction: {!r}" ) # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc - path = to_bytes(path) + path = to_bytes(path, encoding="utf-8") if is_readonly_path(path): # Apply write permission and call original function set_write_bit(path) try: func(path) except (OSError, IOError) as e: - if e.errno in [errno.EACCES, errno.EPERM]: - warnings.warn( - default_warning_message.format( - to_text(path, encoding=locale_encoding) - ), ResourceWarning - ) + if e.errno in PERM_ERRORS: + warnings.warn(default_warning_message.format(path), ResourceWarning) return - if exc_exception.errno in [errno.EACCES, errno.EPERM]: - warnings.warn( - default_warning_message.format(to_text(path)), - ResourceWarning - ) - return - - raise + if exc_exception.errno in PERM_ERRORS: + set_write_bit(path) + try: + func(path) + except (OSError, IOError) as e: + if e.errno in PERM_ERRORS: + warnings.warn(default_warning_message.format(path), ResourceWarning) + elif e.errno == errno.ENOENT: # File already gone + return + else: + raise + return + else: + raise + raise exc def walk_up(bottom): @@ -356,6 +378,7 @@ def get_converted_relative_path(path, relative_to=None): >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder') '.' """ + from .misc import to_text, to_bytes # noqa if not relative_to: relative_to = os.getcwdu() if six.PY2 else os.getcwd() diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py new file mode 100644 index 0000000000..d2cddd7947 --- /dev/null +++ b/pipenv/vendor/vistir/spin.py @@ -0,0 +1,149 @@ +# -*- coding=utf-8 -*- +import os +import signal +import sys + +from .termcolors import colored +from .compat import fs_str + +import cursor +import functools +try: + import yaspin +except ImportError: + yaspin = None + Spinners = None +else: + from yaspin.spinners import Spinners + +handler = None +if yaspin and os.name == "nt": + handler = yaspin.signal_handlers.default_handler +elif yaspin and os.name != "nt": + handler = yaspin.signal_handlers.fancy_handler + +CLEAR_LINE = chr(27) + "[K" + + +class DummySpinner(object): + def __init__(self, text="", **kwargs): + self.text = text + + def __enter__(self): + if self.text: + self.write(self.text) + return self + + def __exit__(self, exc_type, exc_val, traceback): + if not exc_type: + self.ok() + else: + self.write_err(traceback) + return False + + def fail(self, exitcode=1, text=None): + if text: + self.write_err(text) + raise SystemExit(exitcode, text) + + def ok(self, text=None): + if text: + self.write(self.text) + return 0 + + def write(self, text=None): + if text: + line = fs_str("{0}\n".format(text)) + sys.stdout.write(line) + + def write_err(self, text=None): + if text: + line = fs_str("{0}\n".format(text)) + sys.stderr.write(line) + + +base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner + + +class VistirSpinner(base_obj): + def __init__(self, *args, **kwargs): + """Get a spinner object or a dummy spinner to wrap a context. + + Keyword Arguments: + :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) + :param str start_text: Text to start off the spinner with (default: {None}) + :param dict handler_map: Handler map for signals to be handled gracefully (default: {None}) + :param bool nospin: If true, use the dummy spinner (default: {False}) + """ + + self.handler = handler + sigmap = {} + if handler: + sigmap.update({ + signal.SIGINT: handler, + signal.SIGTERM: handler + }) + handler_map = kwargs.pop("handler_map", {}) + if os.name == "nt": + sigmap[signal.SIGBREAK] = handler + else: + sigmap[signal.SIGALRM] = handler + if handler_map: + sigmap.update(handler_map) + spinner_name = kwargs.pop("spinner_name", "bouncingBar") + text = kwargs.pop("start_text", "") + " " + kwargs.pop("text", "") + if not text: + text = "Running..." + kwargs["sigmap"] = sigmap + kwargs["spinner"] = getattr(Spinners, spinner_name, Spinners.bouncingBar) + super(VistirSpinner, self).__init__(*args, **kwargs) + self.is_dummy = bool(yaspin is None) + + def fail(self, exitcode=1, *args, **kwargs): + super(VistirSpinner, self).fail(**kwargs) + + def ok(self, *args, **kwargs): + super(VistirSpinner, self).ok(*args, **kwargs) + + def write(self, *args, **kwargs): + super(VistirSpinner, self).write(*args, **kwargs) + + def write_err(self, text): + """Write error text in the terminal without breaking the spinner.""" + + sys.stderr.write("\r") + self._clear_err() + text = fs_str("{0}\n".format(text)) + sys.stderr.write(text) + + def _compose_color_func(self): + fn = functools.partial( + colored, + color=self._color, + on_color=self._on_color, + attrs=list(self._attrs), + ) + return fn + + @staticmethod + def _hide_cursor(): + cursor.hide() + + @staticmethod + def _show_cursor(): + cursor.show() + + @staticmethod + def _clear_err(): + sys.stderr.write(CLEAR_LINE) + + @staticmethod + def _clear_line(): + sys.stdout.write(CLEAR_LINE) + + +def create_spinner(*args, **kwargs): + nospin = kwargs.pop("nospin", False) + if nospin: + return DummySpinner(*args, **kwargs) + return VistirSpinner(*args, **kwargs) diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py new file mode 100644 index 0000000000..6f3ad32cf7 --- /dev/null +++ b/pipenv/vendor/vistir/termcolors.py @@ -0,0 +1,116 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals +import colorama +import os + + +ATTRIBUTES = dict( + list(zip([ + 'bold', + 'dark', + '', + 'underline', + 'blink', + '', + 'reverse', + 'concealed' + ], + list(range(1, 9)) + )) + ) +del ATTRIBUTES[''] + + +HIGHLIGHTS = dict( + list(zip([ + 'on_grey', + 'on_red', + 'on_green', + 'on_yellow', + 'on_blue', + 'on_magenta', + 'on_cyan', + 'on_white' + ], + list(range(40, 48)) + )) + ) + + +COLORS = dict( + list(zip([ + 'grey', + 'red', + 'green', + 'yellow', + 'blue', + 'magenta', + 'cyan', + 'white', + ], + list(range(30, 38)) + )) + ) + + +RESET = colorama.Style.RESET_ALL + + +def colored(text, color=None, on_color=None, attrs=None): + """Colorize text using a reimplementation of the colorizer from + https://github.com/pavdmyt/yaspin so that it works on windows. + + Available text colors: + red, green, yellow, blue, magenta, cyan, white. + + Available text highlights: + on_red, on_green, on_yellow, on_blue, on_magenta, on_cyan, on_white. + + Available attributes: + bold, dark, underline, blink, reverse, concealed. + + Example: + colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) + colored('Hello, World!', 'green') + """ + if os.getenv('ANSI_COLORS_DISABLED') is None: + style = "NORMAL" + if 'bold' in attrs: + style = "BRIGHT" + attrs.remove('bold') + if color is not None: + text = text = "%s%s%s%s%s" % ( + getattr(colorama.Fore, color), + getattr(colorama.Style, style), + text, + colorama.Fore.RESET, + colorama.Style.NORMAL, + ) + + if on_color is not None: + text = "%s%s%s%s" % ( + getattr(colorama.Back, color), + text, + colorama.Back.RESET, + colorama.Style.NORMAL, + ) + + if attrs is not None: + fmt_str = "%s[%%dm%%s%s[9m" % ( + chr(27), + chr(27) + ) + for attr in attrs: + text = fmt_str % (ATTRIBUTES[attr], text) + + text += RESET + return text + + +def cprint(text, color=None, on_color=None, attrs=None, **kwargs): + """Print colorize text. + + It accepts arguments of print function. + """ + + print((colored(text, color, on_color, attrs)), **kwargs) From 4c8617237ccdf53bcf87607479afbd9e8cee99bb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 21 Oct 2018 23:54:30 -0400 Subject: [PATCH 02/13] Update requirementslib, requests and vistir Signed-off-by: Dan Ryan --- pipenv/vendor/requests/__init__.py | 17 +-- pipenv/vendor/requests/__version__.py | 4 +- pipenv/vendor/requests/adapters.py | 27 +++-- pipenv/vendor/requests/api.py | 20 ++-- pipenv/vendor/requests/auth.py | 4 +- pipenv/vendor/requests/compat.py | 3 +- pipenv/vendor/requests/cookies.py | 31 ++--- pipenv/vendor/requests/help.py | 3 +- pipenv/vendor/requests/hooks.py | 4 +- pipenv/vendor/requests/models.py | 17 +-- pipenv/vendor/requests/sessions.py | 52 ++++++--- pipenv/vendor/requests/status_codes.py | 2 +- pipenv/vendor/requests/utils.py | 19 ++-- .../requirementslib/models/dependencies.py | 11 +- .../vendor/requirementslib/models/lockfile.py | 49 ++++++-- .../vendor/requirementslib/models/pipfile.py | 78 +++++++++++-- .../requirementslib/models/requirements.py | 4 +- pipenv/vendor/requirementslib/utils.py | 106 ++++++++++++++++++ pipenv/vendor/vistir/contextmanagers.py | 2 +- pipenv/vendor/vistir/misc.py | 55 ++++++++- 20 files changed, 393 insertions(+), 115 deletions(-) diff --git a/pipenv/vendor/requests/__init__.py b/pipenv/vendor/requests/__init__.py index a5b3c9c33f..bc168ee533 100644 --- a/pipenv/vendor/requests/__init__.py +++ b/pipenv/vendor/requests/__init__.py @@ -22,7 +22,7 @@ ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.23 + # urllib3 >= 1.21.1, <= 1.24 assert major == 1 assert minor >= 21 - assert minor <= 23 + assert minor <= 24 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -79,14 +79,14 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) warnings.warn(warning, RequestsDependencyWarning) # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " + warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " "version!".format(urllib3.__version__, chardet.__version__), RequestsDependencyWarning) @@ -123,12 +123,7 @@ def _check_cryptography(cryptography_version): # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/pipenv/vendor/requests/__version__.py b/pipenv/vendor/requests/__version__.py index ef61ec0f55..be8a45fe0e 100644 --- a/pipenv/vendor/requests/__version__.py +++ b/pipenv/vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.19.1' -__build__ = 0x021901 +__version__ = '2.20.0' +__build__ = 0x022000 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/pipenv/vendor/requests/adapters.py b/pipenv/vendor/requests/adapters.py index a4b0284208..fa4d9b3cc9 100644 --- a/pipenv/vendor/requests/adapters.py +++ b/pipenv/vendor/requests/adapters.py @@ -26,6 +26,7 @@ from urllib3.exceptions import ReadTimeoutError from urllib3.exceptions import SSLError as _SSLError from urllib3.exceptions import ResponseError +from urllib3.exceptions import LocationValueError from .models import Response from .compat import urlparse, basestring @@ -35,7 +36,8 @@ from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL, + InvalidURL) from .auth import _basic_auth_str try: @@ -127,8 +129,7 @@ def __init__(self, pool_connections=DEFAULT_POOLSIZE, self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -224,7 +225,7 @@ def cert_verify(self, conn, url, verify, cert): if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + "invalid path: {}".format(cert_loc)) conn.cert_reqs = 'CERT_REQUIRED' @@ -246,10 +247,10 @@ def cert_verify(self, conn, url, verify, cert): conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + "invalid path: {}".format(conn.cert_file)) if conn.key_file and not os.path.exists(conn.key_file): raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + "invalid path: {}".format(conn.key_file)) def build_response(self, req, resp): """Builds a :class:`Response ` object from a urllib3 @@ -378,7 +379,7 @@ def proxy_headers(self, proxy): when subclassing the :class:`HTTPAdapter `. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} @@ -407,7 +408,10 @@ def send(self, request, stream=False, timeout=None, verify=True, cert=None, prox :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection(request.url, proxies) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) @@ -421,7 +425,7 @@ def send(self, request, stream=False, timeout=None, verify=True, cert=None, prox timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " + err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) @@ -471,11 +475,10 @@ def send(self, request, stream=False, timeout=None, verify=True, cert=None, prox # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For compatibility with Python 3.3+ r = low_conn.getresponse() resp = HTTPResponse.from_httplib( diff --git a/pipenv/vendor/requests/api.py b/pipenv/vendor/requests/api.py index a2cc84d769..abada96d46 100644 --- a/pipenv/vendor/requests/api.py +++ b/pipenv/vendor/requests/api.py @@ -18,8 +18,10 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -47,7 +49,7 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') """ @@ -62,7 +64,8 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response @@ -102,7 +105,8 @@ def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -116,7 +120,8 @@ def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -130,7 +135,8 @@ def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object diff --git a/pipenv/vendor/requests/auth.py b/pipenv/vendor/requests/auth.py index 4ae459474d..bdde51c7fd 100644 --- a/pipenv/vendor/requests/auth.py +++ b/pipenv/vendor/requests/auth.py @@ -38,7 +38,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,7 +48,7 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(password), category=DeprecationWarning, diff --git a/pipenv/vendor/requests/compat.py b/pipenv/vendor/requests/compat.py index 6b9c6facb4..c44b35efb9 100644 --- a/pipenv/vendor/requests/compat.py +++ b/pipenv/vendor/requests/compat.py @@ -43,9 +43,8 @@ import cookielib from Cookie import Morsel from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping + from collections import Callable, Mapping, MutableMapping, OrderedDict - from urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str diff --git a/pipenv/vendor/requests/cookies.py b/pipenv/vendor/requests/cookies.py index 50883a84f3..56fccd9c25 100644 --- a/pipenv/vendor/requests/cookies.py +++ b/pipenv/vendor/requests/cookies.py @@ -444,20 +444,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: @@ -511,6 +512,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -529,6 +531,7 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') diff --git a/pipenv/vendor/requests/help.py b/pipenv/vendor/requests/help.py index 06e06b2a75..e53d35ef6d 100644 --- a/pipenv/vendor/requests/help.py +++ b/pipenv/vendor/requests/help.py @@ -89,8 +89,7 @@ def info(): 'version': getattr(idna, '__version__', ''), } - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) + system_ssl = ssl.OPENSSL_VERSION_NUMBER system_ssl_info = { 'version': '%x' % system_ssl if system_ssl is not None else '' } diff --git a/pipenv/vendor/requests/hooks.py b/pipenv/vendor/requests/hooks.py index 32b32de750..7a51f212c8 100644 --- a/pipenv/vendor/requests/hooks.py +++ b/pipenv/vendor/requests/hooks.py @@ -15,14 +15,14 @@ def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/pipenv/vendor/requests/models.py b/pipenv/vendor/requests/models.py index 3d0e1f42a1..3dded57eff 100644 --- a/pipenv/vendor/requests/models.py +++ b/pipenv/vendor/requests/models.py @@ -204,9 +204,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -214,7 +218,7 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() """ @@ -274,7 +278,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() @@ -648,10 +652,7 @@ def __getstate__(self): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): diff --git a/pipenv/vendor/requests/sessions.py b/pipenv/vendor/requests/sessions.py index ba135268ab..a448bd83f2 100644 --- a/pipenv/vendor/requests/sessions.py +++ b/pipenv/vendor/requests/sessions.py @@ -115,6 +115,22 @@ def get_redirect_target(self, resp): return to_native_string(location, 'utf8') return None + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) + and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): + return False + # Standard case: root URI must match + return old_parsed.port != new_parsed.port or old_parsed.scheme != new_parsed.scheme + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests.""" @@ -236,14 +252,10 @@ def rebuild_auth(self, prepared_request, response): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers['Authorization'] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None @@ -299,7 +311,7 @@ def rebuild_method(self, prepared_request, response): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 + # https://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' @@ -325,13 +337,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') """ @@ -453,8 +465,8 @@ def request(self, method, url, :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -550,7 +562,8 @@ def post(self, url, data=None, json=None, **kwargs): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response @@ -562,7 +575,8 @@ def put(self, url, data=None, **kwargs): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -573,7 +587,8 @@ def patch(self, url, data=None, **kwargs): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -723,7 +738,7 @@ def mount(self, prefix, adapter): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -735,7 +750,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/pipenv/vendor/requests/status_codes.py b/pipenv/vendor/requests/status_codes.py index ff462c6c69..813e8c4e62 100644 --- a/pipenv/vendor/requests/status_codes.py +++ b/pipenv/vendor/requests/status_codes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" +r""" The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. diff --git a/pipenv/vendor/requests/utils.py b/pipenv/vendor/requests/utils.py index 431f6be074..0ce7fe115c 100644 --- a/pipenv/vendor/requests/utils.py +++ b/pipenv/vendor/requests/utils.py @@ -173,10 +173,10 @@ def get_netrc_auth(url, raise_errors=False): for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{0}'.format(f)) + loc = os.path.expanduser('~/{}'.format(f)) except KeyError: # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & + # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 return @@ -466,7 +466,7 @@ def _parse_content_type_header(header): if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + params_dict[key.lower()] = value return content_type, params_dict @@ -706,6 +706,10 @@ def should_bypass_proxies(url, no_proxy): no_proxy = get_proxy('no_proxy') parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. @@ -725,7 +729,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += ':{}'.format(parsed.port) for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -733,13 +737,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index ae643517f4..d9f1b65394 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -17,9 +17,10 @@ FormatControl, InstallRequirement, PackageFinder, RequirementPreparer, RequirementSet, RequirementTracker, Resolver, WheelCache, pip_version ) -from vistir.compat import JSONDecodeError, TemporaryDirectory, fs_str +from vistir.compat import JSONDecodeError, fs_str from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass +from vistir.path import create_tracked_tempdir from ..utils import get_pip_command, prepare_pip_source_args, _ensure_dir from .cache import CACHE_DIR, DependencyCache @@ -580,12 +581,12 @@ def start_resolver(finder=None, wheel_cache=None): download_dir = PKGS_DOWNLOAD_DIR _ensure_dir(download_dir) - _build_dir = TemporaryDirectory(fs_str("build")) - _source_dir = TemporaryDirectory(fs_str("source")) + _build_dir = create_tracked_tempdir(fs_str("build")) + _source_dir = create_tracked_tempdir(fs_str("source")) preparer = partialclass( RequirementPreparer, - build_dir=_build_dir.name, - src_dir=_source_dir.name, + build_dir=_build_dir, + src_dir=_source_dir, download_dir=download_dir, wheel_download_dir=WHEEL_DOWNLOAD_DIR, progress_bar="off", diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index bd76ca0177..1997fc1f55 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -14,6 +14,7 @@ from .requirements import Requirement from .utils import optional_instance_of +from ..utils import is_vcs, is_editable, merge_items DEFAULT_NEWLINES = u"\n" @@ -49,6 +50,27 @@ def _get_projectfile(self): def _get_lockfile(self): return self.projectfile.lockfile + def __getitem__(self, k, *args, **kwargs): + retval = None + lockfile = self._lockfile + section = None + pkg_type = None + try: + retval = lockfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(lockfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + def __getattr__(self, k, *args, **kwargs): retval = None lockfile = super(Lockfile, self).__getattribute__("_lockfile") @@ -56,9 +78,18 @@ def __getattr__(self, k, *args, **kwargs): return super(Lockfile, self).__getattribute__(k) except AttributeError: retval = getattr(lockfile, k, None) - if not retval: - retval = super(Lockfile, self).__getattribute__(k, *args, **kwargs) - return retval + if retval is not None: + return retval + return super(Lockfile, self).__getattribute__(k, *args, **kwargs) + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.develop._data) + if only: + return deps + deps = merge_items([deps, self.default._data]) + return deps @classmethod def read_projectfile(cls, path): @@ -135,7 +166,7 @@ def develop(self): def default(self): return self._lockfile.default - def get_requirements(self, dev=False): + def get_requirements(self, dev=True, only=False): """Produces a generator which generates requirements from the desired section. :param bool dev: Indicates whether to use dev requirements, defaults to False @@ -143,20 +174,20 @@ def get_requirements(self, dev=False): :rtype: :class:`~requirementslib.models.requirements.Requirement` """ - section = self.develop if dev else self.default - for k in section.keys(): - yield Requirement.from_pipfile(k, section[k]._data) + deps = self.get_deps(dev=dev, only=only) + for k, v in deps.items(): + yield Requirement.from_pipfile(k, v) @property def dev_requirements(self): if not self._dev_requirements: - self._dev_requirements = list(self.get_requirements(dev=True)) + self._dev_requirements = list(self.get_requirements(dev=True, only=True)) return self._dev_requirements @property def requirements(self): if not self._requirements: - self._requirements = list(self.get_requirements(dev=False)) + self._requirements = list(self.get_requirements(dev=False, only=True)) return self._requirements @property diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 94e9a2a1f9..0d1c04c883 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -6,12 +6,15 @@ import copy import os +import tomlkit + from vistir.compat import Path, FileNotFoundError from .requirements import Requirement from .project import ProjectFile from .utils import optional_instance_of from ..exceptions import RequirementError +from ..utils import is_vcs, is_editable, merge_items import plette.pipfiles @@ -20,6 +23,31 @@ is_projectfile = optional_instance_of(ProjectFile) +class PipfileLoader(plette.pipfiles.Pipfile): + @classmethod + def validate(cls, data): + for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): + if key not in data or key == "source": + continue + klass.validate(data[key]) + + @classmethod + def load(cls, f, encoding=None): + content = f.read() + if encoding is not None: + content = content.decode(encoding) + _data = tomlkit.loads(content) + if "source" not in _data: + # HACK: There is no good way to prepend a section to an existing + # TOML document, but there's no good way to copy non-structural + # content from one TOML document to another either. Modify the + # TOML content directly, and load the new in-memory document. + sep = "" if content.startswith("\n") else "\n" + content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content + data = tomlkit.loads(content) + return cls(data) + + @attr.s(slots=True) class Pipfile(object): path = attr.ib(validator=is_path, type=Path) @@ -40,16 +68,50 @@ def _get_projectfile(self): def _get_pipfile(self): return self.projectfile.model + @property + def pipfile(self): + return self._pipfile + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.pipfile["dev-packages"]._data) + if only: + return deps + deps = merge_items([deps, self.pipfile["packages"]._data]) + return deps + + def __getitem__(self, k, *args, **kwargs): + retval = None + pipfile = self._pipfile + section = None + pkg_type = None + try: + retval = pipfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(pipfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + def __getattr__(self, k, *args, **kwargs): retval = None pipfile = super(Pipfile, self).__getattribute__("_pipfile") try: - return super(Pipfile, self).__getattribute__(k) + retval = super(Pipfile, self).__getattribute__(k) except AttributeError: retval = getattr(pipfile, k, None) - if not retval: - retval = super(Pipfile, self).__getattribute__(k, *args, **kwargs) - return retval + if retval is not None: + return retval + return super(Pipfile, self).__getattribute__(k, *args, **kwargs) @property def requires_python(self): @@ -69,7 +131,7 @@ def read_projectfile(cls, path): """ pf = ProjectFile.read( path, - plette.pipfiles.Pipfile, + PipfileLoader, invalid_ok=True ) return pf @@ -88,7 +150,7 @@ def load_projectfile(cls, path, create=False): if not path: raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): - path = Path(path) + path = Path(path).absolute() pipfile_path = path if path.name == "Pipfile" else path.joinpath("Pipfile") project_path = pipfile_path.parent if not project_path.exists(): @@ -113,10 +175,10 @@ def load(cls, path, create=False): projectfile = cls.load_projectfile(path, create=create) pipfile = projectfile.model dev_requirements = [ - Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("dev-packages", {}).items() + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("dev-packages", {}).items() ] requirements = [ - Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("packages", {}).items() + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("packages", {}).items() ] creation_args = { "projectfile": projectfile, diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index c2768417c3..db004869c6 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -22,11 +22,11 @@ ) from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import FileNotFoundError, Path, TemporaryDirectory +from vistir.compat import FileNotFoundError, Path from vistir.misc import dedup from vistir.path import ( create_tracked_tempdir, get_converted_relative_path, is_file_url, - is_valid_url, mkdir_p + is_valid_url ) from ..exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index b490d3cf8c..abc8983167 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -5,9 +5,15 @@ import logging import os +import boltons.iterutils import six import tomlkit +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) +from six.moves import Mapping, Sequence, Set, ItemsView from six.moves.urllib.parse import urlparse, urlsplit from pip_shims.shims import ( @@ -18,6 +24,8 @@ from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir + + VCS_LIST = ("git", "svn", "hg", "bzr") VCS_SCHEMES = [] SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") @@ -66,6 +74,12 @@ def is_vcs(pipfile_entry): return False +def is_editable(pipfile_entry): + if isinstance(pipfile_entry, Mapping): + return pipfile_entry.get("editable", False) is True + return False + + def multi_split(s, split): """Splits on multiple given separators.""" for r in split: @@ -181,3 +195,95 @@ def ensure_setup_py(base_dir): finally: if is_new: setup_py.unlink() + + +# Modified from https://github.com/mahmoud/boltons/blob/master/boltons/iterutils.py +def dict_path_enter(path, key, value): + if isinstance(value, six.string_types): + return value, False + elif isinstance(value, (Mapping, dict)): + return value.__class__(), ItemsView(value) + elif isinstance(value, tomlkit.items.Array): + return value.__class__([], value.trivia), enumerate(value) + elif isinstance(value, (Sequence, list)): + return value.__class__(), enumerate(value) + elif isinstance(value, (Set, set)): + return value.__class__(), enumerate(value) + else: + return value, False + + +def dict_path_exit(path, key, old_parent, new_parent, new_items): + ret = new_parent + if isinstance(new_parent, (Mapping, dict)): + vals = dict(new_items) + try: + new_parent.update(new_items) + except AttributeError: + # Handle toml containers specifically + try: + new_parent.update(vals) + # Now use default fallback if needed + except AttributeError: + ret = new_parent.__class__(vals) + elif isinstance(new_parent, tomlkit.items.Array): + vals = tomlkit.items.item([v for i, v in new_items]) + try: + new_parent._value.extend(vals._value) + except AttributeError: + ret = tomlkit.items.item(vals) + elif isinstance(new_parent, (Sequence, list)): + vals = [v for i, v in new_items] + try: + new_parent.extend(vals) + except AttributeError: + ret = new_parent.__class__(vals) # tuples + elif isinstance(new_parent, (Set, set)): + vals = [v for i, v in new_items] + try: + new_parent.update(vals) + except AttributeError: + ret = new_parent.__class__(vals) # frozensets + else: + raise RuntimeError('unexpected iterable type: %r' % type(new_parent)) + return ret + + +def merge_items(target_list, sourced=False): + if not sourced: + target_list = [(id(t), t) for t in target_list] + + ret = None + source_map = {} + + def remerge_enter(path, key, value): + new_parent, new_items = dict_path_enter(path, key, value) + if ret and not path and key is None: + new_parent = ret + + try: + cur_val = boltons.iterutils.get_path(ret, path + (key,)) + except KeyError as ke: + pass + else: + new_parent = cur_val + + return new_parent, new_items + + def remerge_exit(path, key, old_parent, new_parent, new_items): + return dict_path_exit(path, key, old_parent, new_parent, new_items) + + for t_name, target in target_list: + if sourced: + def remerge_visit(path, key, value): + source_map[path + (key,)] = t_name + return True + else: + remerge_visit = boltons.iterutils.default_visit + + ret = boltons.iterutils.remap(target, enter=remerge_enter, visit=remerge_visit, + exit=remerge_exit) + + if not sourced: + return ret + return ret, source_map diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 70f95c59b5..8f25e0795b 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -129,7 +129,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): ) else: spinner_name = None - if not start_text: + if not start_text and nospin is False: start_text = "Running..." with spinner_func( spinner_name=spinner_name, diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index f42b4ad165..e2f859853f 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -10,6 +10,7 @@ from collections import OrderedDict from functools import partial +from itertools import islice import six @@ -32,6 +33,9 @@ class WindowsError(OSError): "to_text", "to_bytes", "locale_encoding", + "chunked", + "take", + "divide" ] @@ -283,7 +287,10 @@ def run( cmd = Script.parse(cmd) if block or not return_object: combine_stderr = False - with spinner(spinner_name=spinner_name, start_text="Running...", nospin=nospin) as sp: + start_text = "Running..." + if nospin: + start_text = None + with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin) as sp: return _create_subprocess( cmd, env=_env, @@ -427,6 +434,52 @@ def to_text(string, encoding="utf-8", errors=None): return string +def divide(n, iterable): + """ + split an iterable into n groups, per https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + :return: a list of new iterables derived from the original iterable + :rtype: list + """ + + seq = tuple(iterable) + q, r = divmod(len(seq), n) + + ret = [] + for i in range(n): + start = (i * q) + (i if i < r else r) + stop = ((i + 1) * q) + (i + 1 if i + 1 < r else r) + ret.append(iter(seq[start:stop])) + + return ret + + +def take(n, iterable): + """Take n elements from the supplied iterable without consuming it. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/recipes.py + """ + + return list(islice(iterable, n)) + + +def chunked(n, iterable): + """Split an iterable into lists of length *n*. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/more.py + """ + + return iter(partial(take, n, iter(iterable)), []) + + try: locale_encoding = locale.getdefaultencoding()[1] or "ascii" except Exception: From e375eb3eaa9734f8124c5f7db8aeae3301629eff Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 22 Oct 2018 10:13:34 -0400 Subject: [PATCH 03/13] Fix delegator leaky file handles Signed-off-by: Dan Ryan --- pipenv/vendor/delegator.py | 21 ++++++--- .../vendor/delegator-close-filehandles.patch | 44 +++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 tasks/vendoring/patches/vendor/delegator-close-filehandles.patch diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index d15aeb9783..3ffb2e31ca 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -178,6 +178,7 @@ def run(self, block=True, binary=False, cwd=None, env=None): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() + del popen_kwargs["stdin"] popen_kwargs["universal_newlines"] = not binary if cwd: popen_kwargs["cwd"] = cwd @@ -234,14 +235,22 @@ def block(self): """Blocks until process is complete.""" if self._uses_subprocess: # consume stdout and stderr - try: - stdout, stderr = self.subprocess.communicate() - self.__out = stdout - self.__err = stderr - except ValueError: - pass # Don't read from finished subprocesses. + if self.blocking: + try: + stdout, stderr = self.subprocess.communicate() + self.__out = stdout + self.__err = stderr + except ValueError: + pass # Don't read from finished subprocesses. + else: + self.subprocess.stdin.close() + self.std_out.close() + self.std_err.close() + self.subprocess.wait() else: + self.subprocess.sendeof() self.subprocess.wait() + self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch new file mode 100644 index 0000000000..ac63825c35 --- /dev/null +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -0,0 +1,44 @@ +diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py +index 0c140cad..3ffb2e31 100644 +--- a/pipenv/vendor/delegator.py ++++ b/pipenv/vendor/delegator.py +@@ -178,6 +178,7 @@ class Command(object): + # Use subprocess. + if self.blocking: + popen_kwargs = self._default_popen_kwargs.copy() ++ del popen_kwargs["stdin"] + popen_kwargs["universal_newlines"] = not binary + if cwd: + popen_kwargs["cwd"] = cwd +@@ -233,18 +234,23 @@ class Command(object): + def block(self): + """Blocks until process is complete.""" + if self._uses_subprocess: +- self.subprocess.stdin.close() + # consume stdout and stderr +- try: +- stdout, stderr = self.subprocess.communicate() +- self.__out = stdout +- self.__err = stderr +- except ValueError: +- pass # Don't read from finished subprocesses. ++ if self.blocking: ++ try: ++ stdout, stderr = self.subprocess.communicate() ++ self.__out = stdout ++ self.__err = stderr ++ except ValueError: ++ pass # Don't read from finished subprocesses. ++ else: ++ self.subprocess.stdin.close() ++ self.std_out.close() ++ self.std_err.close() ++ self.subprocess.wait() + else: + self.subprocess.sendeof() +- self.subprocess.proc.stdout.close() + self.subprocess.wait() ++ self.subprocess.proc.stdout.close() + + def pipe(self, command, timeout=None, cwd=None): + """Runs the current command and passes its output to the next From 4dac1676579473e257f68686abdc94191dee237d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 22 Oct 2018 10:16:04 -0400 Subject: [PATCH 04/13] Vendor boltons Signed-off-by: Dan Ryan Update vendored dependencies Signed-off-by: Dan Ryan Fix file handle leaks - Fix #3020 - Fix #3088 - Patch delegator - Add weakref finalizer for tempfiles Signed-off-by: Dan Ryan Fix spinner handlers on windows Signed-off-by: Dan Ryan Fix spinner output and encoding issue Signed-off-by: Dan Ryan fix encoding Signed-off-by: Dan Ryan Fix unicode output on windows, fix tomlkit imports Signed-off-by: Dan Ryan Unvendor boltons, fix compatibility, update merge functionalities Signed-off-by: Dan Ryan Update pythonfinder, vistir version, requirementslib version Signed-off-by: Dan Ryan Fix vendoring script Signed-off-by: Dan Ryan Silence pip version checks Signed-off-by: Dan Ryan Add debugging to locking Signed-off-by: Dan Ryan --- news/{3020.feature => 3020.feature.rst} | 0 news/3088.bugfix.rst | 1 + news/3089.feature.rst | 1 + news/3089.vendor.rst | 11 + pipenv/__init__.py | 1 + pipenv/_compat.py | 6 +- pipenv/core.py | 535 +++++++++--------- pipenv/environments.py | 9 +- pipenv/project.py | 76 ++- pipenv/resolver.py | 86 +-- pipenv/utils.py | 195 +++---- pipenv/vendor/delegator.py | 17 +- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/models/path.py | 1 - pipenv/vendor/requirementslib/__init__.py | 9 +- pipenv/vendor/requirementslib/exceptions.py | 2 + pipenv/vendor/requirementslib/models/cache.py | 16 +- .../vendor/requirementslib/models/lockfile.py | 26 + .../vendor/requirementslib/models/pipfile.py | 14 +- .../requirementslib/models/requirements.py | 48 +- .../requirementslib/models/resolvers.py | 11 +- pipenv/vendor/requirementslib/models/utils.py | 23 +- pipenv/vendor/requirementslib/utils.py | 283 ++++++++- pipenv/vendor/tomlkit/items.py | 2 +- pipenv/vendor/tomlkit/toml_char.py | 2 +- pipenv/vendor/vendor.txt | 6 +- pipenv/vendor/vistir/__init__.py | 7 +- pipenv/vendor/vistir/backports/tempfile.py | 6 +- pipenv/vendor/vistir/compat.py | 3 +- pipenv/vendor/vistir/contextmanagers.py | 26 +- pipenv/vendor/vistir/misc.py | 6 +- pipenv/vendor/vistir/path.py | 71 ++- pipenv/vendor/vistir/spin.py | 49 +- pipenv/vendor/vistir/termcolors.py | 4 +- pipenv/vendor/yaspin/core.py | 27 +- tasks/vendoring/__init__.py | 2 + .../vendor/delegator-close-filehandles.patch | 57 +- .../patches/vendor/vistir-imports.patch | 39 +- .../vendor/yaspin-signal-handling.patch | 62 ++ tests/integration/conftest.py | 138 ++++- tests/integration/test_lock.py | 4 +- tests/unit/test_utils.py | 11 +- 42 files changed, 1334 insertions(+), 561 deletions(-) rename news/{3020.feature => 3020.feature.rst} (100%) create mode 100644 news/3088.bugfix.rst create mode 100644 news/3089.feature.rst create mode 100644 news/3089.vendor.rst create mode 100644 tasks/vendoring/patches/vendor/yaspin-signal-handling.patch diff --git a/news/3020.feature b/news/3020.feature.rst similarity index 100% rename from news/3020.feature rename to news/3020.feature.rst diff --git a/news/3088.bugfix.rst b/news/3088.bugfix.rst new file mode 100644 index 0000000000..b10c4b2b3d --- /dev/null +++ b/news/3088.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when PIP awaited input from users who put login credentials in their environment. diff --git a/news/3089.feature.rst b/news/3089.feature.rst new file mode 100644 index 0000000000..47f280ee6a --- /dev/null +++ b/news/3089.feature.rst @@ -0,0 +1 @@ +Added windows-compatible spinner via upgraded ``vistir`` dependency. diff --git a/news/3089.vendor.rst b/news/3089.vendor.rst new file mode 100644 index 0000000000..c57209f665 --- /dev/null +++ b/news/3089.vendor.rst @@ -0,0 +1,11 @@ +Updated vendored dependencies: + - ``certifi 2018.08.24 => 2018.10.15`` + - ``urllib3 1.23 => 1.24`` + - ``requests 2.19.1 => 2.20.0`` + - ``shellingham ``1.2.6 => 1.2.7`` + - ``tomlkit 0.4.4. => 0.4.6`` + - ``vistir 0.1.6 => 0.1.8`` + - ``pythonfinder 0.1.2 => 0.1.3`` + - ``requirementslib 1.1.9 => 1.1.10`` + - ``backports.functools_lru_cache 1.5.0 (new)`` + - ``cursor 1.2.0 (new)`` diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 471dcb947e..1fea44d514 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -14,6 +14,7 @@ sys.path.insert(0, PIPENV_VENDOR) # Inject patched directory into system path. sys.path.insert(0, PIPENV_PATCHED) +os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" # Hack to make things work better. try: if "concurrency" in sys.modules: diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 558df3b846..223baec1e2 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -59,10 +59,10 @@ def detach(self): return False -if six.PY2: +from vistir.compat import ResourceWarning - class ResourceWarning(Warning): - pass + +warnings.filterwarnings("ignore", category=ResourceWarning) def pip_import(module_path, subimport=None, old_path=None): diff --git a/pipenv/core.py b/pipenv/core.py index ca37bd5c0c..82c11f1d31 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -5,7 +5,6 @@ import sys import shutil import time -import tempfile import json as simplejson import click import click_completion @@ -13,10 +12,12 @@ import dotenv import delegator import pipfile -from blindspin import spinner import vistir +import warnings import six +import urllib3.util as urllib3_util + from .cmdparse import Script from .project import Project, SourceNotFound from .utils import ( @@ -24,8 +25,6 @@ is_required_version, proper_case, pep423_name, - split_file, - merge_deps, venv_resolve_deps, escape_grouped_arguments, python_version, @@ -45,9 +44,7 @@ from . import environments, pep508checker, progress from .environments import ( PIPENV_COLORBLIND, - PIPENV_NOSPIN, PIPENV_SHELL_FANCY, - PIPENV_TIMEOUT, PIPENV_SKIP_VALIDATION, PIPENV_HIDE_EMOJIS, PIPENV_YES, @@ -96,12 +93,36 @@ # Disable colors, for the color blind and others who do not prefer colors. if PIPENV_COLORBLIND: crayons.disable() -# Disable spinner, for cleaner build logs (the unworthy). -if PIPENV_NOSPIN: - @contextlib.contextmanager # noqa: F811 - def spinner(): - yield + +UNICODE_TO_ASCII_TRANSLATION_MAP = { + 8230: u"...", + 8211: u"-" +} + + +def fix_utf8(text): + if not isinstance(text, six.string_types): + return text + if six.PY2: + text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) + return u"{0}".format(text) + + +@contextlib.contextmanager +def spinner(text=None, nospin=None, spinner_name=None): + if not text: + text = "Running..." + if not spinner_name: + spinner_name = environments.PIPENV_SPINNER + if nospin is None: + nospin = environments.PIPENV_NOSPIN + with vistir.spin.create_spinner( + spinner_name=spinner_name, + start_text=text, + nospin=nospin + ) as sp: + yield sp def which(command, location=None, allow_global=False): @@ -132,7 +153,7 @@ def which(command, location=None, allow_global=False): def do_clear(): - click.echo(crayons.white("Clearing caches…", bold=True)) + click.echo(crayons.white(fix_utf8("Clearing caches…"), bold=True)) try: from pip._internal import locations except ImportError: # pip 9. @@ -161,7 +182,7 @@ def load_dot_env(): if os.path.isfile(dotenv_file): click.echo( - crayons.normal("Loading .env environment variables…", bold=True), + crayons.normal(fix_utf8("Loading .env environment variables…"), bold=True), err=True, ) else: @@ -295,7 +316,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): if project.requirements_exists and not skip_requirements: click.echo( crayons.normal( - u"requirements.txt found, instead of Pipfile! Converting…", + fix_utf8("requirements.txt found, instead of Pipfile! Converting…"), bold=True, ) ) @@ -317,7 +338,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): ) else: click.echo( - crayons.normal(u"Creating a Pipfile for this project…", bold=True), + crayons.normal(fix_utf8("Creating a Pipfile for this project…"), bold=True), err=True, ) # Create the pipfile if it doesn't exist. @@ -405,7 +426,7 @@ def abort(): u"{0}: Python {1} {2}".format( crayons.red("Warning", bold=True), crayons.blue(python), - u"was not found on your system…", + fix_utf8("was not found on your system…"), ), err=True, ) @@ -424,7 +445,7 @@ def abort(): except ValueError: abort() except PyenvError as e: - click.echo(u"Something went wrong…") + click.echo(fix_utf8("Something went wrong…")) click.echo(crayons.blue(e.err), err=True) abort() s = "{0} {1} {2}".format( @@ -443,14 +464,14 @@ def abort(): crayons.green(u"CPython {0}".format(version), bold=True), crayons.normal(u"with pyenv", bold=True), crayons.normal(u"(this may take a few minutes)"), - crayons.normal(u"…", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) ) with spinner(): try: c = pyenv.install(version) except PyenvError as e: - click.echo(u"Something went wrong…") + click.echo(fix_utf8("Something went wrong…")) click.echo(crayons.blue(e.err), err=True) # Print the results, in a beautiful blue… click.echo(crayons.blue(c.out), err=True) @@ -518,7 +539,7 @@ def abort(): ): abort() click.echo( - crayons.normal(u"Removing existing virtualenv…", bold=True), err=True + crayons.normal(fix_utf8("Removing existing virtualenv…"), bold=True), err=True ) # Remove the virtualenv. cleanup_virtualenv(bare=True) @@ -665,26 +686,31 @@ def do_install_dependencies( If requirements is True, simply spits out a requirements format to stdout. """ - from .vendor.requirementslib.models.requirements import Requirement + from six.moves import queue def cleanup_procs(procs, concurrent): - for c in procs: - if concurrent: - c.block() + while not procs.empty(): + c = procs.get() + # if concurrent: + c.block() + failed = False + if c.return_code != 0: + failed = True if "Ignoring" in c.out: click.echo(crayons.yellow(c.out.strip())) elif environments.is_verbose(): - click.echo(crayons.blue(c.out or c.err)) + click.echo(crayons.blue(c.out.strip() or c.err.strip())) # The Installation failed… - if c.return_code != 0: + if failed: # Save the Failed Dependency for later. - failed_deps_list.append((c.dep, c.ignore_hash)) + dep = c.dep.copy() + failed_deps_list.append(dep) # Alert the user. click.echo( "{0} {1}! Will try again.".format( crayons.red("An error occurred while installing"), - crayons.green(c.dep.as_line()), - ) + crayons.green(dep.as_line()), + ), err=True ) if requirements: @@ -694,143 +720,138 @@ def cleanup_procs(procs, concurrent): if skip_lock or only or not project.lockfile_exists: if not bare: click.echo( - crayons.normal(u"Installing dependencies from Pipfile…", bold=True) + crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True) ) - lockfile = split_file(project._lockfile) + lockfile = project.get_or_create_lockfile() else: - with open(project.lockfile_location) as f: - lockfile = split_file(simplejson.load(f)) + lockfile = project.get_or_create_lockfile() if not bare: click.echo( crayons.normal( - u"Installing dependencies from Pipfile.lock ({0})…".format( + fix_utf8("Installing dependencies from Pipfile.lock ({0})…".format( lockfile["_meta"].get("hash", {}).get("sha256")[-6:] - ), + )), bold=True, ) ) # Allow pip to resolve dependencies when in skip-lock mode. no_deps = not skip_lock - deps_list, dev_deps_list = merge_deps( - lockfile, - project, - dev=dev, - requirements=requirements, - ignore_hashes=ignore_hashes, - blocking=blocking, - only=only, - ) failed_deps_list = [] + deps_list = list(lockfile.get_requirements(dev=dev, only=only)) if requirements: index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") - deps_list = [dep for dep, ignore_hash, block in deps_list] - dev_deps_list = [dep for dep, ignore_hash, block in dev_deps_list] + deps = [ + req.as_line(sources=project.sources, include_hashes=False) for req in deps_list + ] # Output only default dependencies click.echo(index_args) - if not dev: - click.echo( - "\n".join(d.partition("--hash")[0].strip() for d in sorted(deps_list)) - ) - sys.exit(0) - # Output only dev dependencies - if dev: - click.echo( - "\n".join( - d.partition("--hash")[0].strip() for d in sorted(dev_deps_list) - ) - ) - sys.exit(0) - procs = [] - deps_list_bar = progress.bar( - deps_list, label=INSTALL_LABEL if os.name != "nt" else "" - ) - for dep, ignore_hash, block in deps_list_bar: - if len(procs) < PIPENV_MAX_SUBPROCESS: - # Use a specific index, if specified. - indexes, trusted_hosts, dep = parse_indexes(dep) - index = None - extra_indexes = [] - if indexes: - index = indexes[0] - if len(indexes) > 0: - extra_indexes = indexes[1:] - dep = Requirement.from_line(" ".join(dep)) - if index: - _index = None - try: - _index = project.find_source(index).get("name") - except SourceNotFound: - _index = None - dep.index = _index - dep._index = index - dep.extra_indexes = extra_indexes - # Install the module. - prev_no_deps_setting = no_deps - if dep.is_file_or_url and any( - dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] - ): - no_deps = False + click.echo( + "\n".join(sorted(deps)) + ) + sys.exit(0) + + procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) + trusted_hosts = [] + + deps_list_bar = progress.bar(deps_list, width=32, + label=INSTALL_LABEL if os.name != "nt" else "") + + indexes = [] + for dep in deps_list_bar: + index = None + if dep.index: + index = project.find_source(dep.index) + indexes.append(index) + if not index.get("verify_ssl", False): + trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) + # Install the module. + is_artifact = False + if dep.is_file_or_url and any( + dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] + ): + is_artifact = True + + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + with vistir.contextmanagers.temp_environ(): + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] c = pip_install( dep, - ignore_hashes=ignore_hash, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, - no_deps=no_deps, - block=block, + no_deps=False if is_artifact else no_deps, + block=any([dep.editable, blocking]), index=index, requirements_dir=requirements_dir, - extra_indexes=extra_indexes, pypi_mirror=pypi_mirror, - trusted_hosts=trusted_hosts + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes ) - c.dep = dep - c.ignore_hash = ignore_hash - c.index = index - c.extra_indexes = extra_indexes - procs.append(c) - no_deps = prev_no_deps_setting - if len(procs) >= PIPENV_MAX_SUBPROCESS or len(procs) == len(deps_list): - cleanup_procs(procs, concurrent) - procs = [] - cleanup_procs(procs, concurrent) + if procs.qsize() < PIPENV_MAX_SUBPROCESS: + c.dep = dep + procs.put(c) + + if procs.full() or procs.qsize() == len(deps_list): + cleanup_procs(procs, concurrent) + if not procs.empty(): + cleanup_procs(procs, concurrent) + # Iterate over the hopefully-poorly-packaged dependencies… if failed_deps_list: click.echo( - crayons.normal(u"Installing initially failed dependencies…", bold=True) + crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True) ) - for dep, ignore_hash in progress.bar(failed_deps_list, label=INSTALL_LABEL2): + for dep in progress.bar(failed_deps_list, label=INSTALL_LABEL2): # Use a specific index, if specified. # Install the module. - prev_no_deps_setting = no_deps + is_artifact = False + index = None + if dep.index: + index = project.find_source(dep.index) if dep.is_file_or_url and any( dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] ): - no_deps = False - c = pip_install( - dep, - ignore_hashes=ignore_hash, - allow_global=allow_global, - no_deps=no_deps, - index=getattr(dep, "_index", None), - requirements_dir=requirements_dir, - extra_indexes=getattr(dep, "extra_indexes", None), - ) - no_deps = prev_no_deps_setting - # The Installation failed… - if c.return_code != 0: - # We echo both c.out and c.err because pip returns error details on out. - click.echo(crayons.blue(format_pip_output(c.out))) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - # Return the subprocess' return code. - sys.exit(c.return_code) - else: - click.echo( - "{0} {1}{2}".format( - crayons.green("Success installing"), - crayons.green(dep.name), - crayons.green("!"), - ) + is_artifact = True + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + with vistir.contextmanagers.temp_environ(): + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] + c = pip_install( + dep, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), + allow_global=allow_global, + no_deps=True if is_artifact else no_deps, + index=index, + requirements_dir=requirements_dir, + pypi_mirror=pypi_mirror, + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes, + block=True ) + # The Installation failed… + if c.return_code != 0: + # We echo both c.out and c.err because pip returns error details on out. + click.echo(crayons.blue(format_pip_output(c.out))) + click.echo(crayons.blue(format_pip_error(c.err)), err=True) + # Return the subprocess' return code. + sys.exit(c.return_code) + else: + click.echo( + "{0} {1}{2}".format( + crayons.green("Success installing"), + crayons.green(dep.name), + crayons.green("!"), + ) + ) def convert_three_to_python(three, python): @@ -851,7 +872,7 @@ def convert_three_to_python(three, python): def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv.""" click.echo( - crayons.normal(u"Creating a virtualenv for this project…", bold=True), err=True + crayons.normal(fix_utf8("Creating a virtualenv for this project…"), bold=True), err=True ) click.echo( u"Pipfile: {0}".format(crayons.red(project.pipfile_location, bold=True)), @@ -865,14 +886,14 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): u"{0} {1} {3} {2}".format( crayons.normal("Using", bold=True), crayons.red(python, bold=True), - crayons.normal(u"to create virtualenv…", bold=True), + crayons.normal(fix_utf8("to create virtualenv…"), bold=True), crayons.green("({0})".format(python_version(python))), ), err=True, ) cmd = [ - sys.executable, + vistir.compat.Path(sys.executable).absolute().as_posix(), "-m", "virtualenv", "--prompt=({0}) ".format(project.name), @@ -883,7 +904,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): # Pass site-packages flag to virtualenv, if desired… if site_packages: click.echo( - crayons.normal(u"Making site-packages available…", bold=True), err=True + crayons.normal(fix_utf8("Making site-packages available…"), bold=True), err=True ) cmd.append("--system-site-packages") @@ -893,11 +914,12 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): pip_config = {} # Actually create the virtualenv. - with spinner(): - c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config) - c.block() + nospin = os.environ.get("PIPENV_ACTIVE", environments.PIPENV_NOSPIN) + c = vistir.misc.run(cmd, verbose=False, return_object=True, + spinner_name=environments.PIPENV_SPINNER, combine_stderr=False, + block=False, nospin=nospin, env=pip_config) click.echo(crayons.blue("{0}".format(c.out)), err=True) - if c.return_code != 0: + if c.returncode != 0: click.echo(crayons.blue("{0}".format(c.err)), err=True) click.echo( u"{0}: Failed to create virtual environment.".format( @@ -1019,9 +1041,9 @@ def do_lock( # Alert the user of progress. click.echo( u"{0} {1} {2}".format( - crayons.normal("Locking"), - crayons.red("[{0}]".format(settings["log_string"])), - crayons.normal("dependencies…"), + crayons.normal(u"Locking"), + crayons.red(u"[{0}]".format(settings["log_string"])), + crayons.normal(fix_utf8("dependencies…")), ), err=True, ) @@ -1125,7 +1147,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): if downloads: if not bare: - click.echo(crayons.normal(u"Clearing out downloads directory…", bold=True)) + click.echo(crayons.normal(fix_utf8("Clearing out downloads directory…"), bold=True)) shutil.rmtree(project.download_location) return @@ -1154,7 +1176,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): actually_installed.append(dep) if not bare: click.echo( - u"Found {0} installed package(s), purging…".format(len(actually_installed)) + fix_utf8("Found {0} installed package(s), purging…".format(len(actually_installed))) ) command = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=allow_global)), @@ -1185,7 +1207,6 @@ def do_init( """Executes the init functionality.""" from .environments import PIPENV_VIRTUALENV - cleanup_reqdir = False if not system: if not project.virtualenv_exists: try: @@ -1197,8 +1218,7 @@ def do_init( if not deploy: ensure_pipfile(system=system) if not requirements_dir: - cleanup_reqdir = True - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored @@ -1215,26 +1235,25 @@ def do_init( ) ) click.echo(crayons.normal("Aborting deploy.", bold=True), err=True) - requirements_dir.cleanup() sys.exit(1) elif (system or allow_global) and not (PIPENV_VIRTUALENV): click.echo( - crayons.red( - u"Pipfile.lock ({0}) out of date, but installation " - u"uses {1}… re-building lockfile must happen in " - u"isolation. Please rebuild lockfile in a virtualenv. " - u"Continuing anyway…".format( + crayons.red(fix_utf8( + "Pipfile.lock ({0}) out of date, but installation " + "uses {1}… re-building lockfile must happen in " + "isolation. Please rebuild lockfile in a virtualenv. " + "Continuing anyway…".format( crayons.white(old_hash[-6:]), crayons.white("--system") - ), + )), bold=True, ), err=True, ) else: if old_hash: - msg = u"Pipfile.lock ({1}) out of date, updating to ({0})…" + msg = fix_utf8("Pipfile.lock ({1}) out of date, updating to ({0})…") else: - msg = u"Pipfile.lock is corrupted, replaced with ({0})…" + msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({0})…") click.echo( crayons.red(msg.format(old_hash[-6:], new_hash[-6:]), bold=True), err=True, @@ -1259,11 +1278,10 @@ def do_init( err=True, ) click.echo("See also: --deploy flag.", err=True) - requirements_dir.cleanup() sys.exit(1) else: click.echo( - crayons.normal(u"Pipfile.lock not found, creating…", bold=True), + crayons.normal(fix_utf8("Pipfile.lock not found, creating…"), bold=True), err=True, ) do_lock( @@ -1279,11 +1297,9 @@ def do_init( allow_global=allow_global, skip_lock=skip_lock, concurrent=concurrent, - requirements_dir=requirements_dir.name, + requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, ) - if cleanup_reqdir: - requirements_dir.cleanup() # Hint the user what to do to activate the virtualenv. if not allow_global and not deploy and "PIPENV_ACTIVE" not in os.environ: @@ -1312,14 +1328,14 @@ def pip_install( trusted_hosts=None ): from notpip._internal import logger as piplogger + from .utils import Mapping from .vendor.urllib3.util import parse_url src = [] write_to_tmpfile = False if requirement: - editable_with_markers = requirement.editable and requirement.markers needs_hashes = not requirement.editable and not ignore_hashes and r is None - write_to_tmpfile = needs_hashes or editable_with_markers + write_to_tmpfile = needs_hashes if not trusted_hosts: trusted_hosts = [] @@ -1333,12 +1349,16 @@ def pip_install( ) # Create files for hash mode. if write_to_tmpfile: - with vistir.compat.NamedTemporaryFile( + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements") + f = vistir.compat.NamedTemporaryFile( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, delete=False - ) as f: - f.write(vistir.misc.to_bytes(requirement.as_line())) - r = f.name + ) + f.write(vistir.misc.to_bytes(requirement.as_line())) + r = f.name + f.close() # Install dependencies when a package is a VCS dependency. if requirement and requirement.vcs: no_deps = False @@ -1348,21 +1368,27 @@ def pip_install( # Try installing for each source in project.sources. if index: - try: - index_source = project.find_source(index) - index_source = index_source.copy() - except SourceNotFound: - src_name = project.src_name_from_url(index) - index_url = parse_url(index) - verify_ssl = index_url.host not in trusted_hosts - index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} + if isinstance(index, (Mapping, dict)): + index_source = index + else: + try: + index_source = project.find_source(index) + index_source = index_source.copy() + except SourceNotFound: + src_name = project.src_name_from_url(index) + index_url = parse_url(index) + verify_ssl = index_url.host not in trusted_hosts + index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} sources = [index_source.copy(),] if extra_indexes: if isinstance(extra_indexes, six.string_types): extra_indexes = [extra_indexes,] for idx in extra_indexes: + extra_src = None + if isinstance(idx, (Mapping, dict)): + extra_src = idx try: - extra_src = project.find_source(idx) + extra_src = project.find_source(idx) if not extra_src else extra_src except SourceNotFound: src_name = project.src_name_from_url(idx) src_url = parse_url(idx) @@ -1382,12 +1408,15 @@ def pip_install( for source in sources ] if (requirement and requirement.editable) and not r: - install_reqs = requirement.as_line(as_list=True) + line_kwargs = {"as_list": True} + if requirement.markers: + line_kwargs["include_markers"] = False + install_reqs = requirement.as_line(**line_kwargs) if requirement.editable and install_reqs[0].startswith("-e "): req, install_reqs = install_reqs[0], install_reqs[1:] editable_opt, req = req.split(" ", 1) install_reqs = [editable_opt, req] + install_reqs - if not any(item.startswith("--hash") for item in install_reqs): + if not all(item.startswith("--hash") for item in install_reqs): ignore_hashes = True elif r: install_reqs = ["-r", r] @@ -1396,7 +1425,7 @@ def pip_install( ignore_hashes = True else: ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) + install_reqs = [escape_cmd(r) for r in requirement.as_line(as_list=True)] pip_command = [which_pip(allow_global=allow_global), "install"] if pre: pip_command.append("--pre") @@ -1409,7 +1438,6 @@ def pip_install( pip_command.append("--upgrade-strategy=only-if-needed") if no_deps: pip_command.append("--no-deps") - install_reqs = [escape_cmd(req) for req in install_reqs] pip_command.extend(install_reqs) pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: @@ -1431,7 +1459,8 @@ def pip_install( pip_config.update( {"PIP_SRC": vistir.misc.fs_str(project.virtualenv_src_location)} ) - pip_command = Script.parse(pip_command).cmdify() + cmd = Script.parse(pip_command) + pip_command = cmd.cmdify() c = delegator.run(pip_command, block=block, env=pip_config) return c @@ -1623,9 +1652,9 @@ def ensure_lockfile(keep_outdated=False, pypi_mirror=None): if new_hash != old_hash: click.echo( crayons.red( - u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( + fix_utf8("Pipfile.lock ({0}) out of date, updating to ({1})…".format( old_hash[-6:], new_hash[-6:] - ), + )), bold=True, ), err=True, @@ -1700,9 +1729,10 @@ def do_install( from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM from notpip._internal.exceptions import PipError - requirements_directory = vistir.compat.TemporaryDirectory( + requirements_directory = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) + warnings.filterwarnings("default", category=vistir.compat.ResourceWarning) if selective_upgrade: keep_outdated = True packages = packages if packages else [] @@ -1739,27 +1769,29 @@ def do_install( err=True, ) click.echo("See also: --deploy flag.", err=True) - requirements_directory.cleanup() sys.exit(1) # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True # Check if the file is remote or not if remote: - fd, temp_reqs = tempfile.mkstemp( - prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory.name - ) - requirements_url = requirements - # Download requirements file click.echo( crayons.normal( - u"Remote requirements file provided! Downloading…", bold=True + fix_utf8("Remote requirements file provided! Downloading…"), bold=True ), err=True, ) + fd = vistir.path.create_tracked_tempfile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory + ) + temp_reqs = fd.name + requirements_url = requirements + # Download requirements file try: download_file(requirements, temp_reqs) except IOError: + fd.close() + os.unlink(temp_reqs) click.echo( crayons.red( u"Unable to find requirements file at {0}.".format( @@ -1768,8 +1800,9 @@ def do_install( ), err=True, ) - requirements_directory.cleanup() sys.exit(1) + finally: + fd.close() # Replace the url with the temporary requirements file requirements = temp_reqs remote = True @@ -1777,7 +1810,7 @@ def do_install( error, traceback = None, None click.echo( crayons.normal( - u"Requirements file provided! Importing into Pipfile…", bold=True + fix_utf8("Requirements file provided! Importing into Pipfile…"), bold=True ), err=True, ) @@ -1800,16 +1833,15 @@ def do_install( finally: # If requirements file was provided by remote url delete the temporary file if remote: - os.close(fd) # Close for windows to allow file cleanup. - os.remove(project.path_to(temp_reqs)) + fd.close() # Close for windows to allow file cleanup. + os.remove(temp_reqs) if error and traceback: click.echo(crayons.red(error)) click.echo(crayons.blue(str(traceback)), err=True) - requirements_directory.cleanup() sys.exit(1) if code: click.echo( - crayons.normal(u"Discovering imports from local codebase…", bold=True) + crayons.normal(fix_utf8("Discovering imports from local codebase…"), bold=True) ) for req in import_from_code(code): click.echo(" Found {0}!".format(crayons.green(req))) @@ -1886,17 +1918,21 @@ def do_install( for pkg_line in pkg_list: click.echo( crayons.normal( - u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), + fix_utf8("Installing {0}…".format(crayons.green(pkg_line, bold=True))), bold=True, ) ) # pip install: - with spinner(): + with vistir.contextmanagers.temp_environ(), spinner(text="Installing...", + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: - click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) - requirements_directory.cleanup() + sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) sys.exit(1) if index_url: pkg_requirement.index = index_url @@ -1907,14 +1943,14 @@ def do_install( selective_upgrade=selective_upgrade, no_deps=False, pre=pre, - requirements_dir=requirements_directory.name, + requirements_dir=requirements_directory, index=index_url, extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) # Warn if --editable wasn't passed. if pkg_requirement.is_vcs and not pkg_requirement.editable: - click.echo( + sp.write_err( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." "\n To enable this sub-dependency functionality, specify that this dependency is editable." @@ -1923,37 +1959,34 @@ def do_install( crayons.red("$ pipenv lock"), ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - try: - assert c.return_code == 0 - except AssertionError: - click.echo( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - err=True, - ) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - if "setup.py egg_info" in c.err: - click.echo( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) + click.echo(crayons.blue(format_pip_output(c.out))) + # Ensure that package was successfully installed. + if c.return_code != 0: + sp.write_err(vistir.compat.fs_str( + "{0} An error occurred while installing {1}!".format( + crayons.red("Error: ", bold=True), crayons.green(pkg_line) ), - err=True, + )) + sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) + sp.write(vistir.compat.fs_str( + u"{0} {1} {2} {3}{4}".format( + crayons.normal(u"Adding", bold=True), + crayons.green(u"{0}".format(pkg_requirement.name), bold=True), + crayons.normal(u"to Pipfile's", bold=True), + crayons.red(u"[dev-packages]" if dev else u"[packages]", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) - requirements_directory.cleanup() - sys.exit(1) - click.echo( - "{0} {1} {2} {3}{4}".format( - crayons.normal("Adding", bold=True), - crayons.green(pkg_requirement.name, bold=True), - crayons.normal("to Pipfile's", bold=True), - crayons.red("[dev-packages]" if dev else "[packages]", bold=True), - crayons.normal("…", bold=True), - ) - ) + )) + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded")) # Add the package to the Pipfile. try: project.add_package_to_pipfile(pkg_requirement, dev) @@ -1975,7 +2008,6 @@ def do_install( pypi_mirror=pypi_mirror, skip_lock=skip_lock, ) - requirements_directory.cleanup() sys.exit(0) @@ -2009,7 +2041,7 @@ def do_uninstall( # Un-install all dependencies, if --all was provided. if all is True: click.echo( - crayons.normal(u"Un-installing all packages from virtualenv…", bold=True) + crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) ) do_purge(allow_global=system) return @@ -2025,7 +2057,7 @@ def do_uninstall( return click.echo( crayons.normal( - u"Un-installing {0}…".format(crayons.red("[dev-packages]")), bold=True + fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True ) ) package_names = project.dev_packages.keys() @@ -2033,7 +2065,7 @@ def do_uninstall( click.echo(crayons.red("No package provided!"), err=True) return 1 for package_name in package_names: - click.echo(u"Un-installing {0}…".format(crayons.green(package_name))) + click.echo(fix_utf8("Un-installing {0}…".format(crayons.green(package_name)))) cmd = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=system)), package_name ) @@ -2055,7 +2087,7 @@ def do_uninstall( continue click.echo( - u"Removing {0} from Pipfile…".format(crayons.green(package_name)) + fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) ) # Remove package from both packages and dev-packages. project.remove_package_from_pipfile(package_name, dev=True) @@ -2076,7 +2108,7 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror from .shells import choose_shell shell = choose_shell() - click.echo("Launching subshell in virtual environment…", err=True) + click.echo(fix_utf8("Launching subshell in virtual environment…"), err=True) fork_args = (project.virtualenv_location, project.project_directory, shell_args) @@ -2087,9 +2119,9 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror try: shell.fork_compat(*fork_args) except (AttributeError, ImportError): - click.echo( - u"Compatibility mode not supported. " - u"Trying to continue as well-configured shell…", + click.echo(fix_utf8( + "Compatibility mode not supported. " + "Trying to continue as well-configured shell…"), err=True, ) shell.fork(*fork_args) @@ -2099,11 +2131,11 @@ def _inline_activate_virtualenv(): try: activate_this = which("activate_this.py") if not activate_this or not os.path.exists(activate_this): - click.echo( - u"{0}: activate_this.py not found. Your environment is most " - u"certainly not activated. Continuing anyway…" - u"".format(crayons.red("Warning", bold=True)), - err=True, + click.echo(fix_utf8( + "{0}: activate_this.py not found. Your environment is most " + "certainly not activated. Continuing anyway…").format( + crayons.red("Warning", bold=True) + ), err=True, ) return with open(activate_this) as f: @@ -2267,7 +2299,7 @@ def do_check( sys.exit(1) else: sys.exit(0) - click.echo(crayons.normal(u"Checking PEP 508 requirements…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking PEP 508 requirements…"), bold=True)) if system: python = system_which("python") else: @@ -2303,7 +2335,7 @@ def do_check( sys.exit(1) else: click.echo(crayons.green("Passed!")) - click.echo(crayons.normal(u"Checking installed package safety…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking installed package safety…"), bold=True)) path = pep508checker.__file__.rstrip("cdo") path = os.sep.join(__file__.split(os.sep)[:-1] + ["patched", "safety.zip"]) if not system: @@ -2501,7 +2533,7 @@ def do_sync( ) # Install everything. - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) do_init( @@ -2513,7 +2545,6 @@ def do_sync( deploy=deploy, system=system, ) - requirements_dir.cleanup() click.echo(crayons.green("All dependencies are now up-to-date!")) @@ -2548,7 +2579,7 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro else: click.echo( crayons.white( - "Uninstalling {0}…".format(repr(apparent_bad_package)), bold=True + fix_utf8("Uninstalling {0}…".format(repr(apparent_bad_package))), bold=True ) ) # Uninstall the package. diff --git a/pipenv/environments.py b/pipenv/environments.py index e6de4f0d73..c58741363e 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -1,7 +1,9 @@ +# -*- coding=utf-8 -*- + import os import sys from appdirs import user_cache_dir -from .vendor.vistir.misc import fs_str +from .vendor.vistir.misc import fs_str, to_text # HACK: avoid resolver.py uses the wrong byte code files. @@ -259,3 +261,8 @@ def is_verbose(threshold=1): def is_quiet(threshold=-1): return PIPENV_VERBOSITY <= threshold + + +PIPENV_SPINNER_FAIL_TEXT = fs_str(to_text(u"✘ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") + +PIPENV_SPINNER_OK_TEXT = fs_str(to_text(u"✔ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") diff --git a/pipenv/project.py b/pipenv/project.py index c2dc966816..280a6f8b06 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -10,6 +10,7 @@ import contoml from first import first from cached_property import cached_property +import operator import pipfile import pipfile.api import six @@ -144,7 +145,7 @@ def __init__(self, which=None, python_version=None, chdir=True): self._lockfile_newlines = DEFAULT_NEWLINES self._requirements_location = None self._original_dir = os.path.abspath(os.curdir) - self.which = which + self._which = which self.python_version = python_version # Hack to skip this during pipenv run, or -r. if ("run" not in sys.argv) and chdir: @@ -678,6 +679,54 @@ def create_pipfile(self, python=None): data[u"requires"] = {"python_version": version[: len("2.7")]} self.write_toml(data, "Pipfile") + def get_or_create_lockfile(self): + from requirementslib.models.lockfile import Lockfile as Req_Lockfile + lockfile = None + try: + lockfile = Req_Lockfile.load(self.lockfile_location) + except OSError: + lockfile = Req_Lockfile(self.lockfile_content) + return lockfile + else: + if lockfile._lockfile is not None: + return lockfile + if self.lockfile_exists and self.lockfile_content: + from .vendor.plette.lockfiles import Lockfile + lockfile_dict = self.lockfile_content.copy() + sources = lockfile_dict["_meta"].get("sources", []) + if not sources: + sources = self.pipfile_sources + elif not isinstance(sources, list): + sources = [sources,] + lockfile_dict["_meta"]["sources"] = [ + { + "name": s["name"], + "url": s["url"], + "verify_ssl": ( + s["verify_ssl"] if isinstance(s["verify_ssl"], bool) else ( + True if s["verify_ssl"].lower() == "true" else False + ) + ) + } for s in sources + ] + _created_lockfile = Lockfile(lockfile_dict) + lockfile._lockfile = lockfile.projectfile.model = _created_lockfile + return lockfile + elif self.pipfile_exists: + from .vendor.plette.lockfiles import Lockfile, PIPFILE_SPEC_CURRENT + lockfile_dict = { + "_meta": { + "hash": {"sha256": self.calculate_pipfile_hash()}, + "pipfile-spec": PIPFILE_SPEC_CURRENT, + "sources": self.pipfile_sources, + "requires": self.parsed_pipfile.get("requires", {}) + }, + "default": self._lockfile["default"].copy(), + "develop": self._lockfile["develop"].copy() + } + lockfile._lockfile = Lockfile(lockfile_dict) + return lockfile + def write_toml(self, data, path=None): """Writes the given data structure out as TOML.""" if path is None: @@ -939,6 +988,8 @@ def env_paths(self): location = self.virtualenv_location if self.virtualenv_location else sys.prefix prefix = vistir.compat.Path(location).as_posix() scheme = sysconfig._get_default_scheme() + if not scheme: + scheme = "posix_prefix" if not sys.platform == "win32" else "nt" config = { "base": prefix, "installed_base": prefix, @@ -953,3 +1004,26 @@ def env_paths(self): if "prefix" not in paths: paths["prefix"] = prefix return paths + + @cached_property + def finders(self): + from .vendor.pythonfinder import Finder + finders = [ + Finder(path=self.env_paths["scripts"], global_search=gs, system=False) + for gs in (False, True) + ] + return finders + + @property + def finder(self): + return next(iter(self.finders), None) + + def which(self, search, as_path=True): + find = operator.methodcaller("which", search) + result = next(iter(filter(None, (find(finder) for finder in self.finders))), None) + if not result: + result = self._which(search) + else: + if as_path: + result = str(result.path) + return result diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 8c282b71f8..2854a93ab4 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -7,53 +7,52 @@ def _patch_path(): + import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + pipenv_site_dir = os.path.dirname(pipenv_libdir) + site.addsitedir(pipenv_site_dir) for _dir in ("vendor", "patched"): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) - site_packages_dir = os.path.dirname(pipenv_libdir) - if site_packages_dir not in sys.path: - sys.path.append(site_packages_dir) + + +def get_parser(): + from argparse import ArgumentParser + parser = ArgumentParser("pipenvresolver") + parser.add_argument("--pre", action="store_true", default=False) + parser.add_argument("--clear", action="store_true", default=False) + parser.add_argument("--verbose", "-v", action="count", default=False) + parser.add_argument("--debug", action="store_true", default=False) + parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", + default=os.environ.get("PIPENV_REQ_DIR")) + parser.add_argument("packages", nargs="*") + return parser def which(*args, **kwargs): return sys.executable -def main(): - do_pre = "--pre" in " ".join(sys.argv) - do_clear = "--clear" in " ".join(sys.argv) - is_verbose = "--verbose" in " ".join(sys.argv) - is_debug = "--debug" in " ".join(sys.argv) - system = "--system" in " ".join(sys.argv) - new_sys_argv = [] - for v in sys.argv: - if v.startswith("--"): - continue +def handle_parsed_args(parsed): + if parsed.debug: + parsed.verbose = max(parsed.verbose, 2) + if parsed.verbose > 1: + logging.getLogger("notpip").setLevel(logging.DEBUG) + elif parsed.verbose > 0: + logging.getLogger("notpip").setLevel(logging.INFO) + if "PIPENV_PACKAGES" in os.environ: + parsed.packages += os.environ["PIPENV_PACKAGES"].strip().split("\n") + return parsed - else: - new_sys_argv.append(v) - sys.argv = new_sys_argv +def main(pre, clear, verbose, system, requirements_dir, packages): os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) os.environ["PIP_PYTHON_PATH"] = sys.executable - verbosity = int(os.environ.get("PIPENV_VERBOSITY", 0)) - if is_debug: - verbosity = max(verbosity, 2) - elif is_verbose: - verbosity = max(verbosity, 1) - if verbosity > 1: # Shit's getting real at this point. - logging.getLogger("notpip").setLevel(logging.DEBUG) - elif verbosity > 0: - logging.getLogger("notpip").setLevel(logging.INFO) + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + warnings.filterwarnings("ignore", category=ResourceWarning) - if "PIPENV_PACKAGES" in os.environ: - packages = os.environ["PIPENV_PACKAGES"].strip().split("\n") - else: - packages = sys.argv[1:] - for i, package in enumerate(packages): - if package.startswith("--"): - del packages[i] from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources pypi_mirror_source = ( @@ -62,7 +61,7 @@ def main(): else None ) - def resolve(packages, pre, project, sources, clear, system): + def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): return resolve_deps( packages, which, @@ -71,6 +70,7 @@ def resolve(packages, pre, project, sources, clear, system): sources=sources, clear=clear, allow_global=system, + req_dir=requirements_dir ) from pipenv.core import project @@ -82,19 +82,31 @@ def resolve(packages, pre, project, sources, clear, system): ) results = resolve( packages, - pre=do_pre, + pre=pre, project=project, sources=sources, - clear=do_clear, + clear=clear, system=system, + requirements_dir=requirements_dir, ) print("RESULTS:") if results: - print(json.dumps(results)) + import traceback + if isinstance(results, (Exception, traceback.types.TracebackType)): + sys.stderr.write(traceback.print_tb(results)) + sys.stderr.write(sys.exc_value()) + else: + print(json.dumps(results)) else: print(json.dumps([])) if __name__ == "__main__": + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" _patch_path() - main() + parser = get_parser() + parsed, remaining = parser.parse_known_intermixed_args() + sys.argv = remaining + parsed = handle_parsed_args(parsed) + main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir, + parsed.packages) diff --git a/pipenv/utils.py b/pipenv/utils.py index b965e46d3b..d8f2432bdb 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -16,6 +16,11 @@ from first import first from vistir.misc import fs_str +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +from six.moves import Mapping + +from vistir.compat import ResourceWarning + try: from weakref import finalize except ImportError: @@ -38,14 +43,8 @@ def detach(self): from . import environments from .pep508checker import lookup -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) from six.moves.urllib.parse import urlparse -from six.moves import Mapping - -if six.PY2: - - class ResourceWarning(Warning): - pass +from urllib3 import util as urllib3_util specifiers = [k for k in lookup.keys()] @@ -127,20 +126,14 @@ def parse_python_version(output): def python_version(path_to_python): - import delegator + from .vendor.pythonfinder.utils import get_python_version if not path_to_python: return None try: - c = delegator.run([path_to_python, "--version"], block=False) + version = get_python_version(path_to_python) except Exception: return None - c.block() - version = parse_python_version(c.out.strip() or c.err.strip()) - try: - version = u"{major}.{minor}.{micro}".format(**version) - except TypeError: - return None return version @@ -194,7 +187,7 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(sources[0]["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(sources[0]["url"]).host] ) # Add additional sources as extra indexes. if len(sources) > 1: @@ -203,7 +196,7 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(source["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(source["url"]).host] ) return pip_args @@ -228,7 +221,7 @@ def actually_resolve_deps( from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory, NamedTemporaryFile + from .vendor.vistir.path import create_tracked_tempdir, create_tracked_tempfile class PipCommand(basecommand.Command): """Needed for pip-tools.""" @@ -236,10 +229,8 @@ class PipCommand(basecommand.Command): name = "PipCommand" constraints = [] - cleanup_req_dir = False if not req_dir: - req_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-") - cleanup_req_dir = True + req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") for dep in deps: if not dep: continue @@ -267,26 +258,26 @@ class PipCommand(basecommand.Command): if sources: pip_args = prepare_pip_source_args(sources, pip_args) if environments.is_verbose(): - print("Using pip: {0}".format(" ".join(pip_args))) - with NamedTemporaryFile( + click_echo(crayons.blue("Using pip: {0}".format(" ".join(pip_args)))) + constraints_file = create_tracked_tempfile( mode="w", prefix="pipenv-", suffix="-constraints.txt", - dir=req_dir.name, + dir=req_dir, delete=False, - ) as f: - if sources: - requirementstxt_sources = " ".join(pip_args) if pip_args else "" - requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") - f.write(u"{0}\n".format(requirementstxt_sources)) - f.write(u"\n".join([_constraint for _constraint in constraints])) - constraints_file = f.name + ) + if sources: + requirementstxt_sources = " ".join(pip_args) if pip_args else "" + requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") + constraints_file.write(u"{0}\n".format(requirementstxt_sources)) + constraints_file.write(u"\n".join([_constraint for _constraint in constraints])) + constraints_file.close() pip_options, _ = pip_command.parser.parse_args(pip_args) pip_options.cache_dir = environments.PIPENV_CACHE_DIR session = pip_command._build_session(pip_options) pypi = PyPIRepository(pip_options=pip_options, use_json=False, session=session) constraints = parse_requirements( - constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options + constraints_file.name, finder=pypi.finder, session=pypi.session, options=pip_options ) constraints = [c for c in constraints] if environments.is_verbose(): @@ -326,11 +317,7 @@ class PipCommand(basecommand.Command): "Please check your version specifier and version number. See PEP440 for more information." ) ) - if cleanup_req_dir: - req_dir.cleanup() raise RuntimeError - if cleanup_req_dir: - req_dir.cleanup() return (resolved_tree, hashes, markers_lookup, resolver) @@ -343,43 +330,72 @@ def venv_resolve_deps( allow_global=False, pypi_mirror=None, ): - from .vendor.vistir.misc import fs_str + from .vendor.vistir.misc import fs_str, run + from .vendor.vistir.compat import Path + from .vendor.vistir.path import create_tracked_tempdir + from .cmdparse import Script + from .core import spinner + from .vendor.pexpect.exceptions import EOF from .vendor import delegator from . import resolver import json if not deps: return [] - resolver = escape_grouped_arguments(resolver.__file__.rstrip("co")) - cmd = "{0} {1} {2} {3} {4}".format( - escape_grouped_arguments(which("python", allow_global=allow_global)), - resolver, - "--pre" if pre else "", - "--clear" if clear else "", - "--system" if allow_global else "", - ) + + req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") + + cmd = [ + which("python", allow_global=allow_global), + Path(resolver.__file__.rstrip("co")).as_posix() + ] + if pre: + cmd.append("--pre") + if clear: + cmd.append("--clear") + if allow_global: + cmd.append("--system") with temp_environ(): os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()} os.environ["PIPENV_PACKAGES"] = str("\n".join(deps)) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) - c = delegator.run(cmd, block=True) - try: - assert c.return_code == 0 - except AssertionError: - if environments.is_verbose(): - click_echo(c.out, err=True) - click_echo(c.err, err=True) - else: - click_echo(c.err[(int(len(c.err) / 2) - 1):], err=True) - sys.exit(c.return_code) + os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) + os.environ["PIP_NO_INPUT"] = fs_str("1") + + out = "" + EOF.__module__ = "pexpect.exceptions" + with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: + c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + _out = u"" + while True: + result = c.expect(u"\n", timeout=-1) + if result is EOF or result is None: + break + _out = c.out + out += _out + sp.text = fs_str("Locking... {0}".format(_out[:100])) + if environments.is_verbose(): + sp.write_err(_out.rstrip()) + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) + else: + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) if environments.is_verbose(): click_echo(c.out.split("RESULTS:")[0], err=True) try: return json.loads(c.out.split("RESULTS:")[1].strip()) except IndexError: + click_echo(c.out.strip()) + click_echo(c.err.strip(), err=True) raise RuntimeError("There was a problem with locking.") @@ -392,13 +408,13 @@ def resolve_deps( clear=False, pre=False, allow_global=False, + req_dir=None ): """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ from .patched.notpip._vendor.requests.exceptions import ConnectionError from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory index_lookup = {} markers_lookup = {} @@ -408,7 +424,10 @@ def resolve_deps( if not deps: return results # First (proper) attempt: - req_dir = TemporaryDirectory(prefix="pipenv-", suffix="-requirements") + req_dir = req_dir if req_dir else os.environ.get("req_dir", None) + if not req_dir: + from .vendor.vistir.path import create_tracked_tempdir + req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_version=python, python_path=python_path): try: resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( @@ -444,7 +463,6 @@ def resolve_deps( req_dir=req_dir, ) except RuntimeError: - req_dir.cleanup() sys.exit(1) for result in resolved_tree: if not result.editable: @@ -503,7 +521,6 @@ def resolve_deps( entry.update({"markers": markers_lookup.get(result.name)}) entry = translate_markers(entry) results.append(entry) - req_dir.cleanup() return results @@ -526,7 +543,6 @@ def is_pinned(val): def convert_deps_to_pip(deps, project=None, r=True, include_index=True): """"Converts a Pipfile-formatted dependency to a pip-formatted one.""" - from ._compat import NamedTemporaryFile from .vendor.requirementslib.models.requirements import Requirement dependencies = [] @@ -541,7 +557,8 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=True): return dependencies # Write requirements.txt to tmp directory. - f = NamedTemporaryFile(suffix="-requirements.txt", delete=False) + from .vendor.vistir.path import create_tracked_tempfile + f = create_tracked_tempfile(suffix="-requirements.txt", delete=False) f.write("\n".join(dependencies).encode("utf-8")) f.close() return f.name @@ -1054,54 +1071,6 @@ def escape_cmd(cmd): return cmd -@contextmanager -def atomic_open_for_write(target, binary=False, newline=None, encoding=None): - """Atomically open `target` for writing. - - This is based on Lektor's `atomic_open()` utility, but simplified a lot - to handle only writing, and skip many multi-process/thread edge cases - handled by Werkzeug. - - How this works: - - * Create a temp file (in the same directory of the actual target), and - yield for surrounding code to write to it. - * If some thing goes wrong, try to remove the temp file. The actual target - is not touched whatsoever. - * If everything goes well, close the temp file, and replace the actual - target with this new file. - """ - from ._compat import NamedTemporaryFile - - mode = "w+b" if binary else "w" - f = NamedTemporaryFile( - dir=os.path.dirname(target), - prefix=".__atomic-write", - mode=mode, - encoding=encoding, - newline=newline, - delete=False, - ) - # set permissions to 0644 - os.chmod(f.name, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - try: - yield f - except BaseException: - f.close() - try: - os.remove(f.name) - except OSError: - pass - raise - else: - f.close() - try: - os.remove(target) # This is needed on Windows. - except OSError: - pass - os.rename(f.name, target) # No os.replace() on Python 2. - - def safe_expandvars(value): """Call os.path.expandvars if value is a string, otherwise do nothing. """ @@ -1127,8 +1096,8 @@ def get_vcs_deps( dev=False, pypi_mirror=None, ): - from ._compat import TemporaryDirectory, Path - import atexit + from .vendor.vistir.compat import Path + from .vendor.vistir.path import create_tracked_tempdir from .vendor.requirementslib.models.requirements import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" @@ -1144,8 +1113,7 @@ def get_vcs_deps( ) src_dir.mkdir(mode=0o775, exist_ok=True) else: - src_dir = TemporaryDirectory(prefix="pipenv-lock-dir") - atexit.register(src_dir.cleanup) + src_dir = create_tracked_tempdir(prefix="pipenv-lock-dir") for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name @@ -1280,7 +1248,6 @@ def is_virtual_environment(path): @contextmanager def locked_repository(requirement): from .vendor.vistir.path import create_tracked_tempdir - from .vendor.vistir.misc import fs_str src_dir = create_tracked_tempdir(prefix="pipenv-src") if not requirement.is_vcs: return diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index 3ffb2e31ca..56d1245815 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -7,6 +7,8 @@ import errno from pexpect.popen_spawn import PopenSpawn +import pexpect +pexpect.EOF.__module__ = "pexpect.exceptions" # Include `unicode` in STR_TYPES for Python 2.X try: @@ -110,7 +112,7 @@ def _pexpect_out(self): if self.subprocess.before: result += self.subprocess.before - if self.subprocess.after: + if self.subprocess.after and self.subprocess.after is not pexpect.EOF: result += self.subprocess.after result += self.subprocess.read() @@ -206,7 +208,10 @@ def expect(self, pattern, timeout=-1): if self.blocking: raise RuntimeError("expect can only be used on non-blocking commands.") - self.subprocess.expect(pattern=pattern, timeout=timeout) + try: + self.subprocess.expect(pattern=pattern, timeout=timeout) + except pexpect.EOF: + pass def send(self, s, end=os.linesep, signal=False): """Sends the given string or signal to std_in.""" @@ -249,8 +254,11 @@ def block(self): self.subprocess.wait() else: self.subprocess.sendeof() - self.subprocess.wait() - self.subprocess.proc.stdout.close() + try: + self.subprocess.wait() + finally: + if self.subprocess.proc.stdout: + self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next @@ -272,7 +280,6 @@ def pipe(self, command, timeout=None, cwd=None): c.run(block=False, cwd=cwd) if data: c.send(data) - c.subprocess.sendeof() c.block() return c diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 9ac6031ca4..91e9cabb10 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.3' +__version__ = '1.1.3.post1' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index d54393b6b3..f39299a3b6 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -341,7 +341,6 @@ def find_python_version( self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) else: self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver] - print(ver) return ver @classmethod diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index e0bc6746a8..7b4b6376d6 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,6 +1,9 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.10' +__version__ = '1.2.0' +from .models.requirements import Requirement +from .models.lockfile import Lockfile +from .models.pipfile import Pipfile -from .exceptions import RequirementError -from .models import Requirement, Lockfile, Pipfile + +__all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 8257862476..de8bf8ef94 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -9,6 +9,8 @@ class FileExistsError(OSError): def __init__(self, *args, **kwargs): self.errno = errno.EEXIST super(FileExistsError, self).__init__(*args, **kwargs) +else: + from six.moves.builtins import FileExistsError class RequirementError(Exception): diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 16fc4ba8af..71701090fb 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -5,21 +5,19 @@ import hashlib import json import os -import six import sys import requests -import pip_shims import vistir from appdirs import user_cache_dir +from pip_shims.shims import FAVORITE_HASH, SafeFileCache from packaging.requirements import Requirement from .utils import as_tuple, key_from_req, lookup_table, get_pinned_version - -if six.PY2: - from ..exceptions import FileExistsError +from ..exceptions import FileExistsError +from ..utils import VCS_SUPPORT CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) @@ -189,7 +187,7 @@ def _reverse_dependencies(self, cache_keys): for dep_name in self.cache[name][version_and_extras]) -class HashCache(pip_shims.SafeFileCache): +class HashCache(SafeFileCache): """Caches hashes of PyPI artifacts so we do not need to re-download them. Hashes are only cached when the URL appears to contain a hash in it and the @@ -206,7 +204,7 @@ def __init__(self, *args, **kwargs): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs = pip_shims.VcsSupport() + vcs = VCS_SUPPORT orig_scheme = location.scheme new_location = copy.deepcopy(location) if orig_scheme in vcs.all_schemes: @@ -223,11 +221,11 @@ def get_hash(self, location): return hash_value.decode('utf8') def _get_file_hash(self, location): - h = hashlib.new(pip_shims.FAVORITE_HASH) + h = hashlib.new(FAVORITE_HASH) with vistir.contextmanagers.open_file(location, self.session) as fp: for chunk in iter(lambda: fp.read(8096), b""): h.update(chunk) - return ":".join([pip_shims.FAVORITE_HASH, h.hexdigest()]) + return ":".join([FAVORITE_HASH, h.hexdigest()]) class _JSONCache(object): diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 1997fc1f55..3e48281347 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -5,6 +5,7 @@ import os import attr +import itertools import plette.lockfiles import six @@ -50,6 +51,31 @@ def _get_projectfile(self): def _get_lockfile(self): return self.projectfile.lockfile + @property + def lockfile(self): + return self._lockfile + + @property + def section_keys(self): + return ["default", "develop"] + + @property + def extended_keys(self): + return [k for k in itertools.product(self.section_keys, ["", "vcs", "editable"])] + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_lockfile = k in self.extended_keys or self.lockfile.__contains__(k) + if check_lockfile: + return True + return super(Lockfile, self).__contains__(k) + + def __setitem__(self, k, v): + lockfile = self._lockfile + lockfile.__setitem__(k, v) + def __getitem__(self, k, *args, **kwargs): retval = None lockfile = self._lockfile diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 0d1c04c883..fe7743c2ca 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -75,11 +75,19 @@ def pipfile(self): def get_deps(self, dev=False, only=True): deps = {} if dev: - deps.update(self.pipfile["dev-packages"]._data) + deps.update(self.pipfile._data["dev-packages"]) if only: return deps - deps = merge_items([deps, self.pipfile["packages"]._data]) - return deps + return merge_items([deps, self.pipfile._data["packages"]]) + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) + if check_pipfile: + return True + return super(Pipfile, self).__contains__(k) def __getitem__(self, k, *args, **kwargs): retval = None diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index db004869c6..3a029cbcda 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -16,10 +16,7 @@ from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from pip_shims.shims import ( - InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url, - url_to_path -) +from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path @@ -32,21 +29,16 @@ from ..exceptions import RequirementError from ..utils import VCS_LIST, is_installable_file, is_vcs, ensure_setup_py from .baserequirement import BaseRequirement -from .dependencies import ( - AbstractDependency, find_all_matches, get_abstract_dependencies, - get_dependencies, get_finder -) from .markers import PipenvMarkers from .utils import ( HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, filter_none, format_requirement, get_version, init_requirement, - is_pinned_requirement, make_install_requirement, optional_instance_of, - parse_extras, specs_to_string, split_markers_from_line, + is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, + specs_to_string, split_markers_from_line, ireq_from_editable, ireq_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs, normalize_name, + validate_specifiers, validate_vcs, normalize_name, create_link, Requirement as PkgResourcesRequirement ) -from .vcs import VCSRepository @attr.s @@ -128,7 +120,7 @@ class FileRequirement(BaseRequirement): editable = attr.ib(default=False, type=bool) extras = attr.ib(default=attr.Factory(list), type=list) uri = attr.ib(type=six.string_types) - link = attr.ib(type=Link) + link = attr.ib() name = attr.ib(type=six.string_types) req = attr.ib(type=PkgResourcesRequirement) _has_hashed_name = False @@ -166,6 +158,7 @@ def get_link_from_line(cls, line): See `https://bugs.python.org/issue23505#msg277350`. """ + # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. fixed_line = add_ssh_scheme_to_git_uri(line) @@ -176,7 +169,7 @@ def get_link_from_line(cls, line): p = Path(fixed_line).absolute() path = p.as_posix() uri = p.as_uri() - link = Link(uri) + link = create_link(uri) try: relpath = get_converted_relative_path(path) except ValueError: @@ -225,7 +218,7 @@ def get_link_from_line(cls, line): uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. - link = Link( + link = create_link( urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) ) @@ -246,6 +239,7 @@ def get_name(self): if self.link and self.link.egg_fragment: return self.link.egg_fragment elif self.link and self.link.is_wheel: + from pip_shims import Wheel return Wheel(self.link.filename).name if ( self._uri_scheme != "uri" @@ -263,7 +257,7 @@ def get_name(self): except (FileNotFoundError, IOError) as e: dist = None except Exception as e: - from pip_shims.shims import InstallRequirement, make_abstract_dist + from pip_shims.shims import make_abstract_dist try: if not isinstance(Path, self.path): @@ -271,9 +265,9 @@ def get_name(self): else: _path = self.path if self.editable: - _ireq = InstallRequirement.from_editable(_path.as_uri()) + _ireq = ireq_from_editable(_path.as_uri()) else: - _ireq = InstallRequirement.from_line(_path.as_posix()) + _ireq = ireq_from_line(_path.as_posix()) dist = make_abstract_dist(_ireq).get_dist() name = dist.project_name except (TypeError, ValueError, AttributeError) as e: @@ -286,7 +280,7 @@ def get_name(self): self._has_hashed_name = True name = hashed_name if self.link and not self._has_hashed_name: - self.link = Link("{0}#egg={1}".format(self.link.url, name)) + self.link = create_link("{0}#egg={1}".format(self.link.url, name)) return name @link.default @@ -294,7 +288,7 @@ def get_link(self): target = "{0}".format(self.uri) if hasattr(self, "name"): target = "{0}#egg={1}".format(target, self.name) - link = Link(target) + link = create_link(target) return link @req.default @@ -359,6 +353,7 @@ def from_line(cls, line): "uri_scheme": prefer, } if link and link.is_wheel: + from pip_shims import Wheel arg_dict["name"] = Wheel(link.filename).name elif link.egg_fragment: arg_dict["name"] = link.egg_fragment @@ -398,7 +393,7 @@ def from_pipfile(cls, name, pipfile): if not uri: uri = path_to_url(path) - link = Link(uri) + link = create_link(uri) arg_dict = { "name": name, @@ -588,6 +583,7 @@ def get_checkout_dir(self, src_dir=None): return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) def get_vcs_repo(self, src_dir=None): + from .vcs import VCSRepository checkout_dir = self.get_checkout_dir(src_dir=src_dir) url = "{0}#egg={1}".format(self.vcs_uri, self.name) vcsrepo = VCSRepository( @@ -825,6 +821,7 @@ def copy(self): @classmethod def from_line(cls, line): + from pip_shims import InstallRequirement if isinstance(line, InstallRequirement): line = format_requirement(line) hashes = None @@ -1074,9 +1071,9 @@ def as_ireq(self): if ireq_line.startswith("-e "): ireq_line = ireq_line[len("-e "):] with ensure_setup_py(self.req.path): - ireq = InstallRequirement.from_editable(ireq_line) + ireq = ireq_from_editable(ireq_line) else: - ireq = InstallRequirement.from_line(ireq_line) + ireq = ireq_from_line(ireq_line) if not getattr(ireq, "req", None): ireq.req = self.req.req else: @@ -1103,6 +1100,8 @@ def get_dependencies(self, sources=None): :return: A set of requirement strings of the dependencies of this requirement. :rtype: set(str) """ + + from .dependencies import get_dependencies if not sources: sources = [{ 'name': 'pypi', @@ -1122,6 +1121,7 @@ def get_abstract_dependencies(self, sources=None): :rtype: list[ :class:`~requirementslib.models.dependency.AbstractDependency` ] """ + from .dependencies import AbstractDependency, get_dependencies, get_abstract_dependencies if not self.abstract_dep: parent = getattr(self, 'parent', None) self.abstract_dep = AbstractDependency.from_requirement(self, parent=parent) @@ -1144,6 +1144,8 @@ def find_all_matches(self, sources=None, finder=None): :return: A list of Installation Candidates :rtype: list[ :class:`~pip._internal.index.InstallationCandidate` ] """ + + from .dependencies import get_finder, find_all_matches if not finder: finder = get_finder(sources=sources) return find_all_matches(finder, self.as_ireq()) diff --git a/pipenv/vendor/requirementslib/models/resolvers.py b/pipenv/vendor/requirementslib/models/resolvers.py index da6d0dda5f..1a2393903e 100644 --- a/pipenv/vendor/requirementslib/models/resolvers.py +++ b/pipenv/vendor/requirementslib/models/resolvers.py @@ -4,11 +4,10 @@ import attr import six -from pip_shims.shims import VcsSupport, Wheel +from pip_shims.shims import Wheel -from ..utils import log +from ..utils import log, VCS_SUPPORT from .cache import HashCache -from .dependencies import AbstractDependency, find_all_matches, get_finder from .utils import format_requirement, is_pinned_requirement, version_from_ireq @@ -41,6 +40,7 @@ class DependencyResolver(object): @classmethod def create(cls, finder=None, allow_prereleases=False, get_all_hashes=True): if not finder: + from .dependencies import get_finder finder_args = [] if allow_prereleases: finder_args.append('--pre') @@ -140,6 +140,7 @@ def resolve(self, root_nodes, max_rounds=20): # Coerce input into AbstractDependency instances. # We accept str, Requirement, and AbstractDependency as input. + from .dependencies import AbstractDependency for dep in root_nodes: if isinstance(dep, six.string_types): dep = AbstractDependency.from_string(dep) @@ -183,6 +184,7 @@ def get_hashes(self): def get_hashes_for_one(self, ireq): if not self.finder: + from .dependencies import get_finder finder_args = [] if self.allow_prereleases: finder_args.append('--pre') @@ -191,7 +193,7 @@ def get_hashes_for_one(self, ireq): if ireq.editable: return set() - vcs = VcsSupport() + vcs = VCS_SUPPORT if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: return set() @@ -201,6 +203,7 @@ def get_hashes_for_one(self, ireq): matching_candidates = set() with self.allow_all_wheels(): + from .dependencies import find_all_matches matching_candidates = ( find_all_matches(self.finder, ireq, pre=self.allow_prereleases) ) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 7350d0ac08..3103580e55 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -19,7 +19,7 @@ from pkg_resources import Requirement from vistir.misc import dedup -from pip_shims.shims import InstallRequirement, Link + from ..utils import SCHEME_LIST, VCS_LIST, is_star @@ -37,6 +37,21 @@ def optional_instance_of(cls): return validators.optional(validators.instance_of(cls)) +def create_link(link): + from pip_shims import Link + return Link(link) + + +def ireq_from_line(ireq): + from pip_shims import InstallRequirement + return InstallRequirement.from_line(ireq) + + +def ireq_from_editable(ireq): + from pip_shims import InstallRequirement + return InstallRequirement.from_editable(ireq) + + def init_requirement(name): req = Requirement.parse(name) req.vcs = None @@ -92,7 +107,7 @@ def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None uri = "{0}{1}".format(uri, extras) if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) - return Link(uri) + return create_link(uri) def get_version(pipfile_entry): @@ -443,11 +458,11 @@ def make_install_requirement(name, version, extras, markers, constraint=False): extras_string = "[{}]".format(",".join(sorted(extras))) if not markers: - return InstallRequirement.from_line( + return ireq_from_line( str('{}{}=={}'.format(name, extras_string, version)), constraint=constraint) else: - return InstallRequirement.from_line( + return ireq_from_line( str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), constraint=constraint) diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index abc8983167..085f32415c 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -5,7 +5,6 @@ import logging import os -import boltons.iterutils import six import tomlkit @@ -30,8 +29,10 @@ VCS_SCHEMES = [] SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") +VCS_SUPPORT = VcsSupport() + if not VCS_SCHEMES: - VCS_SCHEMES = VcsSupport().all_schemes + VCS_SCHEMES = VCS_SUPPORT.all_schemes def setup_logger(): @@ -197,6 +198,127 @@ def ensure_setup_py(base_dir): setup_py.unlink() + +_UNSET = object() +_REMAP_EXIT = object() + + +# The following functionality is either borrowed or modified from the itertools module +# in the boltons library by Mahmoud Hashemi and distributed under the BSD license +# the text of which is included below: + +# (original text from https://github.com/mahmoud/boltons/blob/master/LICENSE) +# Copyright (c) 2013, Mahmoud Hashemi +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * The names of the contributors may not be used to endorse or +# promote products derived from this software without specific +# prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class PathAccessError(KeyError, IndexError, TypeError): + """An amalgamation of KeyError, IndexError, and TypeError, + representing what can occur when looking up a path in a nested + object. + """ + def __init__(self, exc, seg, path): + self.exc = exc + self.seg = seg + self.path = path + + def __repr__(self): + cn = self.__class__.__name__ + return '%s(%r, %r, %r)' % (cn, self.exc, self.seg, self.path) + + def __str__(self): + return ('could not access %r from path %r, got error: %r' + % (self.seg, self.path, self.exc)) + + +def get_path(root, path, default=_UNSET): + """Retrieve a value from a nested object via a tuple representing the + lookup path. + >>> root = {'a': {'b': {'c': [[1], [2], [3]]}}} + >>> get_path(root, ('a', 'b', 'c', 2, 0)) + 3 + The path format is intentionally consistent with that of + :func:`remap`. + One of get_path's chief aims is improved error messaging. EAFP is + great, but the error messages are not. + For instance, ``root['a']['b']['c'][2][1]`` gives back + ``IndexError: list index out of range`` + What went out of range where? get_path currently raises + ``PathAccessError: could not access 2 from path ('a', 'b', 'c', 2, + 1), got error: IndexError('list index out of range',)``, a + subclass of IndexError and KeyError. + You can also pass a default that covers the entire operation, + should the lookup fail at any level. + Args: + root: The target nesting of dictionaries, lists, or other + objects supporting ``__getitem__``. + path (tuple): A list of strings and integers to be successively + looked up within *root*. + default: The value to be returned should any + ``PathAccessError`` exceptions be raised. + """ + if isinstance(path, six.string_types): + path = path.split('.') + cur = root + try: + for seg in path: + try: + cur = cur[seg] + except (KeyError, IndexError) as exc: + raise PathAccessError(exc, seg, path) + except TypeError as exc: + # either string index in a list, or a parent that + # doesn't support indexing + try: + seg = int(seg) + cur = cur[seg] + except (ValueError, KeyError, IndexError, TypeError): + if not getattr(cur, "__iter__", None): + exc = TypeError('%r object is not indexable' + % type(cur).__name__) + raise PathAccessError(exc, seg, path) + except PathAccessError: + if default is _UNSET: + raise + return default + return cur + + +def default_visit(path, key, value): + return key, value + + +_orig_default_visit = default_visit + + # Modified from https://github.com/mahmoud/boltons/blob/master/boltons/iterutils.py def dict_path_enter(path, key, value): if isinstance(value, six.string_types): @@ -249,6 +371,157 @@ def dict_path_exit(path, key, old_parent, new_parent, new_items): return ret +def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, + **kwargs): + """The remap ("recursive map") function is used to traverse and + transform nested structures. Lists, tuples, sets, and dictionaries + are just a few of the data structures nested into heterogenous + tree-like structures that are so common in programming. + Unfortunately, Python's built-in ways to manipulate collections + are almost all flat. List comprehensions may be fast and succinct, + but they do not recurse, making it tedious to apply quick changes + or complex transforms to real-world data. + remap goes where list comprehensions cannot. + Here's an example of removing all Nones from some data: + >>> from pprint import pprint + >>> reviews = {'Star Trek': {'TNG': 10, 'DS9': 8.5, 'ENT': None}, + ... 'Babylon 5': 6, 'Dr. Who': None} + >>> pprint(remap(reviews, lambda p, k, v: v is not None)) + {'Babylon 5': 6, 'Star Trek': {'DS9': 8.5, 'TNG': 10}} + Notice how both Nones have been removed despite the nesting in the + dictionary. Not bad for a one-liner, and that's just the beginning. + See `this remap cookbook`_ for more delicious recipes. + .. _this remap cookbook: http://sedimental.org/remap.html + remap takes four main arguments: the object to traverse and three + optional callables which determine how the remapped object will be + created. + Args: + root: The target object to traverse. By default, remap + supports iterables like :class:`list`, :class:`tuple`, + :class:`dict`, and :class:`set`, but any object traversable by + *enter* will work. + visit (callable): This function is called on every item in + *root*. It must accept three positional arguments, *path*, + *key*, and *value*. *path* is simply a tuple of parents' + keys. *visit* should return the new key-value pair. It may + also return ``True`` as shorthand to keep the old item + unmodified, or ``False`` to drop the item from the new + structure. *visit* is called after *enter*, on the new parent. + The *visit* function is called for every item in root, + including duplicate items. For traversable values, it is + called on the new parent object, after all its children + have been visited. The default visit behavior simply + returns the key-value pair unmodified. + enter (callable): This function controls which items in *root* + are traversed. It accepts the same arguments as *visit*: the + path, the key, and the value of the current item. It returns a + pair of the blank new parent, and an iterator over the items + which should be visited. If ``False`` is returned instead of + an iterator, the value will not be traversed. + The *enter* function is only called once per unique value. The + default enter behavior support mappings, sequences, and + sets. Strings and all other iterables will not be traversed. + exit (callable): This function determines how to handle items + once they have been visited. It gets the same three + arguments as the other functions -- *path*, *key*, *value* + -- plus two more: the blank new parent object returned + from *enter*, and a list of the new items, as remapped by + *visit*. + Like *enter*, the *exit* function is only called once per + unique value. The default exit behavior is to simply add + all new items to the new parent, e.g., using + :meth:`list.extend` and :meth:`dict.update` to add to the + new parent. Immutable objects, such as a :class:`tuple` or + :class:`namedtuple`, must be recreated from scratch, but + use the same type as the new parent passed back from the + *enter* function. + reraise_visit (bool): A pragmatic convenience for the *visit* + callable. When set to ``False``, remap ignores any errors + raised by the *visit* callback. Items causing exceptions + are kept. See examples for more details. + remap is designed to cover the majority of cases with just the + *visit* callable. While passing in multiple callables is very + empowering, remap is designed so very few cases should require + passing more than one function. + When passing *enter* and *exit*, it's common and easiest to build + on the default behavior. Simply add ``from boltons.iterutils import + default_enter`` (or ``default_exit``), and have your enter/exit + function call the default behavior before or after your custom + logic. See `this example`_. + Duplicate and self-referential objects (aka reference loops) are + automatically handled internally, `as shown here`_. + .. _this example: http://sedimental.org/remap.html#sort_all_lists + .. _as shown here: http://sedimental.org/remap.html#corner_cases + """ + # TODO: improve argument formatting in sphinx doc + # TODO: enter() return (False, items) to continue traverse but cancel copy? + if not callable(visit): + raise TypeError('visit expected callable, not: %r' % visit) + if not callable(enter): + raise TypeError('enter expected callable, not: %r' % enter) + if not callable(exit): + raise TypeError('exit expected callable, not: %r' % exit) + reraise_visit = kwargs.pop('reraise_visit', True) + if kwargs: + raise TypeError('unexpected keyword arguments: %r' % kwargs.keys()) + + path, registry, stack = (), {}, [(None, root)] + new_items_stack = [] + while stack: + key, value = stack.pop() + id_value = id(value) + if key is _REMAP_EXIT: + key, new_parent, old_parent = value + id_value = id(old_parent) + path, new_items = new_items_stack.pop() + value = exit(path, key, old_parent, new_parent, new_items) + registry[id_value] = value + if not new_items_stack: + continue + elif id_value in registry: + value = registry[id_value] + else: + res = enter(path, key, value) + try: + new_parent, new_items = res + except TypeError: + # TODO: handle False? + raise TypeError('enter should return a tuple of (new_parent,' + ' items_iterator), not: %r' % res) + if new_items is not False: + # traverse unless False is explicitly passed + registry[id_value] = new_parent + new_items_stack.append((path, [])) + if value is not root: + path += (key,) + stack.append((_REMAP_EXIT, (key, new_parent, value))) + if new_items: + stack.extend(reversed(list(new_items))) + continue + if visit is _orig_default_visit: + # avoid function call overhead by inlining identity operation + visited_item = (key, value) + else: + try: + visited_item = visit(path, key, value) + except Exception: + if reraise_visit: + raise + visited_item = True + if visited_item is False: + continue # drop + elif visited_item is True: + visited_item = (key, value) + # TODO: typecheck? + # raise TypeError('expected (key, value) from visit(),' + # ' not: %r' % visited_item) + try: + new_items_stack[-1][1].append(visited_item) + except IndexError: + raise TypeError('expected remappable root, not: %r' % root) + return value + + def merge_items(target_list, sourced=False): if not sourced: target_list = [(id(t), t) for t in target_list] @@ -262,7 +535,7 @@ def remerge_enter(path, key, value): new_parent = ret try: - cur_val = boltons.iterutils.get_path(ret, path + (key,)) + cur_val = get_path(ret, path + (key,)) except KeyError as ke: pass else: @@ -279,9 +552,9 @@ def remerge_visit(path, key, value): source_map[path + (key,)] = t_name return True else: - remerge_visit = boltons.iterutils.default_visit + remerge_visit = default_visit - ret = boltons.iterutils.remap(target, enter=remerge_enter, visit=remerge_visit, + ret = remap(target, enter=remerge_enter, visit=remerge_visit, exit=remerge_exit) if not sourced: diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index abece0200c..781e2e9843 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -18,7 +18,7 @@ from ._utils import escape_string if PY2: - from functools32 import lru_cache + from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 92f1a9c92c..5164ea8b2a 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -4,7 +4,7 @@ from ._compat import unicode if PY2: - from functools32 import lru_cache + from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 8812d2ae79..741ae3193c 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,13 +21,13 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.3 +pythonfinder==1.1.3.post1 requests==2.20.0 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.1.10 +requirementslib==1.2.0 attrs==18.2.0 distlib==0.2.8 packaging==18.0 @@ -41,7 +41,7 @@ semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 -vistir==0.1.7 +vistir==0.2.0 pip-shims==0.3.1 ptyprocess==0.6.0 enum34==1.1.6 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index 881985d2a0..c8b253b8f4 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -11,11 +11,11 @@ spinner, ) from .misc import load_path, partialclass, run, shell_escape -from .path import mkdir_p, rmtree, create_tracked_tempdir +from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile from .spin import VistirSpinner, create_spinner -__version__ = '0.1.8' +__version__ = '0.2.0' __all__ = [ @@ -36,5 +36,6 @@ "spinner", "VistirSpinner", "create_spinner", - "create_tracked_tempdir" + "create_tracked_tempdir", + "create_tracked_tempfile", ] diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 43470a6ee9..7b8066eeed 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -175,6 +175,7 @@ def NamedTemporaryFile( prefix=None, dir=None, delete=True, + wrapper_class_override=None ): """Create and return a temporary file. Arguments: @@ -203,7 +204,10 @@ def NamedTemporaryFile( file = io.open( fd, mode, buffering=buffering, newline=newline, encoding=encoding ) - return _TemporaryFileWrapper(file, name, delete) + if wrapper_class_override is not None: + return wrapper_class_override(file, name, delete) + else: + return _TemporaryFileWrapper(file, name, delete) except BaseException: os.unlink(name) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index eab879088d..ec3b65cbc6 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -33,9 +33,10 @@ from pathlib2 import Path from pipenv.vendor.backports.functools_lru_cache import lru_cache +from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile if sys.version_info < (3, 3): from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile from shutil import get_terminal_size diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 8f25e0795b..bcbf754192 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -231,12 +231,13 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None): @contextmanager -def open_file(link, session=None): +def open_file(link, session=None, stream=True): """ Open local or remote file for reading. :type link: pip._internal.index.Link or str :type session: requests.Session + :param bool stream: Try to stream if remote, default True :raises ValueError: If link points to a local directory. :return: a context manager to the opened file-like object """ @@ -255,11 +256,8 @@ def open_file(link, session=None): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - try: - local_file = io.open(local_path, "rb") - yield local_file - finally: - local_file.close() + with io.open(local_path, "rb") as local_file: + yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} @@ -267,8 +265,14 @@ def open_file(link, session=None): from requests import Session session = Session() - response = session.get(link, headers=headers, stream=True) - try: - yield response.raw - finally: - response.close() + with session.get(link, headers=headers, stream=stream) as resp: + try: + raw = getattr(resp, "raw", None) + result = raw if raw else resp + yield result + finally: + result.close() + if raw: + conn = getattr(raw, "_connection") + if conn is not None: + conn.close() diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index e2f859853f..2638241990 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -49,7 +49,7 @@ def _get_logger(name=None, level="ERROR"): formatter = logging.Formatter( "%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s" ) - handler = logging.StreamHandler() + handler = logging.StreamHandler(stream=sys.stderr) handler.setFormatter(formatter) logger.addHandler(handler) return logger @@ -157,7 +157,7 @@ def _create_subprocess( raise if not block: c.stdin.close() - log_level = "DEBUG" if verbose else "WARN" + log_level = "DEBUG" if verbose else "ERROR" logger = _get_logger(cmd._parts[0], level=log_level) output = [] err = [] @@ -199,7 +199,7 @@ def _create_subprocess( display_line = "{0}...".format(stdout_line[:display_limit]) if verbose: if spinner: - spinner.write(fs_str(display_line)) + spinner.write_err(fs_str(display_line)) else: logger.debug(display_line) if spinner: diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index ce7ecee055..d580aba23f 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -15,7 +15,15 @@ from six.moves import urllib_parse from six.moves.urllib import request as urllib_request -from .compat import Path, _fs_encoding, TemporaryDirectory, ResourceWarning +from .backports.tempfile import _TemporaryFileWrapper +from .compat import ( + _NamedTemporaryFile, + Path, + ResourceWarning, + TemporaryDirectory, + _fs_encoding, + finalize, +) __all__ = [ @@ -28,6 +36,7 @@ "mkdir_p", "ensure_mkdir_p", "create_tracked_tempdir", + "create_tracked_tempfile", "path_to_url", "rmtree", "safe_expandvars", @@ -37,6 +46,10 @@ ] +if os.name == "nt" and six.PY34: + warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") + + def unicode_path(path): # Paths are supposed to be represented as unicode here if six.PY2 and not isinstance(path, six.text_type): @@ -52,9 +65,10 @@ def native_path(path): # once again thank you django... # https://github.com/django/django/blob/fc6b90b/django/utils/_os.py -if six.PY3 or os.name == 'nt': +if six.PY3 or os.name == "nt": abspathu = os.path.abspath else: + def abspathu(path): """ Version of os.path.abspath that uses the unicode representation @@ -74,6 +88,7 @@ def normalize_drive(path): always converted to uppercase because it seems to be preferred. """ from .misc import to_text + if os.name != "nt" or not isinstance(path, six.string_types): return path @@ -110,6 +125,7 @@ def url_to_path(url): Follows logic taken from pip's equivalent function """ from .misc import to_bytes + assert is_file_url(url), "Only file: urls can be converted to local paths" _, netloc, path, _, _ = urllib_parse.urlsplit(url) # Netlocs are UNC paths @@ -123,6 +139,7 @@ def url_to_path(url): def is_valid_url(url): """Checks if a given string is an url""" from .misc import to_text + if not url: return url pieces = urllib_parse.urlparse(to_text(url)) @@ -132,6 +149,7 @@ def is_valid_url(url): def is_file_url(url): """Returns true if the given url is a file url""" from .misc import to_text + if not url: return False if not isinstance(url, six.string_types): @@ -149,6 +167,7 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ from .misc import to_bytes + fn = to_bytes(fn, encoding="utf-8") if os.path.exists(fn): return bool(os.stat(fn).st_mode & stat.S_IREAD) and not os.access(fn, os.W_OK) @@ -164,6 +183,7 @@ def mkdir_p(newdir, mode=0o777): """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ from .misc import to_bytes, to_text + newdir = to_bytes(newdir, "utf-8") if os.path.exists(newdir): if not os.path.isdir(newdir): @@ -176,7 +196,9 @@ def mkdir_p(newdir, mode=0o777): head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) # Make sure the tail doesn't point to the asame place as the head curdir = to_bytes(".", encoding="utf-8") - tail_and_head_match = os.path.relpath(tail, start=os.path.basename(head)) == curdir + tail_and_head_match = ( + os.path.relpath(tail, start=os.path.basename(head)) == curdir + ) if tail and not tail_and_head_match and not os.path.isdir(newdir): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): @@ -191,6 +213,7 @@ def mkdir_p(newdir, mode=0o777): def ensure_mkdir_p(mode=0o777): """Decorator to ensure `mkdir_p` is called to the function's return value. """ + def decorator(f): @functools.wraps(f) @@ -224,6 +247,19 @@ def create_tracked_tempdir(*args, **kwargs): return tempdir.name +def create_tracked_tempfile(*args, **kwargs): + """Create a tracked temporary file. + + This uses the `NamedTemporaryFile` construct, but does not remove the file + until the interpreter exits. + + The return value is the file object. + """ + + kwargs["wrapper_class_override"] = _TrackedTempfileWrapper + return _NamedTemporaryFile(*args, **kwargs) + + def set_write_bit(fn): """Set read-write permissions for the current user on the target path. Fail silently if the path doesn't exist. @@ -232,6 +268,7 @@ def set_write_bit(fn): """ from .misc import to_bytes, locale_encoding + fn = to_bytes(fn, encoding=locale_encoding) if not os.path.exists(fn): return @@ -253,6 +290,7 @@ def rmtree(directory, ignore_errors=False): """ from .misc import locale_encoding, to_bytes + directory = to_bytes(directory, encoding=locale_encoding) try: shutil.rmtree( @@ -278,10 +316,12 @@ def handle_remove_readonly(func, path, exc): This function will call check :func:`is_readonly_path` before attempting to call :func:`set_write_bit` on the target path and try again. """ + # Check for read-only attribute if six.PY2: from .compat import ResourceWarning from .misc import to_bytes + PERM_ERRORS = (errno.EACCES, errno.EPERM) default_warning_message = ( "Unable to remove file due to permissions restriction: {!r}" @@ -418,3 +458,28 @@ def safe_expandvars(value): if isinstance(value, six.string_types): return os.path.expandvars(value) return value + + +class _TrackedTempfileWrapper(_TemporaryFileWrapper): + def __init__(self, *args, **kwargs): + super(_TrackedTempfileWrapper, self).__init__(*args, **kwargs) + self._finalizer = finalize(self, self.cleanup) + + @classmethod + def _cleanup(cls, fileobj): + try: + fileobj.close() + finally: + os.unlink(fileobj.name) + + def cleanup(self): + if self._finalizer.detach(): + try: + self.close() + finally: + os.unlink(self.name) + else: + try: + self.close() + except OSError: + pass diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index d2cddd7947..e4b4ba66f2 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -3,7 +3,7 @@ import signal import sys -from .termcolors import colored +from .termcolors import colored, COLORS from .compat import fs_str import cursor @@ -15,6 +15,7 @@ Spinners = None else: from yaspin.spinners import Spinners + from yaspin.constants import COLOR_MAP handler = None if yaspin and os.name == "nt": @@ -41,6 +42,16 @@ def __exit__(self, exc_type, exc_val, traceback): self.write_err(traceback) return False + def __getattr__(self, k): + try: + retval = super(DummySpinner, self).__getattribute__(k) + except AttributeError: + if k in COLOR_MAP.keys() or k.upper() in COLORS: + return self + raise + else: + return retval + def fail(self, exitcode=1, text=None): if text: self.write_err(text) @@ -125,6 +136,42 @@ def _compose_color_func(self): ) return fn + def _register_signal_handlers(self): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass + + for sig, sig_handler in self._sigmap.items(): + # A handler for a particular signal, once set, remains + # installed until it is explicitly reset. Store default + # signal handlers for subsequent reset at cleanup phase. + dfl_handler = signal.getsignal(sig) + self._dfl_sigmap[sig] = dfl_handler + + # ``signal.SIG_DFL`` and ``signal.SIG_IGN`` are also valid + # signal handlers and are not callables. + if callable(sig_handler): + # ``signal.signal`` accepts handler function which is + # called with two arguments: signal number and the + # interrupted stack frame. ``functools.partial`` solves + # the problem of passing spinner instance into the handler + # function. + sig_handler = functools.partial(sig_handler, spinner=self) + + signal.signal(sig, sig_handler) + + def _reset_signal_handlers(self): + for sig, sig_handler in self._dfl_sigmap.items(): + signal.signal(sig, sig_handler) + @staticmethod def _hide_cursor(): cursor.hide() diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py index 6f3ad32cf7..b8ccda8e48 100644 --- a/pipenv/vendor/vistir/termcolors.py +++ b/pipenv/vendor/vistir/termcolors.py @@ -79,6 +79,7 @@ def colored(text, color=None, on_color=None, attrs=None): style = "BRIGHT" attrs.remove('bold') if color is not None: + color = color.upper() text = text = "%s%s%s%s%s" % ( getattr(colorama.Fore, color), getattr(colorama.Style, style), @@ -88,8 +89,9 @@ def colored(text, color=None, on_color=None, attrs=None): ) if on_color is not None: + on_color = on_color.upper() text = "%s%s%s%s" % ( - getattr(colorama.Back, color), + getattr(colorama.Back, on_color), text, colorama.Back.RESET, colorama.Style.NORMAL, diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index d01fb98ef1..06b8b62116 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -16,6 +16,9 @@ import threading import time +import colorama +import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS @@ -23,6 +26,9 @@ from .termcolor import colored +colorama.init() + + class Yaspin(object): """Implements a context manager that spawns a thread to write spinner frames into a tty (stdout) during @@ -369,11 +375,14 @@ def _register_signal_handlers(self): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this # signal. - if signal.SIGKILL in self._sigmap.keys(): - raise ValueError( - "Trying to set handler for SIGKILL signal. " - "SIGKILL cannot be cought or ignored in POSIX systems." - ) + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass for sig, sig_handler in iteritems(self._sigmap): # A handler for a particular signal, once set, remains @@ -521,14 +530,12 @@ def _set_text(text): @staticmethod def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() + cursor.hide() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() + cursor.show() @staticmethod def _clear_line(): - sys.stdout.write("\033[K") + sys.stdout.write(chr(27) + "[K") diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 3198c2d40d..38ec032f6b 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -34,6 +34,7 @@ # from time to time, remove the no longer needed ones HARDCODED_LICENSE_URLS = { 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', + 'cursor': 'https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE', 'delegator.py': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', 'click-didyoumean': 'https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE', 'click-completion': 'https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE', @@ -70,6 +71,7 @@ LIBRARY_RENAMES = { 'pip': 'pipenv.patched.notpip', + "functools32": "pipenv.vendor.backports.functools_lru_cache", 'enum34': 'enum', } diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch index ac63825c35..175efaa16d 100644 --- a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -1,8 +1,26 @@ diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py -index 0c140cad..3ffb2e31 100644 +index d15aeb97..56d12458 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py -@@ -178,6 +178,7 @@ class Command(object): +@@ -7,6 +7,8 @@ import locale + import errno + + from pexpect.popen_spawn import PopenSpawn ++import pexpect ++pexpect.EOF.__module__ = "pexpect.exceptions" + + # Include `unicode` in STR_TYPES for Python 2.X + try: +@@ -110,7 +112,7 @@ class Command(object): + if self.subprocess.before: + result += self.subprocess.before + +- if self.subprocess.after: ++ if self.subprocess.after and self.subprocess.after is not pexpect.EOF: + result += self.subprocess.after + + result += self.subprocess.read() +@@ -178,6 +180,7 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() @@ -10,11 +28,21 @@ index 0c140cad..3ffb2e31 100644 popen_kwargs["universal_newlines"] = not binary if cwd: popen_kwargs["cwd"] = cwd -@@ -233,18 +234,23 @@ class Command(object): - def block(self): +@@ -205,7 +208,10 @@ class Command(object): + if self.blocking: + raise RuntimeError("expect can only be used on non-blocking commands.") + +- self.subprocess.expect(pattern=pattern, timeout=timeout) ++ try: ++ self.subprocess.expect(pattern=pattern, timeout=timeout) ++ except pexpect.EOF: ++ pass + + def send(self, s, end=os.linesep, signal=False): + """Sends the given string or signal to std_in.""" +@@ -234,14 +240,25 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: -- self.subprocess.stdin.close() # consume stdout and stderr - try: - stdout, stderr = self.subprocess.communicate() @@ -35,10 +63,21 @@ index 0c140cad..3ffb2e31 100644 + self.std_err.close() + self.subprocess.wait() else: - self.subprocess.sendeof() -- self.subprocess.proc.stdout.close() - self.subprocess.wait() -+ self.subprocess.proc.stdout.close() +- self.subprocess.wait() ++ self.subprocess.sendeof() ++ try: ++ self.subprocess.wait() ++ finally: ++ if self.subprocess.proc.stdout: ++ self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next +@@ -263,7 +280,6 @@ class Command(object): + c.run(block=False, cwd=cwd) + if data: + c.send(data) +- c.subprocess.sendeof() + c.block() + return c + diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index f93e79594b..673efad815 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -1,17 +1,35 @@ +diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py +index 483a479a..43470a6e 100644 +--- a/pipenv/vendor/vistir/backports/tempfile.py ++++ b/pipenv/vendor/vistir/backports/tempfile.py +@@ -13,7 +13,7 @@ import six + try: + from weakref import finalize + except ImportError: +- from backports.weakref import finalize ++ from pipenv.vendor.backports.weakref import finalize + + + __all__ = ["finalize", "NamedTemporaryFile"] diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py -index 1f1b7a96..0c865fe6 100644 +index 9ae33fdc..ec3b65cb 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py -@@ -30,7 +30,7 @@ else: +@@ -31,11 +31,11 @@ if sys.version_info >= (3, 5): + from functools import lru_cache + else: from pathlib2 import Path +- from backports.functools_lru_cache import lru_cache ++ from pipenv.vendor.backports.functools_lru_cache import lru_cache + from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile if sys.version_info < (3, 3): - from backports.shutil_get_terminal_size import get_terminal_size + from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile -@@ -39,7 +39,7 @@ else: +@@ -44,7 +44,7 @@ else: try: from weakref import finalize except ImportError: @@ -20,16 +38,3 @@ index 1f1b7a96..0c865fe6 100644 try: from functools import partialmethod -diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py -index 483a479a..43470a6e 100644 ---- a/pipenv/vendor/vistir/backports/tempfile.py -+++ b/pipenv/vendor/vistir/backports/tempfile.py -@@ -13,7 +13,7 @@ import six - try: - from weakref import finalize - except ImportError: -- from backports.weakref import finalize -+ from pipenv.vendor.backports.weakref import finalize - - - __all__ = ["finalize", "NamedTemporaryFile"] diff --git a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch new file mode 100644 index 0000000000..a1d27cd303 --- /dev/null +++ b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch @@ -0,0 +1,62 @@ +diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py +index d01fb98e..06b8b621 100644 +--- a/pipenv/vendor/yaspin/core.py ++++ b/pipenv/vendor/yaspin/core.py +@@ -16,6 +16,9 @@ import sys + import threading + import time + ++import colorama ++import cursor ++ + from .base_spinner import default_spinner + from .compat import PY2, basestring, builtin_str, bytes, iteritems, str + from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS +@@ -23,6 +26,9 @@ from .helpers import to_unicode + from .termcolor import colored + + ++colorama.init() ++ ++ + class Yaspin(object): + """Implements a context manager that spawns a thread + to write spinner frames into a tty (stdout) during +@@ -369,11 +375,14 @@ class Yaspin(object): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. +- if signal.SIGKILL in self._sigmap.keys(): +- raise ValueError( +- "Trying to set handler for SIGKILL signal. " +- "SIGKILL cannot be cought or ignored in POSIX systems." +- ) ++ try: ++ if signal.SIGKILL in self._sigmap.keys(): ++ raise ValueError( ++ "Trying to set handler for SIGKILL signal. " ++ "SIGKILL cannot be cought or ignored in POSIX systems." ++ ) ++ except AttributeError: ++ pass + + for sig, sig_handler in iteritems(self._sigmap): + # A handler for a particular signal, once set, remains +@@ -521,14 +530,12 @@ class Yaspin(object): + + @staticmethod + def _hide_cursor(): +- sys.stdout.write("\033[?25l") +- sys.stdout.flush() ++ cursor.hide() + + @staticmethod + def _show_cursor(): +- sys.stdout.write("\033[?25h") +- sys.stdout.flush() ++ cursor.show() + + @staticmethod + def _clear_line(): +- sys.stdout.write("\033[K") ++ sys.stdout.write(chr(27) + "[K") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 12f2734237..7f797f9905 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,6 @@ import json import os +import sys import warnings import pytest @@ -7,13 +8,12 @@ from pipenv._compat import TemporaryDirectory, Path from pipenv.vendor import delegator from pipenv.vendor import requests -from pipenv.vendor import six from pipenv.vendor import toml from pytest_pypi.app import prepare_packages as prepare_pypi_packages +from vistir.compat import ResourceWarning -if six.PY2: - class ResourceWarning(Warning): - pass + +warnings.filterwarnings("default", category=ResourceWarning) HAS_WARNED_GITHUB = False @@ -25,8 +25,8 @@ def check_internet(): resp = requests.get('http://httpbin.org/ip', timeout=1.0) resp.raise_for_status() except Exception: - warnings.warn('Cannot connect to HTTPBin...', ResourceWarning) - warnings.warn('Will skip tests requiring Internet', ResourceWarning) + warnings.warn('Cannot connect to HTTPBin...', RuntimeWarning) + warnings.warn('Will skip tests requiring Internet', RuntimeWarning) return False return True @@ -46,10 +46,10 @@ def check_github_ssh(): global HAS_WARNED_GITHUB if not res and not HAS_WARNED_GITHUB: warnings.warn( - 'Cannot connect to GitHub via SSH', ResourceWarning + 'Cannot connect to GitHub via SSH', RuntimeWarning ) warnings.warn( - 'Will skip tests requiring SSH access to GitHub', ResourceWarning + 'Will skip tests requiring SSH access to GitHub', RuntimeWarning ) HAS_WARNED_GITHUB = True return res @@ -70,18 +70,109 @@ def pytest_runtest_setup(item): pytest.skip('requires github ssh') +@pytest.yield_fixture +def pathlib_tmpdir(request, tmpdir): + yield Path(str(tmpdir)) + tmpdir.remove(ignore_errors=True) + + +# Borrowed from pip's test runner filesystem isolation +@pytest.fixture(autouse=True) +def isolate(pathlib_tmpdir): + """ + Isolate our tests so that things like global configuration files and the + like do not affect our test results. + We use an autouse function scoped fixture because we want to ensure that + every test has it's own isolated home directory. + """ + warnings.filterwarnings("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + + + # Create a directory to use as our home location. + home_dir = os.path.join(str(pathlib_tmpdir), "home") + os.environ["PIPENV_NOSPIN"] = "1" + os.makedirs(home_dir) + + # Create a directory to use as a fake root + fake_root = os.path.join(str(pathlib_tmpdir), "fake-root") + os.makedirs(fake_root) + + # if sys.platform == 'win32': + # # Note: this will only take effect in subprocesses... + # home_drive, home_path = os.path.splitdrive(home_dir) + # os.environ.update({ + # 'USERPROFILE': home_dir, + # 'HOMEDRIVE': home_drive, + # 'HOMEPATH': home_path, + # }) + # for env_var, sub_path in ( + # ('APPDATA', 'AppData/Roaming'), + # ('LOCALAPPDATA', 'AppData/Local'), + # ): + # path = os.path.join(home_dir, *sub_path.split('/')) + # os.environ[env_var] = path + # os.makedirs(path) + # else: + # # Set our home directory to our temporary directory, this should force + # # all of our relative configuration files to be read from here instead + # # of the user's actual $HOME directory. + # os.environ["HOME"] = home_dir + # # Isolate ourselves from XDG directories + # os.environ["XDG_DATA_HOME"] = os.path.join(home_dir, ".local", "share") + # os.environ["XDG_CONFIG_HOME"] = os.path.join(home_dir, ".config") + # os.environ["XDG_CACHE_HOME"] = os.path.join(home_dir, ".cache") + # os.environ["XDG_RUNTIME_DIR"] = os.path.join(home_dir, ".runtime") + # os.environ["XDG_DATA_DIRS"] = ":".join([ + # os.path.join(fake_root, "usr", "local", "share"), + # os.path.join(fake_root, "usr", "share"), + # ]) + # os.environ["XDG_CONFIG_DIRS"] = os.path.join(fake_root, "etc", "xdg") + + # Configure git, because without an author name/email git will complain + # and cause test failures. + os.environ["GIT_CONFIG_NOSYSTEM"] = "1" + os.environ["GIT_AUTHOR_NAME"] = "pipenv" + os.environ["GIT_AUTHOR_EMAIL"] = "pipenv@pipenv.org" + + # We want to disable the version check from running in the tests + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" + workon_home = os.path.join(home_dir, ".virtualenvs") + os.makedirs(workon_home) + os.environ["WORKON_HOME"] = workon_home + project_dir = os.path.join(home_dir, "pipenv_project") + os.makedirs(project_dir) + os.environ["PIPENV_PROJECT_DIR"] = project_dir + os.environ["CI"] = "1" + + # Make sure tests don't share a requirements tracker. + os.environ.pop('PIP_REQ_TRACKER', None) + + # FIXME: Windows... + os.makedirs(os.path.join(home_dir, ".config", "git")) + with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: + fp.write( + b"[user]\n\tname = pipenv\n\temail = pipenv@pipenv.org\n" + ) + + class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False): + def __init__(self, pypi=None, pipfile=True, chdir=False, path=None): self.pypi = pypi self.original_umask = os.umask(0o007) self.original_dir = os.path.abspath(os.curdir) - self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') - path = Path(self._path.name) - try: - self.path = str(path.resolve()) - except OSError: - self.path = str(path.absolute()) + path = os.environ.get("PIPENV_PROJECT_DIR", None) + if not path: + self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + path = Path(self._path.name) + try: + self.path = str(path.resolve()) + except OSError: + self.path = str(path.absolute()) + else: + self._path = None + self.path = path # set file creation perms self.pipfile_path = None self.chdir = chdir @@ -101,6 +192,7 @@ def __enter__(self): os.environ['PIPENV_DONT_USE_PYENV'] = '1' os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' os.environ['PIPENV_VENV_IN_PROJECT'] = '1' + os.environ['PIPENV_NOSPIN'] = '1' if self.chdir: os.chdir(self.path) return self @@ -110,13 +202,13 @@ def __exit__(self, *args): if self.chdir: os.chdir(self.original_dir) self.path = None - try: - self._path.cleanup() - except OSError as e: - _warn_msg = warn_msg.format(e) - warnings.warn(_warn_msg, ResourceWarning) - finally: - os.umask(self.original_umask) + if self._path: + try: + self._path.cleanup() + except OSError as e: + _warn_msg = warn_msg.format(e) + warnings.warn(_warn_msg, ResourceWarning) + os.umask(self.original_umask) def pipenv(self, cmd, block=True): if self.pipfile_path: @@ -162,7 +254,7 @@ def lockfile_path(self): @pytest.fixture() def PipenvInstance(): - return _PipenvInstance + yield _PipenvInstance @pytest.fixture(scope='module') diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 1f1719d00a..7743804df0 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -37,9 +37,9 @@ def test_lock_requirements_file(PipenvInstance, pypi): """.strip() f.write(contents) - req_list = ("requests==2.14.0") + req_list = ("requests==2.14.0",) - dev_req_list = ("flask==0.12.2") + dev_req_list = ("flask==0.12.2",) c = p.pipenv('lock -r') d = p.pipenv('lock -r -d') diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 98c8560203..40977ede67 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4,6 +4,7 @@ from mock import patch, Mock from first import first import pipenv.utils +import pythonfinder.utils # Pipfile format <-> requirements.txt format. @@ -215,13 +216,13 @@ def test_python_version_from_non_python(self): ), ], ) - @patch("delegator.run") + # @patch(".vendor.pythonfinder.utils.get_python_version") def test_python_version_output_variants( - self, mocked_delegator, version_output, version + self, monkeypatch, version_output, version ): - run_ret = Mock() - run_ret.out = version_output - mocked_delegator.return_value = run_ret + def mock_version(path): + return version_output.split()[1] + monkeypatch.setattr("pipenv.vendor.pythonfinder.utils.get_python_version", mock_version) assert pipenv.utils.python_version("some/path") == version @pytest.mark.utils From 6294c57070a95887199f1ce21afa09c2cc1ada22 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 25 Oct 2018 01:23:41 -0400 Subject: [PATCH 05/13] Update pythonfinder Signed-off-by: Dan Ryan More debugging info Signed-off-by: Dan Ryan Fix stdout write bugs Signed-off-by: Dan Ryan Fix spinner invocations Signed-off-by: Dan Ryan Create missing directory Signed-off-by: Dan Ryan Filesystem encode envvars Signed-off-by: Dan Ryan convert envvars to strings Signed-off-by: Dan Ryan Update encodings Signed-off-by: Dan Ryan Update vistir to init colorama Signed-off-by: Dan Ryan Update vistir version number Signed-off-by: Dan Ryan add some debugging and vsts changes Signed-off-by: Dan Ryan fix delegator mod Signed-off-by: Dan Ryan Log to stderr so we can see it Signed-off-by: Dan Ryan Log to stderr so we can see it Signed-off-by: Dan Ryan Try importing colorama... Signed-off-by: Dan Ryan change variable setting syntax Signed-off-by: Dan Ryan --- .vsts-ci/phases/run-tests.yml | 8 +- .vsts-ci/steps/run-tests.yml | 11 +- pipenv/__init__.py | 4 +- pipenv/core.py | 54 ++-- pipenv/resolver.py | 41 +-- pipenv/test_script.py | 32 ++ pipenv/utils.py | 41 ++- .../shutil_get_terminal_size/__init__.py | 8 +- .../get_terminal_size.py | 17 +- pipenv/vendor/delegator.py | 5 +- pipenv/vendor/dotenv/main.py | 7 - pipenv/vendor/passa/internals/dependencies.py | 3 +- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/cli.py | 12 +- pipenv/vendor/pythonfinder/models/path.py | 3 +- pipenv/vendor/pythonfinder/models/pyenv.py | 2 - pipenv/vendor/pythonfinder/pythonfinder.py | 11 + pipenv/vendor/pythonfinder/utils.py | 7 +- pipenv/vendor/requirementslib/__init__.py | 8 +- .../vendor/requirementslib/models/__init__.py | 3 +- .../requirementslib/models/dependencies.py | 29 +- pipenv/vendor/requirementslib/utils.py | 5 +- pipenv/vendor/resolvelib/__init__.py | 16 + pipenv/vendor/resolvelib/providers.py | 78 +++++ pipenv/vendor/resolvelib/reporters.py | 23 ++ pipenv/vendor/resolvelib/resolvers.py | 287 ++++++++++++++++++ pipenv/vendor/resolvelib/structs.py | 67 ++++ pipenv/vendor/shutil_backports/__init__.py | 9 + .../shutil_backports/get_terminal_size.py | 100 ++++++ pipenv/vendor/vendor.txt | 6 +- pipenv/vendor/vistir/__init__.py | 5 +- pipenv/vendor/vistir/backports/tempfile.py | 6 +- pipenv/vendor/vistir/compat.py | 8 + pipenv/vendor/vistir/contextmanagers.py | 7 +- pipenv/vendor/vistir/misc.py | 61 ++-- pipenv/vendor/vistir/path.py | 9 +- pipenv/vendor/vistir/spin.py | 158 +++++++--- pipenv/vendor/vistir/termcolors.py | 52 +++- pipenv/vendor/yaspin/core.py | 27 +- .../vendor/delegator-close-filehandles.patch | 184 ++++++++++- tests/integration/conftest.py | 96 ++---- 41 files changed, 1201 insertions(+), 311 deletions(-) create mode 100644 pipenv/test_script.py create mode 100644 pipenv/vendor/resolvelib/__init__.py create mode 100644 pipenv/vendor/resolvelib/providers.py create mode 100644 pipenv/vendor/resolvelib/reporters.py create mode 100644 pipenv/vendor/resolvelib/resolvers.py create mode 100644 pipenv/vendor/resolvelib/structs.py create mode 100644 pipenv/vendor/shutil_backports/__init__.py create mode 100644 pipenv/vendor/shutil_backports/get_terminal_size.py diff --git a/.vsts-ci/phases/run-tests.yml b/.vsts-ci/phases/run-tests.yml index 81f02875ff..daf23e6a8c 100644 --- a/.vsts-ci/phases/run-tests.yml +++ b/.vsts-ci/phases/run-tests.yml @@ -11,8 +11,8 @@ steps: mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" mkdir -p "$WORKON_HOME" pip install certifi - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" echo "Path: $PATH" echo "Installing Pipenv…" @@ -23,8 +23,8 @@ steps: - script: | # Fix Git SSL errors - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" pipenv run pytest --junitxml=test-results.xml displayName: Run integration tests diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index cb24b092f5..d893ccf934 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -3,13 +3,14 @@ steps: # Fix Git SSL errors pip install certifi python -m certifi > cacert.txt - $env:GIT_SSL_CAINFO = $(Get-Content cacert.txt) + Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]"$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail # https://bugs.python.org/issue18199 - subst T: $env:TEMP - $env:TEMP = "T:\" - $env:TMP = "T:\" - D:\.venv\Scripts\pipenv run pytest -n 4 --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + subst T: "$env:TEMP" + Write-Host "##vso[task.setvariable variable=TEMP]"T:\" + Write-Host "##vso[task.setvariable variable=TMP]"T:\" + Get-ChildItem Env + D:\.venv\Scripts\pipenv run pytest -n 4 -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 1fea44d514..8c6cec195a 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -6,7 +6,6 @@ import sys from .__version__ import __version__ - PIPENV_ROOT = os.path.dirname(os.path.realpath(__file__)) PIPENV_VENDOR = os.sep.join([PIPENV_ROOT, "vendor"]) PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) @@ -14,7 +13,8 @@ sys.path.insert(0, PIPENV_VENDOR) # Inject patched directory into system path. sys.path.insert(0, PIPENV_PATCHED) -os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" +from vistir.compat import fs_str +os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") # Hack to make things work better. try: if "concurrency" in sys.modules: diff --git a/pipenv/core.py b/pipenv/core.py index 82c11f1d31..64f486d71e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -17,6 +17,7 @@ import six import urllib3.util as urllib3_util +from functools import partial from .cmdparse import Script from .project import Project, SourceNotFound @@ -110,7 +111,7 @@ def fix_utf8(text): @contextlib.contextmanager -def spinner(text=None, nospin=None, spinner_name=None): +def _spinner(text=None, nospin=None, spinner_name=None): if not text: text = "Running..." if not spinner_name: @@ -125,6 +126,10 @@ def spinner(text=None, nospin=None, spinner_name=None): yield sp +spinner = partial(_spinner, text="Running...", nospin=environments.PIPENV_NOSPIN, + spinner_name=environments.PIPENV_SPINNER) + + def which(command, location=None, allow_global=False): if not allow_global and location is None: if project.virtualenv_exists: @@ -322,9 +327,16 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): ) # Create a Pipfile… project.create_pipfile(python=python) - with spinner(): + with spinner(text=vistir.compat.fs_str("Importing requirements..."), + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: # Import requirements.txt. - import_requirements() + try: + import_requirements() + except Exception: + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed...")) + else: + sp.ok(environments.PIPENV_SPINNER_FAIL_TEXT.format("Success!")) # Warn the user of side-effects. click.echo( u"{0}: Your {1} now contains pinned versions, if your {2} did. \n" @@ -467,14 +479,21 @@ def abort(): crayons.normal(fix_utf8("…"), bold=True), ) ) - with spinner(): + with spinner(text=vistir.compat.fs_str("Installing python..."), + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: try: c = pyenv.install(version) except PyenvError as e: - click.echo(fix_utf8("Something went wrong…")) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Failed...") + ) + click.echo(fix_utf8("Something went wrong…"), err=True) click.echo(crayons.blue(e.err), err=True) + else: + environments.PIPENV_SPINNER_OK_TEXT.format("Success!") # Print the results, in a beautiful blue… - click.echo(crayons.blue(c.out), err=True) + click.echo(crayons.blue(c.out), err=True) # Find the newly installed Python, hopefully. version = str(version) path_to_python = find_a_system_python(version) @@ -756,7 +775,6 @@ def cleanup_procs(procs, concurrent): deps_list_bar = progress.bar(deps_list, width=32, label=INSTALL_LABEL if os.name != "nt" else "") - indexes = [] for dep in deps_list_bar: index = None @@ -845,13 +863,14 @@ def cleanup_procs(procs, concurrent): # Return the subprocess' return code. sys.exit(c.return_code) else: - click.echo( - "{0} {1}{2}".format( - crayons.green("Success installing"), - crayons.green(dep.name), - crayons.green("!"), + if environments.is_verbose(): + click.echo( + "{0} {1}{2}".format( + crayons.green("Success installing"), + crayons.green(dep.as_line(include_hashes=False)), + crayons.green("!"), + ), ) - ) def convert_three_to_python(three, python): @@ -1014,9 +1033,6 @@ def do_lock( if dev_package in project.packages: dev_packages[dev_package] = project.packages[dev_package] # Resolve dev-package dependencies, with pip-tools. - pip_freeze = delegator.run( - "{0} freeze".format(escape_grouped_arguments(which_pip(allow_global=system))) - ).out sections = { "dev": { "packages": project.dev_packages, @@ -1072,7 +1088,6 @@ def do_lock( # TODO: be smarter about this. vcs_reqs, vcs_lockfile = get_vcs_deps( project, - pip_freeze, which=which, clear=clear, pre=pre, @@ -2185,6 +2200,10 @@ def _launch_windows_subprocess(script): command = system_which(script.command) options = {"universal_newlines": True} + env_strings = [ + vistir.compat.to_native_string("{0}: {1}".format(k, v)) for k, v in os.environ.items() + ] + click.echo(vistir.compat.to_native_string("\n".join(env_strings)), err=True) # Command not found, maybe this is a shell built-in? if not command: @@ -2204,6 +2223,7 @@ def _launch_windows_subprocess(script): def do_run_nt(script): + os.environ = {k: vistir.compat.fs_str(val) for k, val in os.environ.items()} p = _launch_windows_subprocess(script) p.communicate() sys.exit(p.returncode) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 2854a93ab4..b82088a22d 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -3,7 +3,7 @@ import json import logging -os.environ["PIP_PYTHON_PATH"] = sys.executable +os.environ["PIP_PYTHON_PATH"] = str(sys.executable) def _patch_path(): @@ -17,7 +17,7 @@ def _patch_path(): def get_parser(): from argparse import ArgumentParser - parser = ArgumentParser("pipenvresolver") + parser = ArgumentParser("pipenv-resolver") parser.add_argument("--pre", action="store_true", default=False) parser.add_argument("--clear", action="store_true", default=False) parser.add_argument("--verbose", "-v", action="count", default=False) @@ -41,17 +41,13 @@ def handle_parsed_args(parsed): elif parsed.verbose > 0: logging.getLogger("notpip").setLevel(logging.INFO) if "PIPENV_PACKAGES" in os.environ: - parsed.packages += os.environ["PIPENV_PACKAGES"].strip().split("\n") + parsed.packages += os.environ.get("PIPENV_PACKAGES", "").strip().split("\n") return parsed -def main(pre, clear, verbose, system, requirements_dir, packages): +def _main(pre, clear, verbose, system, requirements_dir, packages): os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) - os.environ["PIP_PYTHON_PATH"] = sys.executable - - import warnings - from pipenv.vendor.vistir.compat import ResourceWarning - warnings.filterwarnings("ignore", category=ResourceWarning) + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources @@ -91,22 +87,27 @@ def resolve(packages, pre, project, sources, clear, system, requirements_dir=Non ) print("RESULTS:") if results: - import traceback - if isinstance(results, (Exception, traceback.types.TracebackType)): - sys.stderr.write(traceback.print_tb(results)) - sys.stderr.write(sys.exc_value()) - else: - print(json.dumps(results)) + print(json.dumps(results)) else: print(json.dumps([])) -if __name__ == "__main__": - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" +def main(): _patch_path() + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + warnings.simplefilter("ignore", category=ResourceWarning) + from pipenv.vendor import colorama + colorama.init() + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") parser = get_parser() - parsed, remaining = parser.parse_known_intermixed_args() - sys.argv = remaining + parsed, remaining = parser.parse_known_args() + # sys.argv = remaining parsed = handle_parsed_args(parsed) - main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir, + _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir, parsed.packages) + + +if __name__ == "__main__": + _patch_path() + main() diff --git a/pipenv/test_script.py b/pipenv/test_script.py new file mode 100644 index 0000000000..d599ded637 --- /dev/null +++ b/pipenv/test_script.py @@ -0,0 +1,32 @@ +# -*- coding=utf-8 -*- + +import os +import sys + + +def _patch_path(): + import site + pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + pipenv_site_dir = os.path.dirname(pipenv_libdir) + site.addsitedir(pipenv_site_dir) + for _dir in ("vendor", "patched"): + sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) + + +def test_install(): + from pipenv.vendor.vistir.contextmanagers import cd + from pipenv.vendor.click.testing import CliRunner + runner = CliRunner() + with cd("/tmp/test"): + from pipenv.core import do_lock + locked = do_lock(system=False, clear=False, pre=False, keep_outdated=False, + write=True, pypi_mirror=None) + # result = runner.invoke(cli, ["lock", "--verbose"]) + # print(result.output) + # print(result.exit_code) + print(locked) + + +if __name__ == "__main__": + _patch_path() + test_install() diff --git a/pipenv/utils.py b/pipenv/utils.py index d8f2432bdb..7c78de3455 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -258,7 +258,7 @@ class PipCommand(basecommand.Command): if sources: pip_args = prepare_pip_source_args(sources, pip_args) if environments.is_verbose(): - click_echo(crayons.blue("Using pip: {0}".format(" ".join(pip_args)))) + click_echo(crayons.blue("Using pip: {0}".format(" ".join(pip_args))), err=True) constraints_file = create_tracked_tempfile( mode="w", prefix="pipenv-", @@ -315,7 +315,7 @@ class PipCommand(basecommand.Command): click_echo( crayons.blue( "Please check your version specifier and version number. See PEP440 for more information." - ) + ), err=True ) raise RuntimeError return (resolved_tree, hashes, markers_lookup, resolver) @@ -330,12 +330,12 @@ def venv_resolve_deps( allow_global=False, pypi_mirror=None, ): - from .vendor.vistir.misc import fs_str, run - from .vendor.vistir.compat import Path + from .vendor.vistir.misc import fs_str + from .vendor.vistir.compat import Path, to_native_string from .vendor.vistir.path import create_tracked_tempdir from .cmdparse import Script from .core import spinner - from .vendor.pexpect.exceptions import EOF + from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor import delegator from . import resolver import json @@ -364,26 +364,36 @@ def venv_resolve_deps( os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - out = "" + out = to_native_string("") EOF.__module__ = "pexpect.exceptions" with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, nospin=environments.PIPENV_NOSPIN) as sp: c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) - _out = u"" + _out = to_native_string("") + result = None while True: - result = c.expect(u"\n", timeout=-1) - if result is EOF or result is None: + try: + result = c.expect(u"\n", timeout=-1) + except (EOF, TIMEOUT): + pass + if result is None: break - _out = c.out - out += _out - sp.text = fs_str("Locking... {0}".format(_out[:100])) + _out = c.subprocess.before + if _out is not None: + _out = to_native_string("{0}".format(_out)) + out += _out + sp.text = to_native_string("Locking... {0}".format(_out[:100])) if environments.is_verbose(): - sp.write_err(_out.rstrip()) + if _out is not None: + sp._hide_cursor() + sp.write(_out.rstrip()) + sp._show_cursor() c.block() if c.return_code != 0: sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( "Locking Failed!" )) + click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) sys.exit(c.return_code) else: @@ -394,7 +404,7 @@ def venv_resolve_deps( return json.loads(c.out.split("RESULTS:")[1].strip()) except IndexError: - click_echo(c.out.strip()) + click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) raise RuntimeError("There was a problem with locking.") @@ -496,7 +506,7 @@ def resolve_deps( click_echo( "{0}: Error generating hash for {1}".format( crayons.red("Warning", bold=True), name - ) + ), err=True ) # # Collect un-collectable hashes (should work with devpi). # try: @@ -1088,7 +1098,6 @@ def extract_uri_from_vcs_dep(dep): def get_vcs_deps( project, - pip_freeze=None, which=None, clear=False, pre=False, diff --git a/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py b/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py index fa12816e27..cfcbdf6671 100644 --- a/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py +++ b/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py @@ -1,9 +1,11 @@ -__title__ = "shutil_backports" -__version__ = "0.1.0" +"""A backport of the get_terminal_size function from Python 3.3's shutil.""" + +__title__ = "backports.shutil_get_terminal_size" +__version__ = "1.0.0" __license__ = "MIT" __author__ = "Christopher Rosell" __copyright__ = "Copyright 2014 Christopher Rosell" __all__ = ["get_terminal_size"] -from .get_terminal_size import * +from .get_terminal_size import get_terminal_size diff --git a/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py b/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py index f1336e5839..28c96da807 100644 --- a/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py +++ b/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py @@ -39,7 +39,7 @@ def _get_terminal_size(fd): except Exception: pass - return columns, lines + return terminal_size(columns, lines) except ImportError: import fcntl @@ -52,7 +52,7 @@ def _get_terminal_size(fd): except Exception: columns = lines = 0 - return columns, lines + return terminal_size(columns, lines) def get_terminal_size(fallback=(80, 24)): @@ -74,7 +74,7 @@ def get_terminal_size(fallback=(80, 24)): The value returned is a named tuple of type os.terminal_size. """ - # Attempt to use the environment first + # Try the environment first try: columns = int(os.environ["COLUMNS"]) except (KeyError, ValueError): @@ -88,13 +88,14 @@ def get_terminal_size(fallback=(80, 24)): # Only query if necessary if columns <= 0 or lines <= 0: try: - columns, lines = _get_terminal_size(sys.__stdout__.fileno()) + size = _get_terminal_size(sys.__stdout__.fileno()) except (NameError, OSError): - pass + size = terminal_size(*fallback) - # Use fallback as last resort - if columns <= 0 and lines <= 0: - columns, lines = fallback + if columns <= 0: + columns = size.columns + if lines <= 0: + lines = size.lines return terminal_size(columns, lines) diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index 56d1245815..cf6f91c884 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -113,7 +113,10 @@ def _pexpect_out(self): result += self.subprocess.before if self.subprocess.after and self.subprocess.after is not pexpect.EOF: - result += self.subprocess.after + try: + result += self.subprocess.after + except (pexpect.EOF, pexpect.TIMEOUT): + pass result += self.subprocess.read() return result diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py index 6ba28bbbc3..4bf72946d3 100644 --- a/pipenv/vendor/dotenv/main.py +++ b/pipenv/vendor/dotenv/main.py @@ -95,13 +95,6 @@ def set_as_environment_variables(self, override=False): for k, v in self.dict().items(): if k in os.environ and not override: continue - # With Python 2 on Windows, ensuree environment variables are - # system strings to avoid "TypeError: environment can only contain - # strings" in Python's subprocess module. - if sys.version_info.major < 3 and sys.platform == 'win32': - from pipenv.utils import fs_str - k = fs_str(k) - v = fs_str(v) os.environ[k] = v return True diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py index 53b19b1719..6ce97aadaa 100644 --- a/pipenv/vendor/passa/internals/dependencies.py +++ b/pipenv/vendor/passa/internals/dependencies.py @@ -154,6 +154,7 @@ def _get_dependencies_from_json(ireq, sources): return dependencies except Exception as e: print("unable to read dependencies via {0} ({1})".format(url, e)) + session.close() return @@ -215,7 +216,7 @@ def _read_requires_python(metadata): def _get_dependencies_from_pip(ireq, sources): - """Retrieves dependencies for the requirement from pipenv.patched.notpip internals. + """Retrieves dependencies for the requirement from pip internals. The current strategy is to try the followings in order, returning the first successful result. diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 91e9cabb10..d800f92662 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.3.post1' +__version__ = '1.1.5' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index 1757c081ad..6e7980fede 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -30,17 +30,23 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup sys.exit(0) finder = Finder(ignore_unsupported=ignore_unsupported) if findall: - versions = finder.find_all_python_versions() + versions = [v for v in finder.find_all_python_versions()] if versions: click.secho("Found python at the following locations:", fg="green") for v in versions: py = v.py_version + comes_from = getattr(py, "comes_from", None) + if comes_from is not None: + comes_from_path = getattr(comes_from, "path", v.path) + else: + comes_from_path = v.path click.secho( - "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {py.comes_from.path!s}".format( - py=py + "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {comes_from!s}".format( + py=py, comes_from=comes_from_path ), fg="yellow", ) + sys.exit(0) else: click.secho( "ERROR: No valid python versions found! Check your path and try again.", diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index f39299a3b6..4a4c50da50 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -27,7 +27,6 @@ path_is_known_executable, unnest, ) -from .python import PythonVersion @attr.s @@ -504,7 +503,6 @@ def create(cls, path, is_root=False, only_python=False, pythons=None, name=None) children = {} child_creation_args = { "is_root": False, - "py_version": python, "only_python": only_python } if not guessed_name: @@ -512,6 +510,7 @@ def create(cls, path, is_root=False, only_python=False, pythons=None, name=None) for pth, python in pythons.items(): pth = ensure_path(pth) children[pth.as_posix()] = PathEntry( + py_version=python, path=pth, **child_creation_args ) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 1595a963a7..a9d6105b2d 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -14,8 +14,6 @@ from ..utils import ( ensure_path, optional_instance_of, - get_python_version, - filter_pythons, unnest, ) from .mixins import BaseFinder, BasePath diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 19a52e0a3b..854cc8e7cf 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -4,6 +4,7 @@ import six import operator from .models import SystemPath +from vistir.compat import lru_cache class Finder(object): @@ -34,6 +35,14 @@ def __init__(self, path=None, system=False, global_search=True, ignore_unsupport self._system_path = None self._windows_finder = None + def __hash__(self): + return hash( + (self.path_prepend, self.system, self.global_search, self.ignore_unsupported) + ) + + def __eq__(self, other): + return self.__hash__() == other.__hash__() + @property def system_path(self): if not self._system_path: @@ -56,6 +65,7 @@ def windows_finder(self): def which(self, exe): return self.system_path.which(exe) + @lru_cache(maxsize=128) def find_python_version( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): @@ -103,6 +113,7 @@ def find_python_version( major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) + @lru_cache(maxsize=128) def find_all_python_versions( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 2f5a860da1..fd5ac99d6f 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -49,12 +49,13 @@ def get_python_version(path): """Get python version string using subprocess from a given path.""" version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: - out, _ = vistir.misc.run(version_cmd, block=True, nospin=True) + c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, + combine_stderr=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) - if not out: + if not c.out: raise InvalidPythonVersion("%s is not a valid python path" % path) - return out.strip() + return c.out.strip() def optional_instance_of(cls): diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 7b4b6376d6..ba40e5f51f 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,9 +1,13 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.0' +__version__ = '1.2.1' + +import logging + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) from .models.requirements import Requirement from .models.lockfile import Lockfile from .models.pipfile import Pipfile - __all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/models/__init__.py b/pipenv/vendor/requirementslib/models/__init__.py index 8d12da457a..99e519d6ed 100644 --- a/pipenv/vendor/requirementslib/models/__init__.py +++ b/pipenv/vendor/requirementslib/models/__init__.py @@ -2,10 +2,9 @@ from __future__ import absolute_import -__all__ = ["Requirement", "Lockfile", "Pipfile", "DependencyResolver"] +__all__ = ["Requirement", "Lockfile", "Pipfile"] from .requirements import Requirement from .lockfile import Lockfile from .pipfile import Pipfile -from .resolvers import DependencyResolver diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index d9f1b65394..c3df222f0d 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -9,6 +9,7 @@ import packaging.markers import packaging.version import requests +import warnings from first import first from packaging.utils import canonicalize_name @@ -17,7 +18,7 @@ FormatControl, InstallRequirement, PackageFinder, RequirementPreparer, RequirementSet, RequirementTracker, Resolver, WheelCache, pip_version ) -from vistir.compat import JSONDecodeError, fs_str +from vistir.compat import JSONDecodeError, fs_str, ResourceWarning from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass from vistir.path import create_tracked_tempdir @@ -53,6 +54,8 @@ def find_all_matches(finder, ireq, pre=False): :return: A list of matching candidates. :rtype: list[:class:`~pip._internal.index.InstallationCandidate`] """ + + candidates = clean_requires_python(finder.find_all_candidates(ireq.name)) versions = {candidate.version for candidate in candidates} allowed_versions = _get_filtered_versions(ireq, versions, pre) @@ -341,9 +344,12 @@ def get_dependencies_from_json(ireq): def gen(ireq): info = None - info = session.get( - "https://pypi.org/pypi/{0}/{1}/json".format(ireq.req.name, version) - ).json()["info"] + try: + info = session.get( + "https://pypi.org/pypi/{0}/{1}/json".format(ireq.req.name, version) + ).json()["info"] + finally: + session.close() requires_dist = info.get("requires_dist", info.get("requires")) if not requires_dist: # The API can return None for this. return @@ -605,13 +611,16 @@ def start_resolver(finder=None, wheel_cache=None): wheel_cache=wheel_cache, use_user_site=False, ) - if packaging.version.parse(pip_version) >= packaging.version.parse('18'): - with RequirementTracker() as req_tracker: - preparer = preparer(req_tracker=req_tracker) + try: + if packaging.version.parse(pip_version) >= packaging.version.parse('18'): + with RequirementTracker() as req_tracker: + preparer = preparer(req_tracker=req_tracker) + yield resolver(preparer=preparer) + else: + preparer = preparer() yield resolver(preparer=preparer) - else: - preparer = preparer() - yield resolver(preparer=preparer) + finally: + finder.session.close() def get_grouped_dependencies(constraints): diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index 085f32415c..75a05ce05b 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -6,6 +6,7 @@ import os import six +import sys import tomlkit six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) @@ -23,8 +24,6 @@ from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir - - VCS_LIST = ("git", "svn", "hg", "bzr") VCS_SCHEMES = [] SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") @@ -38,7 +37,7 @@ def setup_logger(): logger = logging.getLogger("requirementslib") loglevel = logging.DEBUG - handler = logging.StreamHandler() + handler = logging.StreamHandler(stream=sys.stderr) handler.setLevel(loglevel) logger.addHandler(handler) logger.setLevel(loglevel) diff --git a/pipenv/vendor/resolvelib/__init__.py b/pipenv/vendor/resolvelib/__init__.py new file mode 100644 index 0000000000..e0e37434d2 --- /dev/null +++ b/pipenv/vendor/resolvelib/__init__.py @@ -0,0 +1,16 @@ +__all__ = [ + '__version__', + 'AbstractProvider', 'BaseReporter', 'Resolver', + 'NoVersionsAvailable', 'RequirementsConflicted', + 'ResolutionError', 'ResolutionImpossible', 'ResolutionTooDeep', +] + +__version__ = '0.2.2' + + +from .providers import AbstractProvider +from .reporters import BaseReporter +from .resolvers import ( + NoVersionsAvailable, RequirementsConflicted, + Resolver, ResolutionError, ResolutionImpossible, ResolutionTooDeep, +) diff --git a/pipenv/vendor/resolvelib/providers.py b/pipenv/vendor/resolvelib/providers.py new file mode 100644 index 0000000000..515c0db474 --- /dev/null +++ b/pipenv/vendor/resolvelib/providers.py @@ -0,0 +1,78 @@ +class AbstractProvider(object): + """Delegate class to provide requirment interface for the resolver. + """ + def identify(self, dependency): + """Given a dependency, return an identifier for it. + + This is used in many places to identify the dependency, e.g. whether + two requirements should have their specifier parts merged, whether + two specifications would conflict with each other (because they the + same name but different versions). + """ + raise NotImplementedError + + def get_preference(self, resolution, candidates, information): + """Produce a sort key for given specification based on preference. + + The preference is defined as "I think this requirement should be + resolved first". The lower the return value is, the more preferred + this group of arguments is. + + :param resolution: Currently pinned candidate, or `None`. + :param candidates: A list of possible candidates. + :param information: A list of requirement information. + + Each information instance is a named tuple with two entries: + + * `requirement` specifies a requirement contributing to the current + candidate list + * `parent` specifies the candidate that provids (dependend on) the + requirement, or `None` to indicate a root requirement. + + The preference could depend on a various of issues, including (not + necessarily in this order): + + * Is this package pinned in the current resolution result? + * How relaxed is the requirement? Stricter ones should probably be + worked on first? (I don't know, actually.) + * How many possibilities are there to satisfy this requirement? Those + with few left should likely be worked on first, I guess? + * Are there any known conflicts for this requirement? We should + probably work on those with the most known conflicts. + + A sortable value should be returned (this will be used as the `key` + parameter of the built-in sorting function). The smaller the value is, + the more preferred this specification is (i.e. the sorting function + is called with `reverse=False`). + """ + raise NotImplementedError + + def find_matches(self, requirement): + """Find all possible candidates that satisfy a requirement. + + This should try to get candidates based on the requirement's type. + For VCS, local, and archive requirements, the one-and-only match is + returned, and for a "named" requirement, the index(es) should be + consulted to find concrete candidates for this requirement. + + The returned candidates should be sorted by reversed preference, e.g. + the latest should be LAST. This is done so list-popping can be as + efficient as possible. + """ + raise NotImplementedError + + def is_satisfied_by(self, requirement, candidate): + """Whether the given requirement can be satisfied by a candidate. + + A boolean should be retuened to indicate whether `candidate` is a + viable solution to the requirement. + """ + raise NotImplementedError + + def get_dependencies(self, candidate): + """Get dependencies of a candidate. + + This should return a collection of requirements that `candidate` + specifies as its dependencies. + """ + raise NotImplementedError diff --git a/pipenv/vendor/resolvelib/reporters.py b/pipenv/vendor/resolvelib/reporters.py new file mode 100644 index 0000000000..c723031f45 --- /dev/null +++ b/pipenv/vendor/resolvelib/reporters.py @@ -0,0 +1,23 @@ +class BaseReporter(object): + """Delegate class to provider progress reporting for the resolver. + """ + def starting(self): + """Called before the resolution actually starts. + """ + + def starting_round(self, index): + """Called before each round of resolution starts. + + The index is zero-based. + """ + + def ending_round(self, index, state): + """Called before each round of resolution ends. + + This is NOT called if the resolution ends at this round. Use `ending` + if you want to report finalization. The index is zero-based. + """ + + def ending(self, state): + """Called before the resolution ends successfully. + """ diff --git a/pipenv/vendor/resolvelib/resolvers.py b/pipenv/vendor/resolvelib/resolvers.py new file mode 100644 index 0000000000..9c69628e88 --- /dev/null +++ b/pipenv/vendor/resolvelib/resolvers.py @@ -0,0 +1,287 @@ +import collections + +from .structs import DirectedGraph + + +RequirementInformation = collections.namedtuple('RequirementInformation', [ + 'requirement', 'parent', +]) + + +class NoVersionsAvailable(Exception): + def __init__(self, requirement, parent): + super(NoVersionsAvailable, self).__init__() + self.requirement = requirement + self.parent = parent + + +class RequirementsConflicted(Exception): + def __init__(self, criterion): + super(RequirementsConflicted, self).__init__() + self.criterion = criterion + + +class Criterion(object): + """Internal representation of possible resolution results of a package. + + This holds two attributes: + + * `information` is a collection of `RequirementInformation` pairs. Each + pair is a requirement contributing to this criterion, and the candidate + that provides the requirement. + * `candidates` is a collection containing all possible candidates deducted + from the union of contributing requirements. It should never be empty. + """ + def __init__(self, candidates, information): + self.candidates = candidates + self.information = information + + @classmethod + def from_requirement(cls, provider, requirement, parent): + """Build an instance from a requirement. + """ + candidates = provider.find_matches(requirement) + if not candidates: + raise NoVersionsAvailable(requirement, parent) + return cls( + candidates=candidates, + information=[RequirementInformation(requirement, parent)], + ) + + def iter_requirement(self): + return (i.requirement for i in self.information) + + def iter_parent(self): + return (i.parent for i in self.information) + + def merged_with(self, provider, requirement, parent): + """Build a new instance from this and a new requirement. + """ + infos = list(self.information) + infos.append(RequirementInformation(requirement, parent)) + candidates = [ + c for c in self.candidates + if provider.is_satisfied_by(requirement, c) + ] + if not candidates: + raise RequirementsConflicted(self) + return type(self)(candidates, infos) + + +class ResolutionError(Exception): + pass + + +class ResolutionImpossible(ResolutionError): + def __init__(self, requirements): + super(ResolutionImpossible, self).__init__() + self.requirements = requirements + + +class ResolutionTooDeep(ResolutionError): + def __init__(self, round_count): + super(ResolutionTooDeep, self).__init__(round_count) + self.round_count = round_count + + +# Resolution state in a round. +State = collections.namedtuple('State', 'mapping graph') + + +class Resolution(object): + """Stateful resolution object. + + This is designed as a one-off object that holds information to kick start + the resolution process, and holds the results afterwards. + """ + def __init__(self, provider, reporter): + self._p = provider + self._r = reporter + self._criteria = {} + self._states = [] + + @property + def state(self): + try: + return self._states[-1] + except IndexError: + raise AttributeError('state') + + def _push_new_state(self): + """Push a new state into history. + + This new state will be used to hold resolution results of the next + coming round. + """ + try: + base = self._states[-1] + except IndexError: + graph = DirectedGraph() + graph.add(None) # Sentinel as root dependencies' parent. + state = State(mapping={}, graph=graph) + else: + state = State( + mapping=base.mapping.copy(), + graph=base.graph.copy(), + ) + self._states.append(state) + + def _contribute_to_criteria(self, name, requirement, parent): + try: + crit = self._criteria[name] + except KeyError: + crit = Criterion.from_requirement(self._p, requirement, parent) + else: + crit = crit.merged_with(self._p, requirement, parent) + self._criteria[name] = crit + + def _get_criterion_item_preference(self, item): + name, criterion = item + try: + pinned = self.state.mapping[name] + except (IndexError, KeyError): + pinned = None + return self._p.get_preference( + pinned, criterion.candidates, criterion.information, + ) + + def _is_current_pin_satisfying(self, name, criterion): + try: + current_pin = self.state.mapping[name] + except KeyError: + return False + return all( + self._p.is_satisfied_by(r, current_pin) + for r in criterion.iter_requirement() + ) + + def _check_pinnability(self, candidate, dependencies): + backup = self._criteria.copy() + contributed = set() + try: + for subdep in dependencies: + key = self._p.identify(subdep) + self._contribute_to_criteria(key, subdep, parent=candidate) + contributed.add(key) + except RequirementsConflicted: + self._criteria = backup + return None + return contributed + + def _pin_candidate(self, name, criterion, candidate, child_names): + try: + self.state.graph.remove(name) + except KeyError: + pass + self.state.mapping[name] = candidate + self.state.graph.add(name) + for parent in criterion.iter_parent(): + parent_name = None if parent is None else self._p.identify(parent) + try: + self.state.graph.connect(parent_name, name) + except KeyError: + # Parent is not yet pinned. Skip now; this edge will be + # connected when the parent is being pinned. + pass + for child_name in child_names: + try: + self.state.graph.connect(name, child_name) + except KeyError: + # Child is not yet pinned. Skip now; this edge will be + # connected when the child is being pinned. + pass + + def _pin_criteria(self): + criterion_names = [name for name, _ in sorted( + self._criteria.items(), + key=self._get_criterion_item_preference, + )] + for name in criterion_names: + # Any pin may modify any criterion during the loop. Criteria are + # replaced, not updated in-place, so we need to read this value + # in the loop instead of outside. (sarugaku/resolvelib#5) + criterion = self._criteria[name] + + if self._is_current_pin_satisfying(name, criterion): + # If the current pin already works, just use it. + continue + candidates = list(criterion.candidates) + while candidates: + candidate = candidates.pop() + dependencies = self._p.get_dependencies(candidate) + child_names = self._check_pinnability(candidate, dependencies) + if child_names is None: + continue + self._pin_candidate(name, criterion, candidate, child_names) + break + else: # All candidates tried, nothing works. Give up. (?) + raise ResolutionImpossible(list(criterion.iter_requirement())) + + def resolve(self, requirements, max_rounds): + if self._states: + raise RuntimeError('already resolved') + + for requirement in requirements: + try: + name = self._p.identify(requirement) + self._contribute_to_criteria(name, requirement, parent=None) + except RequirementsConflicted as e: + # If initial requirements conflict, nothing would ever work. + raise ResolutionImpossible(e.requirements + [requirement]) + + last = None + self._r.starting() + + for round_index in range(max_rounds): + self._r.starting_round(round_index) + + self._push_new_state() + self._pin_criteria() + + curr = self.state + if last is not None and len(curr.mapping) == len(last.mapping): + # Nothing new added. Done! Remove the duplicated entry. + del self._states[-1] + self._r.ending(last) + return + last = curr + + self._r.ending_round(round_index, curr) + + raise ResolutionTooDeep(max_rounds) + + +class Resolver(object): + """The thing that performs the actual resolution work. + """ + def __init__(self, provider, reporter): + self.provider = provider + self.reporter = reporter + + def resolve(self, requirements, max_rounds=20): + """Take a collection of constraints, spit out the resolution result. + + The return value is a representation to the final resolution result. It + is a tuple subclass with two public members: + + * `mapping`: A dict of resolved candidates. Each key is an identifier + of a requirement (as returned by the provider's `identify` method), + and the value is the resolved candidate. + * `graph`: A `DirectedGraph` instance representing the dependency tree. + The vertices are keys of `mapping`, and each edge represents *why* + a particular package is included. A special vertex `None` is + included to represent parents of user-supplied requirements. + + The following exceptions may be raised if a resolution cannot be found: + + * `NoVersionsAvailable`: A requirement has no available candidates. + * `ResolutionImpossible`: A resolution cannot be found for the given + combination of requirements. + * `ResolutionTooDeep`: The dependency tree is too deeply nested and + the resolver gave up. This is usually caused by a circular + dependency, but you can try to resolve this by increasing the + `max_rounds` argument. + """ + resolution = Resolution(self.provider, self.reporter) + resolution.resolve(requirements, max_rounds=max_rounds) + return resolution.state diff --git a/pipenv/vendor/resolvelib/structs.py b/pipenv/vendor/resolvelib/structs.py new file mode 100644 index 0000000000..97bd009507 --- /dev/null +++ b/pipenv/vendor/resolvelib/structs.py @@ -0,0 +1,67 @@ +class DirectedGraph(object): + """A graph structure with directed edges. + """ + def __init__(self): + self._vertices = set() + self._forwards = {} # -> Set[] + self._backwards = {} # -> Set[] + + def __iter__(self): + return iter(self._vertices) + + def __len__(self): + return len(self._vertices) + + def __contains__(self, key): + return key in self._vertices + + def copy(self): + """Return a shallow copy of this graph. + """ + other = DirectedGraph() + other._vertices = set(self._vertices) + other._forwards = {k: set(v) for k, v in self._forwards.items()} + other._backwards = {k: set(v) for k, v in self._backwards.items()} + return other + + def add(self, key): + """Add a new vertex to the graph. + """ + if key in self._vertices: + raise ValueError('vertex exists') + self._vertices.add(key) + self._forwards[key] = set() + self._backwards[key] = set() + + def remove(self, key): + """Remove a vertex from the graph, disconnecting all edges from/to it. + """ + self._vertices.remove(key) + for f in self._forwards.pop(key): + self._backwards[f].remove(key) + for t in self._backwards.pop(key): + self._forwards[t].remove(key) + + def connected(self, f, t): + return f in self._backwards[t] and t in self._forwards[f] + + def connect(self, f, t): + """Connect two existing vertices. + + Nothing happens if the vertices are already connected. + """ + if t not in self._vertices: + raise KeyError(t) + self._forwards[f].add(t) + self._backwards[t].add(f) + + def iter_edges(self): + for f, children in self._forwards.items(): + for t in children: + yield f, t + + def iter_children(self, key): + return iter(self._forwards[key]) + + def iter_parents(self, key): + return iter(self._backwards[key]) diff --git a/pipenv/vendor/shutil_backports/__init__.py b/pipenv/vendor/shutil_backports/__init__.py new file mode 100644 index 0000000000..fa12816e27 --- /dev/null +++ b/pipenv/vendor/shutil_backports/__init__.py @@ -0,0 +1,9 @@ +__title__ = "shutil_backports" +__version__ = "0.1.0" +__license__ = "MIT" +__author__ = "Christopher Rosell" +__copyright__ = "Copyright 2014 Christopher Rosell" + +__all__ = ["get_terminal_size"] + +from .get_terminal_size import * diff --git a/pipenv/vendor/shutil_backports/get_terminal_size.py b/pipenv/vendor/shutil_backports/get_terminal_size.py new file mode 100644 index 0000000000..f1336e5839 --- /dev/null +++ b/pipenv/vendor/shutil_backports/get_terminal_size.py @@ -0,0 +1,100 @@ +"""This is a backport of shutil.get_terminal_size from Python 3.3. + +The original implementation is in C, but here we use the ctypes and +fcntl modules to create a pure Python version of os.get_terminal_size. +""" + +import os +import struct +import sys + +from collections import namedtuple + +__all__ = ["get_terminal_size"] + + +terminal_size = namedtuple("terminal_size", "columns lines") + +try: + from ctypes import windll, create_string_buffer + + _handles = { + 0: windll.kernel32.GetStdHandle(-10), + 1: windll.kernel32.GetStdHandle(-11), + 2: windll.kernel32.GetStdHandle(-12), + } + + def _get_terminal_size(fd): + columns = lines = 0 + + try: + handle = _handles[fd] + csbi = create_string_buffer(22) + res = windll.kernel32.GetConsoleScreenBufferInfo(handle, csbi) + if res: + res = struct.unpack("hhhhHhhhhhh", csbi.raw) + left, top, right, bottom = res[5:9] + columns = right - left + 1 + lines = bottom - top + 1 + except Exception: + pass + + return columns, lines + +except ImportError: + import fcntl + import termios + + def _get_terminal_size(fd): + try: + res = fcntl.ioctl(fd, termios.TIOCGWINSZ, b"\x00" * 4) + lines, columns = struct.unpack("hh", res) + except Exception: + columns = lines = 0 + + return columns, lines + + +def get_terminal_size(fallback=(80, 24)): + """Get the size of the terminal window. + + For each of the two dimensions, the environment variable, COLUMNS + and LINES respectively, is checked. If the variable is defined and + the value is a positive integer, it is used. + + When COLUMNS or LINES is not defined, which is the common case, + the terminal connected to sys.__stdout__ is queried + by invoking os.get_terminal_size. + + If the terminal size cannot be successfully queried, either because + the system doesn't support querying, or because we are not + connected to a terminal, the value given in fallback parameter + is used. Fallback defaults to (80, 24) which is the default + size used by many terminal emulators. + + The value returned is a named tuple of type os.terminal_size. + """ + # Attempt to use the environment first + try: + columns = int(os.environ["COLUMNS"]) + except (KeyError, ValueError): + columns = 0 + + try: + lines = int(os.environ["LINES"]) + except (KeyError, ValueError): + lines = 0 + + # Only query if necessary + if columns <= 0 or lines <= 0: + try: + columns, lines = _get_terminal_size(sys.__stdout__.fileno()) + except (NameError, OSError): + pass + + # Use fallback as last resort + if columns <= 0 and lines <= 0: + columns, lines = fallback + + return terminal_size(columns, lines) + diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 741ae3193c..091f27d801 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,13 +21,13 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.3.post1 +pythonfinder==1.1.6 requests==2.20.0 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.0 +requirementslib==1.2.1 attrs==18.2.0 distlib==0.2.8 packaging==18.0 @@ -41,7 +41,7 @@ semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 -vistir==0.2.0 +vistir==0.2.2 pip-shims==0.3.1 ptyprocess==0.6.0 enum34==1.1.6 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index c8b253b8f4..912ab0a436 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -1,7 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals -from .compat import NamedTemporaryFile, TemporaryDirectory, partialmethod +from .compat import NamedTemporaryFile, TemporaryDirectory, partialmethod, to_native_string from .contextmanagers import ( atomic_open_for_write, cd, @@ -15,7 +15,7 @@ from .spin import VistirSpinner, create_spinner -__version__ = '0.2.0' +__version__ = '0.2.2' __all__ = [ @@ -38,4 +38,5 @@ "create_spinner", "create_tracked_tempdir", "create_tracked_tempfile", + "to_native_string" ] diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 7b8066eeed..fb044acf20 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -194,6 +194,8 @@ def NamedTemporaryFile( flags = _bin_openflags # Setting O_TEMPORARY in the flags causes the OS to delete # the file when it is closed. This is only supported by Windows. + if not wrapper_class_override: + wrapper_class_override = _TemporaryFileWrapper if os.name == "nt" and delete: flags |= os.O_TEMPORARY if sys.version_info < (3, 5): @@ -205,7 +207,9 @@ def NamedTemporaryFile( fd, mode, buffering=buffering, newline=newline, encoding=encoding ) if wrapper_class_override is not None: - return wrapper_class_override(file, name, delete) + return type( + str("_TempFileWrapper"), (wrapper_class_override, object), {} + )(file, name, delete) else: return _TemporaryFileWrapper(file, name, delete) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index ec3b65cbc6..d6e8578a18 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -24,6 +24,7 @@ "lru_cache", "TemporaryDirectory", "NamedTemporaryFile", + "to_native_string", ] if sys.version_info >= (3, 5): @@ -142,3 +143,10 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + + +def to_native_string(string): + from .misc import to_text, to_bytes + if six.PY2: + return to_bytes(string) + return to_text(string) diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index bcbf754192..4d8a31918f 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -118,7 +118,6 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): """ from .spin import create_spinner - spinner_func = create_spinner if nospin is False: try: import yaspin @@ -128,10 +127,10 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): " pip install --upgrade vistir[spinner]" ) else: - spinner_name = None + spinner_name = "" if not start_text and nospin is False: start_text = "Running..." - with spinner_func( + with create_spinner( spinner_name=spinner_name, text=start_text, handler_map=handler_map, @@ -271,8 +270,8 @@ def open_file(link, session=None, stream=True): result = raw if raw else resp yield result finally: - result.close() if raw: conn = getattr(raw, "_connection") if conn is not None: conn.close() + result.close() diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 2638241990..7d400a4cdb 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -15,7 +15,7 @@ import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod +from .compat import Path, fs_str, partialmethod, to_native_string from .contextmanagers import spinner as spinner if os.name != "nt": @@ -145,7 +145,8 @@ def _create_subprocess( verbose=False, spinner=None, combine_stderr=False, - display_limit=200 + display_limit=200, + start_text="" ): if not env: env = {} @@ -157,13 +158,13 @@ def _create_subprocess( raise if not block: c.stdin.close() - log_level = "DEBUG" if verbose else "ERROR" - logger = _get_logger(cmd._parts[0], level=log_level) output = [] err = [] - spinner_orig_text = "" + spinner_orig_text = None if spinner: - spinner_orig_text = getattr(spinner, "text", "") + spinner_orig_text = getattr(spinner, "text", None) + if spinner_orig_text is None: + spinner_orig_text = start_text if start_text is not None else "" streams = { "stdout": c.stdout, "stderr": c.stderr @@ -185,25 +186,32 @@ def _create_subprocess( stdout_line = line if not (stdout_line or stderr_line): break - if stderr_line: + if stderr_line is not None: err.append(stderr_line) - if verbose: + err_line = fs_str("{0}".format(stderr_line)) + if verbose and err_line is not None: if spinner: - spinner.write_err(fs_str(stderr_line)) + spinner._hide_cursor() + spinner.write_err(err_line) + spinner._show_cursor() else: - logger.error(stderr_line) - if stdout_line: + sys.stderr.write(err_line) + sys.stderr.flush() + if stdout_line is not None: output.append(stdout_line) - display_line = stdout_line + display_line = fs_str("{0}".format(stdout_line)) if len(stdout_line) > display_limit: display_line = "{0}...".format(stdout_line[:display_limit]) - if verbose: + if verbose and display_line is not None: if spinner: - spinner.write_err(fs_str(display_line)) + spinner._hide_cursor() + spinner.write_err(display_line) + spinner._show_cursor() else: - logger.debug(display_line) + sys.stderr.write(display_line) + sys.stderr.flush() if spinner: - spinner.text = fs_str("{0} {1}".format(spinner_orig_text, display_line)) + spinner.text = to_native_string("{0} {1}".format(spinner_orig_text, display_line)) continue try: c.wait() @@ -214,21 +222,19 @@ def _create_subprocess( c.stderr.close() if spinner: if c.returncode > 0: - spinner.fail("Failed...cleaning up...") - else: - spinner.text = "Complete!" + spinner.fail(to_native_string("Failed...cleaning up...")) if not os.name == "nt": - spinner.ok("✔") + spinner.ok(to_native_string("✔ Complete")) else: - spinner.ok() - c.out = "\n".join(output) + spinner.ok(to_native_string("Complete")) + c.out = "\n".join(output) if output else "" c.err = "\n".join(err) if err else "" else: c.out, c.err = c.communicate() if not block: c.wait() - c.out = fs_str("{0}".format(c.out)) if c.out else fs_str("") - c.err = fs_str("{0}".format(c.err)) if c.err else fs_str("") + c.out = to_text("{0}".format(c.out)) if c.out else fs_str("") + c.err = to_text("{0}".format(c.err)) if c.err else fs_str("") if not return_object: return c.out.strip(), c.err.strip() return c @@ -287,9 +293,7 @@ def run( cmd = Script.parse(cmd) if block or not return_object: combine_stderr = False - start_text = "Running..." - if nospin: - start_text = None + start_text = "" with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin) as sp: return _create_subprocess( cmd, @@ -299,7 +303,8 @@ def run( cwd=cwd, verbose=verbose, spinner=sp, - combine_stderr=combine_stderr + combine_stderr=combine_stderr, + start_text=start_text ) diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index d580aba23f..68e6d464df 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -46,7 +46,7 @@ ] -if os.name == "nt" and six.PY34: +if os.name == "nt": warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") @@ -215,7 +215,6 @@ def ensure_mkdir_p(mode=0o777): """ def decorator(f): - @functools.wraps(f) def decorated(*args, **kwargs): path = f(*args, **kwargs) @@ -316,10 +315,8 @@ def handle_remove_readonly(func, path, exc): This function will call check :func:`is_readonly_path` before attempting to call :func:`set_write_bit` on the target path and try again. """ - # Check for read-only attribute - if six.PY2: - from .compat import ResourceWarning + from .compat import ResourceWarning from .misc import to_bytes PERM_ERRORS = (errno.EACCES, errno.EPERM) @@ -460,7 +457,7 @@ def safe_expandvars(value): return value -class _TrackedTempfileWrapper(_TemporaryFileWrapper): +class _TrackedTempfileWrapper(_TemporaryFileWrapper, object): def __init__(self, *args, **kwargs): super(_TrackedTempfileWrapper, self).__init__(*args, **kwargs) self._finalizer = finalize(self, self.cleanup) diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index e4b4ba66f2..57a90277be 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -1,13 +1,18 @@ # -*- coding=utf-8 -*- + +import functools import os import signal import sys -from .termcolors import colored, COLORS -from .compat import fs_str - +import colorama import cursor -import functools +import six + +from .compat import to_native_string +from .termcolors import COLOR_MAP, COLORS, colored +from io import StringIO + try: import yaspin except ImportError: @@ -15,7 +20,6 @@ Spinners = None else: from yaspin.spinners import Spinners - from yaspin.constants import COLOR_MAP handler = None if yaspin and os.name == "nt": @@ -28,16 +32,20 @@ class DummySpinner(object): def __init__(self, text="", **kwargs): - self.text = text + colorama.init() + self.text = to_native_string(text) + self.stdout = kwargs.get("stdout", sys.stdout) + self.stderr = kwargs.get("stderr", sys.stderr) + self.out_buff = StringIO() def __enter__(self): - if self.text: - self.write(self.text) + if self.text and self.text != "None": + self.write_err(self.text) return self def __exit__(self, exc_type, exc_val, traceback): if not exc_type: - self.ok() + self.ok(text=None) else: self.write_err(traceback) return False @@ -52,25 +60,43 @@ def __getattr__(self, k): else: return retval - def fail(self, exitcode=1, text=None): - if text: + def fail(self, exitcode=1, text="FAIL"): + if text and text != "None": self.write_err(text) + if self.out_buff: + self.out_buff.close() raise SystemExit(exitcode, text) - def ok(self, text=None): - if text: - self.write(self.text) + def ok(self, text="OK"): + if text and text != "None": + self.stderr.write(self.text) + if self.out_buff: + self.out_buff.close() return 0 def write(self, text=None): - if text: - line = fs_str("{0}\n".format(text)) - sys.stdout.write(line) + if text is None or isinstance(text, six.string_types) and text == "None": + pass + self.stdout.write(to_native_string("\r")) + line = to_native_string("{0}\n".format(text)) + self.stdout.write(line) + self.stdout.write(CLEAR_LINE) def write_err(self, text=None): - if text: - line = fs_str("{0}\n".format(text)) - sys.stderr.write(line) + if text is None or isinstance(text, six.string_types) and text == "None": + pass + self.stderr.write(to_native_string("\r")) + line = to_native_string("{0}\n".format(text)) + self.stderr.write(line) + self.stderr.write(CLEAR_LINE) + + @staticmethod + def _hide_cursor(): + pass + + @staticmethod + def _show_cursor(): + pass base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner @@ -88,6 +114,7 @@ def __init__(self, *args, **kwargs): """ self.handler = handler + colorama.init() sigmap = {} if handler: sigmap.update({ @@ -102,30 +129,68 @@ def __init__(self, *args, **kwargs): if handler_map: sigmap.update(handler_map) spinner_name = kwargs.pop("spinner_name", "bouncingBar") - text = kwargs.pop("start_text", "") + " " + kwargs.pop("text", "") - if not text: - text = "Running..." + start_text = kwargs.pop("start_text", None) + _text = kwargs.pop("text", "Running...") + kwargs["text"] = start_text if start_text is not None else _text kwargs["sigmap"] = sigmap - kwargs["spinner"] = getattr(Spinners, spinner_name, Spinners.bouncingBar) + kwargs["spinner"] = getattr(Spinners, spinner_name, "") + self.stdout = kwargs.pop("stdout", sys.stdout) + self.stderr = kwargs.pop("stderr", sys.stderr) + self.out_buff = StringIO() super(VistirSpinner, self).__init__(*args, **kwargs) self.is_dummy = bool(yaspin is None) - def fail(self, exitcode=1, *args, **kwargs): - super(VistirSpinner, self).fail(**kwargs) - - def ok(self, *args, **kwargs): - super(VistirSpinner, self).ok(*args, **kwargs) - - def write(self, *args, **kwargs): - super(VistirSpinner, self).write(*args, **kwargs) + def ok(self, text="OK"): + """Set Ok (success) finalizer to a spinner.""" + _text = text if text else "OK" + self._freeze(_text) + + def fail(self, text="FAIL"): + """Set fail finalizer to a spinner.""" + _text = text if text else "FAIL" + self._freeze(_text) + + def write(self, text): + from .misc import to_text + sys.stdout.write("\r") + self.stdout.write(CLEAR_LINE) + if text is None: + text = "" + text = to_native_string("{0}\n".format(text)) + sys.stdout.write(text) + self.out_buff.write(to_text(text)) def write_err(self, text): """Write error text in the terminal without breaking the spinner.""" - - sys.stderr.write("\r") - self._clear_err() - text = fs_str("{0}\n".format(text)) - sys.stderr.write(text) + from .misc import to_text + + self.stderr.write("\r") + self.stderr.write(CLEAR_LINE) + if text is None: + text = "" + text = to_native_string("{0}\n".format(text)) + self.stderr.write(text) + self.out_buff.write(to_text(text)) + + def _freeze(self, final_text): + """Stop spinner, compose last frame and 'freeze' it.""" + if not final_text: + final_text = "" + text = to_native_string(final_text) + self._last_frame = self._compose_out(text, mode="last") + + # Should be stopped here, otherwise prints after + # self._freeze call will mess up the spinner + self.stop() + self.stdout.write(self._last_frame) + + def stop(self, *args, **kwargs): + if self.stderr and self.stderr != sys.stderr: + self.stderr.close() + if self.stdout and self.stdout != sys.stdout: + self.stdout.close() + self.out_buff.close() + super(VistirSpinner, self).stop(*args, **kwargs) def _compose_color_func(self): fn = functools.partial( @@ -136,6 +201,24 @@ def _compose_color_func(self): ) return fn + def _compose_out(self, frame, mode=None): + # Ensure Unicode input + + frame = to_native_string(frame) + if self._text is None: + self._text = "" + text = to_native_string(self._text) + if self._color_func is not None: + frame = self._color_func(frame) + if self._side == "right": + frame, text = text, frame + # Mode + if not mode: + out = to_native_string("\r{0} {1}".format(frame, text)) + else: + out = to_native_string("{0} {1}\n".format(frame, text)) + return out + def _register_signal_handlers(self): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this @@ -191,6 +274,7 @@ def _clear_line(): def create_spinner(*args, **kwargs): nospin = kwargs.pop("nospin", False) + use_yaspin = kwargs.pop("use_yaspin", nospin) if nospin: return DummySpinner(*args, **kwargs) return VistirSpinner(*args, **kwargs) diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py index b8ccda8e48..8395d97d05 100644 --- a/pipenv/vendor/vistir/termcolors.py +++ b/pipenv/vendor/vistir/termcolors.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function, unicode_literals import colorama import os +from .compat import to_native_string ATTRIBUTES = dict( @@ -53,6 +54,33 @@ ) +COLOR_MAP = { + # name: type + "blink": "attrs", + "bold": "attrs", + "concealed": "attrs", + "dark": "attrs", + "reverse": "attrs", + "underline": "attrs", + "blue": "color", + "cyan": "color", + "green": "color", + "magenta": "color", + "red": "color", + "white": "color", + "yellow": "color", + "on_blue": "on_color", + "on_cyan": "on_color", + "on_green": "on_color", + "on_grey": "on_color", + "on_magenta": "on_color", + "on_red": "on_color", + "on_white": "on_color", + "on_yellow": "on_color", +} +COLOR_ATTRS = COLOR_MAP.keys() + + RESET = colorama.Style.RESET_ALL @@ -80,25 +108,25 @@ def colored(text, color=None, on_color=None, attrs=None): attrs.remove('bold') if color is not None: color = color.upper() - text = text = "%s%s%s%s%s" % ( - getattr(colorama.Fore, color), - getattr(colorama.Style, style), - text, - colorama.Fore.RESET, - colorama.Style.NORMAL, + text = to_native_string("%s%s%s%s%s") % ( + to_native_string(getattr(colorama.Fore, color)), + to_native_string(getattr(colorama.Style, style)), + to_native_string(text), + to_native_string(colorama.Fore.RESET), + to_native_string(colorama.Style.NORMAL), ) if on_color is not None: on_color = on_color.upper() - text = "%s%s%s%s" % ( - getattr(colorama.Back, on_color), - text, - colorama.Back.RESET, - colorama.Style.NORMAL, + text = to_native_string("%s%s%s%s") % ( + to_native_string(getattr(colorama.Back, on_color)), + to_native_string(text), + to_native_string(colorama.Back.RESET), + to_native_string(colorama.Style.NORMAL), ) if attrs is not None: - fmt_str = "%s[%%dm%%s%s[9m" % ( + fmt_str = to_native_string("%s[%%dm%%s%s[9m") % ( chr(27), chr(27) ) diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index 06b8b62116..d01fb98ef1 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -16,9 +16,6 @@ import threading import time -import colorama -import cursor - from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS @@ -26,9 +23,6 @@ from .termcolor import colored -colorama.init() - - class Yaspin(object): """Implements a context manager that spawns a thread to write spinner frames into a tty (stdout) during @@ -375,14 +369,11 @@ def _register_signal_handlers(self): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this # signal. - try: - if signal.SIGKILL in self._sigmap.keys(): - raise ValueError( - "Trying to set handler for SIGKILL signal. " - "SIGKILL cannot be cought or ignored in POSIX systems." - ) - except AttributeError: - pass + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) for sig, sig_handler in iteritems(self._sigmap): # A handler for a particular signal, once set, remains @@ -530,12 +521,14 @@ def _set_text(text): @staticmethod def _hide_cursor(): - cursor.hide() + sys.stdout.write("\033[?25l") + sys.stdout.flush() @staticmethod def _show_cursor(): - cursor.show() + sys.stdout.write("\033[?25h") + sys.stdout.flush() @staticmethod def _clear_line(): - sys.stdout.write(chr(27) + "[K") + sys.stdout.write("\033[K") diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch index 175efaa16d..ae0bf088b0 100644 --- a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -1,8 +1,8 @@ -diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py -index d15aeb97..56d12458 100644 ---- a/pipenv/vendor/delegator.py -+++ b/pipenv/vendor/delegator.py -@@ -7,6 +7,8 @@ import locale +diff --git a/delegator.py b/delegator.py +index 25d21f0..582f4fe 100644 +--- a/delegator.py ++++ b/delegator.py +@@ -7,15 +7,18 @@ import locale import errno from pexpect.popen_spawn import PopenSpawn @@ -11,26 +11,150 @@ index d15aeb97..56d12458 100644 # Include `unicode` in STR_TYPES for Python 2.X try: -@@ -110,7 +112,7 @@ class Command(object): + STR_TYPES = (str, unicode) + except NameError: +- STR_TYPES = (str, ) ++ STR_TYPES = (str,) + + TIMEOUT = 30 + ++ + def pid_exists(pid): + """Check whether pid exists in the current process table.""" + if pid == 0: +@@ -43,8 +46,8 @@ def pid_exists(pid): + else: + return True + +-class Command(object): + ++class Command(object): + def __init__(self, cmd, timeout=TIMEOUT): + super(Command, self).__init__() + self.cmd = cmd +@@ -56,7 +59,7 @@ class Command(object): + self.__err = None + + def __repr__(self): +- return ''.format(self.cmd) ++ return "".format(self.cmd) + + @property + def _popen_args(self): +@@ -65,27 +68,23 @@ class Command(object): + @property + def _default_popen_kwargs(self): + return { +- 'env': os.environ.copy(), +- 'stdin': subprocess.PIPE, +- 'stdout': subprocess.PIPE, +- 'stderr': subprocess.PIPE, +- 'shell': True, +- 'universal_newlines': True, +- 'bufsize': 0 ++ "env": os.environ.copy(), ++ "stdin": subprocess.PIPE, ++ "stdout": subprocess.PIPE, ++ "stderr": subprocess.PIPE, ++ "shell": True, ++ "universal_newlines": True, ++ "bufsize": 0, + } + + @property + def _default_pexpect_kwargs(self): +- encoding = 'utf-8' +- if sys.platform == 'win32': ++ encoding = "utf-8" ++ if sys.platform == "win32": + default_encoding = locale.getdefaultlocale()[1] + if default_encoding is not None: + encoding = default_encoding +- return { +- 'env': os.environ.copy(), +- 'encoding': encoding, +- 'timeout': self.timeout +- } ++ return {"env": os.environ.copy(), "encoding": encoding, "timeout": self.timeout} + + @property + def _uses_subprocess(self): +@@ -99,18 +98,25 @@ class Command(object): + def std_out(self): + return self.subprocess.stdout + ++ @property ++ def ok(self): ++ return self.return_code == 0 ++ + @property + def _pexpect_out(self): + if self.subprocess.encoding: +- result = '' ++ result = "" + else: +- result = b'' ++ result = b"" + if self.subprocess.before: result += self.subprocess.before -- if self.subprocess.after: -+ if self.subprocess.after and self.subprocess.after is not pexpect.EOF: - result += self.subprocess.after + if self.subprocess.after: +- result += self.subprocess.after ++ try: ++ result += self.subprocess.after ++ except (pexpect.EOF, pexpect.TIMEOUT): ++ pass result += self.subprocess.read() -@@ -178,6 +180,7 @@ class Command(object): + return result +@@ -148,7 +154,7 @@ class Command(object): + def pid(self): + """The process' PID.""" + # Support for pexpect's functionality. +- if hasattr(self.subprocess, 'proc'): ++ if hasattr(self.subprocess, "proc"): + return self.subprocess.proc.pid + # Standard subprocess method. + return self.subprocess.pid +@@ -177,23 +183,24 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() +- popen_kwargs['universal_newlines'] = not binary + del popen_kwargs["stdin"] - popen_kwargs["universal_newlines"] = not binary ++ popen_kwargs["universal_newlines"] = not binary + if cwd: +- popen_kwargs['cwd'] = cwd ++ popen_kwargs["cwd"] = cwd + if env: +- popen_kwargs['env'].update(env) ++ popen_kwargs["env"].update(env) + s = subprocess.Popen(self._popen_args, **popen_kwargs) + # Otherwise, use pexpect. + else: + pexpect_kwargs = self._default_pexpect_kwargs.copy() + if binary: +- pexpect_kwargs['encoding'] = None ++ pexpect_kwargs["encoding"] = None if cwd: - popen_kwargs["cwd"] = cwd -@@ -205,7 +208,10 @@ class Command(object): +- pexpect_kwargs['cwd'] = cwd ++ pexpect_kwargs["cwd"] = cwd + if env: +- pexpect_kwargs['env'].update(env) ++ pexpect_kwargs["env"].update(env) + # Enable Python subprocesses to work with expect functionality. +- pexpect_kwargs['env']['PYTHONUNBUFFERED'] = '1' ++ pexpect_kwargs["env"]["PYTHONUNBUFFERED"] = "1" + s = PopenSpawn(self._popen_args, **pexpect_kwargs) + self.subprocess = s + self.was_run = True +@@ -202,15 +209,18 @@ class Command(object): + """Waits on the given pattern to appear in std_out""" + if self.blocking: - raise RuntimeError("expect can only be used on non-blocking commands.") +- raise RuntimeError('expect can only be used on non-blocking commands.') ++ raise RuntimeError("expect can only be used on non-blocking commands.") - self.subprocess.expect(pattern=pattern, timeout=timeout) + try: @@ -40,7 +164,14 @@ index d15aeb97..56d12458 100644 def send(self, s, end=os.linesep, signal=False): """Sends the given string or signal to std_in.""" -@@ -234,14 +240,25 @@ class Command(object): + + if self.blocking: +- raise RuntimeError('send can only be used on non-blocking commands.') ++ raise RuntimeError("send can only be used on non-blocking commands.") + + if not signal: + if self._uses_subprocess: +@@ -233,14 +243,25 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: # consume stdout and stderr @@ -73,7 +204,7 @@ index d15aeb97..56d12458 100644 def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next -@@ -263,7 +280,6 @@ class Command(object): +@@ -262,7 +283,6 @@ class Command(object): c.run(block=False, cwd=cwd) if data: c.send(data) @@ -81,3 +212,24 @@ index d15aeb97..56d12458 100644 c.block() return c +@@ -273,12 +293,12 @@ def _expand_args(command): + # Prepare arguments. + if isinstance(command, STR_TYPES): + if sys.version_info[0] == 2: +- splitter = shlex.shlex(command.encode('utf-8')) ++ splitter = shlex.shlex(command.encode("utf-8")) + elif sys.version_info[0] == 3: + splitter = shlex.shlex(command) + else: +- splitter = shlex.shlex(command.encode('utf-8')) +- splitter.whitespace = '|' ++ splitter = shlex.shlex(command.encode("utf-8")) ++ splitter.whitespace = "|" + splitter.whitespace_split = True + command = [] + +@@ -319,4 +339,3 @@ def run(command, block=True, binary=False, timeout=TIMEOUT, cwd=None, env=None): + c.block() + + return c +- diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7f797f9905..5f5c193019 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -10,10 +10,11 @@ from pipenv.vendor import requests from pipenv.vendor import toml from pytest_pypi.app import prepare_packages as prepare_pypi_packages -from vistir.compat import ResourceWarning +from vistir.compat import ResourceWarning, fs_str +from vistir.path import mkdir_p -warnings.filterwarnings("default", category=ResourceWarning) +warnings.simplefilter("default", category=ResourceWarning) HAS_WARNED_GITHUB = False @@ -85,83 +86,32 @@ def isolate(pathlib_tmpdir): We use an autouse function scoped fixture because we want to ensure that every test has it's own isolated home directory. """ - warnings.filterwarnings("ignore", category=ResourceWarning) - warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") - # Create a directory to use as our home location. home_dir = os.path.join(str(pathlib_tmpdir), "home") - os.environ["PIPENV_NOSPIN"] = "1" os.makedirs(home_dir) - - # Create a directory to use as a fake root - fake_root = os.path.join(str(pathlib_tmpdir), "fake-root") - os.makedirs(fake_root) - - # if sys.platform == 'win32': - # # Note: this will only take effect in subprocesses... - # home_drive, home_path = os.path.splitdrive(home_dir) - # os.environ.update({ - # 'USERPROFILE': home_dir, - # 'HOMEDRIVE': home_drive, - # 'HOMEPATH': home_path, - # }) - # for env_var, sub_path in ( - # ('APPDATA', 'AppData/Roaming'), - # ('LOCALAPPDATA', 'AppData/Local'), - # ): - # path = os.path.join(home_dir, *sub_path.split('/')) - # os.environ[env_var] = path - # os.makedirs(path) - # else: - # # Set our home directory to our temporary directory, this should force - # # all of our relative configuration files to be read from here instead - # # of the user's actual $HOME directory. - # os.environ["HOME"] = home_dir - # # Isolate ourselves from XDG directories - # os.environ["XDG_DATA_HOME"] = os.path.join(home_dir, ".local", "share") - # os.environ["XDG_CONFIG_HOME"] = os.path.join(home_dir, ".config") - # os.environ["XDG_CACHE_HOME"] = os.path.join(home_dir, ".cache") - # os.environ["XDG_RUNTIME_DIR"] = os.path.join(home_dir, ".runtime") - # os.environ["XDG_DATA_DIRS"] = ":".join([ - # os.path.join(fake_root, "usr", "local", "share"), - # os.path.join(fake_root, "usr", "share"), - # ]) - # os.environ["XDG_CONFIG_DIRS"] = os.path.join(fake_root, "etc", "xdg") - - # Configure git, because without an author name/email git will complain - # and cause test failures. - os.environ["GIT_CONFIG_NOSYSTEM"] = "1" - os.environ["GIT_AUTHOR_NAME"] = "pipenv" - os.environ["GIT_AUTHOR_EMAIL"] = "pipenv@pipenv.org" - - # We want to disable the version check from running in the tests - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" - workon_home = os.path.join(home_dir, ".virtualenvs") - os.makedirs(workon_home) - os.environ["WORKON_HOME"] = workon_home - project_dir = os.path.join(home_dir, "pipenv_project") - os.makedirs(project_dir) - os.environ["PIPENV_PROJECT_DIR"] = project_dir - os.environ["CI"] = "1" - - # Make sure tests don't share a requirements tracker. - os.environ.pop('PIP_REQ_TRACKER', None) - - # FIXME: Windows... - os.makedirs(os.path.join(home_dir, ".config", "git")) + mkdir_p(os.path.join(home_dir, ".config", "git")) with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: fp.write( b"[user]\n\tname = pipenv\n\temail = pipenv@pipenv.org\n" ) + os.environ["GIT_CONFIG_NOSYSTEM"] = fs_str("1") + os.environ["GIT_AUTHOR_NAME"] = fs_str("pipenv") + os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") + mkdir_p(os.path.join(home_dir, ".virtualenvs")) + os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False, path=None): + def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): self.pypi = pypi self.original_umask = os.umask(0o007) self.original_dir = os.path.abspath(os.curdir) + os.environ["PIPENV_NOSPIN"] = fs_str("1") + os.environ["CI"] = fs_str("1") + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") path = os.environ.get("PIPENV_PROJECT_DIR", None) if not path: self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') @@ -178,7 +128,7 @@ def __init__(self, pypi=None, pipfile=True, chdir=False, path=None): self.chdir = chdir if self.pypi: - os.environ['PIPENV_TEST_INDEX'] = '{0}/simple'.format(self.pypi.url) + os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url)) if pipfile: p_path = os.sep.join([self.path, 'Pipfile']) @@ -189,10 +139,10 @@ def __init__(self, pypi=None, pipfile=True, chdir=False, path=None): self.pipfile_path = p_path def __enter__(self): - os.environ['PIPENV_DONT_USE_PYENV'] = '1' - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' - os.environ['PIPENV_VENV_IN_PROJECT'] = '1' - os.environ['PIPENV_NOSPIN'] = '1' + os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') + os.environ['PIPENV_IGNORE_VIRTUALENVS'] = fs_str('1') + os.environ['PIPENV_VENV_IN_PROJECT'] = fs_str('1') + os.environ['PIPENV_NOSPIN'] = fs_str('1') if self.chdir: os.chdir(self.path) return self @@ -212,11 +162,11 @@ def __exit__(self, *args): def pipenv(self, cmd, block=True): if self.pipfile_path: - os.environ['PIPENV_PIPFILE'] = self.pipfile_path + os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path) # a bit of a hack to make sure the virtualenv is created with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir: - os.environ['PIPENV_CACHE_DIR'] = tempdir.name + os.environ['PIPENV_CACHE_DIR'] = fs_str(tempdir.name) c = delegator.run('pipenv {0}'.format(cmd), block=block) if 'PIPENV_CACHE_DIR' in os.environ: del os.environ['PIPENV_CACHE_DIR'] @@ -261,11 +211,11 @@ def PipenvInstance(): def pip_src_dir(request): old_src_dir = os.environ.get('PIP_SRC', '') new_src_dir = TemporaryDirectory(prefix='pipenv-', suffix='-testsrc') - os.environ['PIP_SRC'] = new_src_dir.name + os.environ['PIP_SRC'] = fs_str(new_src_dir.name) def finalize(): new_src_dir.cleanup() - os.environ['PIP_SRC'] = old_src_dir + os.environ['PIP_SRC'] = fs_str(old_src_dir) request.addfinalizer(finalize) return request From 0135ab65718ba9b3e5af7781a1836e318c994e43 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 26 Oct 2018 18:50:27 -0400 Subject: [PATCH 06/13] Fix syntax issues in vsts Signed-off-by: Dan Ryan Fix syntax issues in vsts Signed-off-by: Dan Ryan --- .vsts-ci/steps/run-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index d893ccf934..b7053baeff 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -3,13 +3,13 @@ steps: # Fix Git SSL errors pip install certifi python -m certifi > cacert.txt - Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]"$(Get-Content cacert.txt)" + Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail # https://bugs.python.org/issue18199 subst T: "$env:TEMP" - Write-Host "##vso[task.setvariable variable=TEMP]"T:\" - Write-Host "##vso[task.setvariable variable=TMP]"T:\" - Get-ChildItem Env + Write-Host "##vso[task.setvariable variable=TEMP]T:\" + Write-Host "##vso[task.setvariable variable=TMP]T:\" + Get-ChildItem Env: D:\.venv\Scripts\pipenv run pytest -n 4 -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests From 2803538701ec23d47d2aea85beacc9c8d6581fd8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 26 Oct 2018 19:06:39 -0400 Subject: [PATCH 07/13] Set environment before calling which Signed-off-by: Dan Ryan --- .vsts-ci/phases/run-tests.yml | 2 +- .vsts-ci/steps/run-tests.yml | 2 ++ pipenv/core.py | 5 ++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.vsts-ci/phases/run-tests.yml b/.vsts-ci/phases/run-tests.yml index daf23e6a8c..fe35ad8dd8 100644 --- a/.vsts-ci/phases/run-tests.yml +++ b/.vsts-ci/phases/run-tests.yml @@ -14,7 +14,7 @@ steps: export GIT_SSL_CAINFO="$(python -m certifi)" export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" - echo "Path: $PATH" + echo "Path $PATH" echo "Installing Pipenv…" pip install -e "$(pwd)" --upgrade pipenv install --deploy --dev diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index b7053baeff..c7aef0df7b 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -8,7 +8,9 @@ steps: # https://bugs.python.org/issue18199 subst T: "$env:TEMP" Write-Host "##vso[task.setvariable variable=TEMP]T:\" + $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" + $env:TEMP='T:\' Get-ChildItem Env: D:\.venv\Scripts\pipenv run pytest -n 4 -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests diff --git a/pipenv/core.py b/pipenv/core.py index 64f486d71e..b776f736cb 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1520,6 +1520,10 @@ def which_pip(allow_global=False): def system_which(command, mult=False): """Emulates the system's which. Returns None if not found.""" _which = "which -a" if not os.name == "nt" else "where" + os.environ = { + vistir.compat.fs_str(k): vistir.compat.fs_str(val) + for k, val in os.environ.items() + } c = delegator.run("{0} {1}".format(_which, command)) try: # Which Not found… @@ -2223,7 +2227,6 @@ def _launch_windows_subprocess(script): def do_run_nt(script): - os.environ = {k: vistir.compat.fs_str(val) for k, val in os.environ.items()} p = _launch_windows_subprocess(script) p.communicate() sys.exit(p.returncode) From b8db36ebdf2733e13282f6b3d0eaa4b045a90258 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 26 Oct 2018 19:10:08 -0400 Subject: [PATCH 08/13] Fix error handling with fallback in `which` implementation Signed-off-by: Dan Ryan --- pipenv/core.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index b776f736cb..bb415317cb 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1524,22 +1524,30 @@ def system_which(command, mult=False): vistir.compat.fs_str(k): vistir.compat.fs_str(val) for k, val in os.environ.items() } - c = delegator.run("{0} {1}".format(_which, command)) try: - # Which Not found… - if c.return_code == 127: - click.echo( - "{}: the {} system utility is required for Pipenv to find Python installations properly." - "\n Please install it.".format( - crayons.red("Warning", bold=True), crayons.red(_which) - ), - err=True, - ) - assert c.return_code == 0 - except AssertionError: - return None if not mult else [] - - result = c.out.strip() or c.err.strip() + c = delegator.run("{0} {1}".format(_which, command)) + try: + # Which Not found… + if c.return_code == 127: + click.echo( + "{}: the {} system utility is required for Pipenv to find Python installations properly." + "\n Please install it.".format( + crayons.red("Warning", bold=True), crayons.red(_which) + ), + err=True, + ) + assert c.return_code == 0 + except AssertionError: + return None if not mult else[] + except TypeError: + from .vendor.pythonfinder import Finder + finder = Finder() + result = finder.which(command) + if result: + return result.path.as_posix() + return + else: + result = c.out.strip() or c.err.strip() if mult: return result.split("\n") From 0cefb5ed9bd89dc89964faeb918f94ae5df4b218 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 26 Oct 2018 20:47:06 -0400 Subject: [PATCH 09/13] Finish updating vendored deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dan Ryan Update delegator patch Signed-off-by: Dan Ryan Update patches for pip Signed-off-by: Dan Ryan Update vendored deps Signed-off-by: Dan Ryan Update patches Signed-off-by: Dan Ryan Fix imports to use pip shims instead of direct imports Signed-off-by: Dan Ryan Update vendoring scripts and pip shims Signed-off-by: Dan Ryan Log to stdout in real time during verbose logging Signed-off-by: Dan Ryan Don’t log environment Fix unicode decoding issues Signed-off-by: Dan Ryan Only set buffers on ttys Signed-off-by: Dan Ryan Fix typo Signed-off-by: Dan Ryan Use default encodings Signed-off-by: Dan Ryan Fix encodings and run only failing tests Signed-off-by: Dan Ryan --- .vsts-ci/steps/run-tests.yml | 4 +- pipenv/__init__.py | 20 +- pipenv/_compat.py | 76 +++ pipenv/core.py | 19 +- pipenv/patched/notpip/__init__.py | 2 +- pipenv/patched/notpip/_internal/__init__.py | 244 +------- pipenv/patched/notpip/_internal/build_env.py | 16 + pipenv/patched/notpip/_internal/cache.py | 12 +- .../patched/notpip/_internal/cli/__init__.py | 4 + .../notpip/_internal/cli/autocompletion.py | 152 +++++ .../{basecommand.py => cli/base_command.py} | 32 +- .../notpip/_internal/{ => cli}/cmdoptions.py | 117 +++- .../notpip/_internal/cli/main_parser.py | 96 +++ .../{baseparser.py => cli/parser.py} | 29 +- .../_internal/{ => cli}/status_codes.py | 0 .../notpip/_internal/commands/__init__.py | 2 +- .../notpip/_internal/commands/check.py | 2 +- .../notpip/_internal/commands/completion.py | 2 +- .../_internal/commands/configuration.py | 4 +- .../notpip/_internal/commands/download.py | 76 +-- .../notpip/_internal/commands/freeze.py | 8 +- .../patched/notpip/_internal/commands/hash.py | 4 +- .../patched/notpip/_internal/commands/help.py | 3 +- .../notpip/_internal/commands/install.py | 29 +- .../patched/notpip/_internal/commands/list.py | 8 +- .../notpip/_internal/commands/search.py | 6 +- .../patched/notpip/_internal/commands/show.py | 4 +- .../notpip/_internal/commands/uninstall.py | 7 +- .../notpip/_internal/commands/wheel.py | 4 +- .../patched/notpip/_internal/configuration.py | 19 +- pipenv/patched/notpip/_internal/download.py | 4 +- pipenv/patched/notpip/_internal/exceptions.py | 19 + pipenv/patched/notpip/_internal/index.py | 566 ++++++------------ pipenv/patched/notpip/_internal/locations.py | 2 +- .../notpip/_internal/models/candidate.py | 24 + .../notpip/_internal/models/format_control.py | 62 ++ .../patched/notpip/_internal/models/index.py | 26 +- .../patched/notpip/_internal/models/link.py | 141 +++++ .../notpip/_internal/operations/freeze.py | 33 +- .../notpip/_internal/operations/prepare.py | 48 +- pipenv/patched/notpip/_internal/pep425tags.py | 14 +- pipenv/patched/notpip/_internal/pyproject.py | 144 +++++ .../notpip/_internal/req/constructors.py | 298 +++++++++ .../patched/notpip/_internal/req/req_file.py | 10 +- .../notpip/_internal/req/req_install.py | 413 +++---------- .../patched/notpip/_internal/req/req_set.py | 138 +++-- .../notpip/_internal/req/req_uninstall.py | 9 +- pipenv/patched/notpip/_internal/resolve.py | 13 +- .../patched/notpip/_internal/utils/appdirs.py | 2 +- .../notpip/_internal/{ => utils}/compat.py | 13 + .../notpip/_internal/utils/filesystem.py | 2 +- .../patched/notpip/_internal/utils/logging.py | 2 +- pipenv/patched/notpip/_internal/utils/misc.py | 57 +- .../patched/notpip/_internal/utils/models.py | 40 ++ .../notpip/_internal/utils/outdated.py | 25 +- .../notpip/_internal/utils/packaging.py | 26 +- pipenv/patched/notpip/_internal/utils/ui.py | 2 +- .../patched/notpip/_internal/vcs/__init__.py | 78 +-- pipenv/patched/notpip/_internal/vcs/bazaar.py | 16 +- pipenv/patched/notpip/_internal/vcs/git.py | 117 ++-- .../patched/notpip/_internal/vcs/mercurial.py | 11 +- .../notpip/_internal/vcs/subversion.py | 103 +--- pipenv/patched/notpip/_internal/wheel.py | 24 +- .../notpip/_vendor/certifi/__init__.py | 2 +- .../notpip/_vendor/certifi/__main__.py | 2 +- .../patched/notpip/_vendor/certifi/cacert.pem | 226 ++----- .../notpip/_vendor/packaging/__about__.py | 4 +- .../notpip/_vendor/packaging/requirements.py | 8 +- .../notpip/_vendor/packaging/specifiers.py | 2 +- .../notpip/_vendor/pep517/LICENSE} | 12 +- .../patched/notpip/_vendor/pep517/__init__.py | 4 + .../notpip/_vendor/pep517/_in_process.py | 182 ++++++ pipenv/patched/notpip/_vendor/pep517/check.py | 194 ++++++ .../patched/notpip/_vendor/pep517/colorlog.py | 110 ++++ .../patched/notpip/_vendor/pep517/compat.py | 23 + .../patched/notpip/_vendor/pep517/envbuild.py | 150 +++++ .../patched/notpip/_vendor/pep517/wrappers.py | 134 +++++ .../notpip/_vendor/pkg_resources/__init__.py | 67 ++- .../_vendor/pkg_resources/py31compat.py | 5 +- pipenv/patched/notpip/_vendor/pyparsing.py | 46 +- .../patched/notpip/_vendor/pytoml/parser.py | 4 +- .../notpip/_vendor/requests/__init__.py | 2 +- pipenv/patched/notpip/_vendor/vendor.txt | 11 +- pipenv/patched/patched.txt | 2 +- pipenv/patched/piptools/_compat/pip_compat.py | 88 +-- pipenv/project.py | 5 +- pipenv/resolver.py | 13 +- pipenv/utils.py | 34 +- pipenv/vendor/backports/__init__.py | 6 +- pipenv/vendor/dotenv/main.py | 7 + pipenv/vendor/modutil.LICENSE | 29 - pipenv/vendor/passa/internals/dependencies.py | 2 +- pipenv/vendor/pip_shims/__init__.py | 2 +- pipenv/vendor/pip_shims/shims.py | 4 + pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/models/path.py | 1 + pipenv/vendor/pythonfinder/models/pyenv.py | 2 + pipenv/vendor/requests/LICENSE | 2 +- pipenv/vendor/requirementslib/models/utils.py | 2 +- pipenv/vendor/shutil_backports/__init__.py | 9 - .../shutil_backports/get_terminal_size.py | 100 ---- pipenv/vendor/vendor.txt | 3 +- pipenv/vendor/vendor_pip.txt | 11 +- pipenv/vendor/vistir/contextmanagers.py | 13 +- pipenv/vendor/vistir/misc.py | 33 + pipenv/vendor/vistir/spin.py | 41 +- pipenv/vendor/yaspin/core.py | 27 +- pytest.ini | 2 +- tasks/vendoring/__init__.py | 13 +- .../patched/_post-pip-update-pep425tags.patch | 16 +- tasks/vendoring/patches/patched/pip18.patch | 263 ++++---- .../vendor/delegator-close-filehandles.patch | 177 +----- .../patches/vendor/passa-close-session.patch | 12 + .../patches/vendor/vistir-spin-colorama.patch | 28 + 114 files changed, 3362 insertions(+), 2258 deletions(-) create mode 100644 pipenv/patched/notpip/_internal/cli/__init__.py create mode 100644 pipenv/patched/notpip/_internal/cli/autocompletion.py rename pipenv/patched/notpip/_internal/{basecommand.py => cli/base_command.py} (92%) rename pipenv/patched/notpip/_internal/{ => cli}/cmdoptions.py (81%) create mode 100644 pipenv/patched/notpip/_internal/cli/main_parser.py rename pipenv/patched/notpip/_internal/{baseparser.py => cli/parser.py} (88%) rename pipenv/patched/notpip/_internal/{ => cli}/status_codes.py (100%) create mode 100644 pipenv/patched/notpip/_internal/models/candidate.py create mode 100644 pipenv/patched/notpip/_internal/models/format_control.py create mode 100644 pipenv/patched/notpip/_internal/models/link.py create mode 100644 pipenv/patched/notpip/_internal/pyproject.py create mode 100644 pipenv/patched/notpip/_internal/req/constructors.py rename pipenv/patched/notpip/_internal/{ => utils}/compat.py (96%) create mode 100644 pipenv/patched/notpip/_internal/utils/models.py rename pipenv/{vendor/pathlib2.LICENSE.rst => patched/notpip/_vendor/pep517/LICENSE} (83%) create mode 100644 pipenv/patched/notpip/_vendor/pep517/__init__.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/_in_process.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/check.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/colorlog.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/compat.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/envbuild.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/wrappers.py delete mode 100644 pipenv/vendor/modutil.LICENSE delete mode 100644 pipenv/vendor/shutil_backports/__init__.py delete mode 100644 pipenv/vendor/shutil_backports/get_terminal_size.py create mode 100644 tasks/vendoring/patches/vendor/passa-close-session.patch create mode 100644 tasks/vendoring/patches/vendor/vistir-spin-colorama.patch diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index c7aef0df7b..4c2640ab37 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -4,6 +4,7 @@ steps: pip install certifi python -m certifi > cacert.txt Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" + $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail # https://bugs.python.org/issue18199 subst T: "$env:TEMP" @@ -11,8 +12,7 @@ steps: $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" $env:TEMP='T:\' - Get-ChildItem Env: - D:\.venv\Scripts\pipenv run pytest -n 4 -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor -k 'test_get_vcs_refs or test_install_editable_git_tag' --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 8c6cec195a..6b8ddf6664 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -1,9 +1,13 @@ +# -*- coding=utf-8 -*- # |~~\' |~~ # |__/||~~\|--|/~\\ / # | ||__/|__| |\/ # | + import os import sys +import warnings + from .__version__ import __version__ PIPENV_ROOT = os.path.dirname(os.path.realpath(__file__)) @@ -13,14 +17,28 @@ sys.path.insert(0, PIPENV_VENDOR) # Inject patched directory into system path. sys.path.insert(0, PIPENV_PATCHED) -from vistir.compat import fs_str + +from pipenv.vendor.urllib3.exceptions import DependencyWarning +from pipenv.vendor.vistir.compat import ResourceWarning, fs_str +warnings.filterwarnings("ignore", category=DependencyWarning) +warnings.filterwarnings("ignore", category=ResourceWarning) + +if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): + if sys.stdout.isatty() and sys.stderr.isatty(): + import io + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") +os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip") + # Hack to make things work better. try: if "concurrency" in sys.modules: del sys.modules["concurrency"] except Exception: pass + from .cli import cli from . import resolver diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 223baec1e2..087025a4a2 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -300,3 +300,79 @@ def NamedTemporaryFile( os.unlink(name) os.close(fd) raise + + +def getpreferredencoding(): + import locale + # Borrowed from Invoke + # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) + _encoding = locale.getpreferredencoding(False) + if six.PY2 and not sys.platform == "win32": + _default_encoding = locale.getdefaultlocale()[1] + if _default_encoding is not None: + _encoding = _default_encoding + return _encoding + + +DEFAULT_ENCODING = getpreferredencoding() + + +# From https://github.com/CarlFK/veyepar/blob/5c5de47/dj/scripts/fixunicode.py +# MIT LIcensed, thanks Carl! +def force_encoding(): + try: + stdout_isatty = sys.stdout.isatty + stderr_isatty = sys.stderr.isatty + except AttributeError: + return DEFAULT_ENCODING, DEFAULT_ENCODING + else: + if not (stdout_isatty() and stderr_isatty()): + return DEFAULT_ENCODING, DEFAULT_ENCODING + stdout_encoding = sys.stdout.encoding + stderr_encoding = sys.stderr.encoding + if sys.platform == "win32" and sys.version_info >= (3, 1): + return DEFAULT_ENCODING, DEFAULT_ENCODING + if stdout_encoding.lower() != "utf-8" or stderr_encoding.lower() != "utf-8": + + from ctypes import pythonapi, py_object, c_char_p + try: + PyFile_SetEncoding = pythonapi.PyFile_SetEncoding + except AttributeError: + return DEFAULT_ENCODING, DEFAULT_ENCODING + else: + PyFile_SetEncoding.argtypes = (py_object, c_char_p) + if stdout_encoding.lower() != "utf-8": + try: + was_set = PyFile_SetEncoding(sys.stdout, "utf-8") + except OSError: + was_set = False + if not was_set: + stdout_encoding = DEFAULT_ENCODING + else: + stdout_encoding = "utf-8" + + if stderr_encoding.lower() != "utf-8": + try: + was_set = PyFile_SetEncoding(sys.stderr, "utf-8") + except OSError: + was_set = False + if not was_set: + stderr_encoding = DEFAULT_ENCODING + else: + stderr_encoding = "utf-8" + + return stdout_encoding, stderr_encoding + + +OUT_ENCODING, ERR_ENCODING = force_encoding() + + +def decode_output(output): + if not isinstance(output, six.string_types): + return output + try: + output = output.encode(DEFAULT_ENCODING) + except AttributeError: + pass + output = output.decode(DEFAULT_ENCODING) + return output diff --git a/pipenv/core.py b/pipenv/core.py index bb415317cb..ee1658bf6e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- + import contextlib import logging import os @@ -105,9 +106,13 @@ def fix_utf8(text): if not isinstance(text, six.string_types): return text - if six.PY2: - text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) - return u"{0}".format(text) + from ._compat import decode_output + try: + text = decode_output(text) + except UnicodeDecodeError: + if six.PY2: + text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) + return text @contextlib.contextmanager @@ -230,7 +235,7 @@ def cleanup_virtualenv(bare=True): def import_requirements(r=None, dev=False): from .patched.notpip._vendor import requests as pip_requests - from .patched.notpip._internal.req.req_file import parse_requirements + from .vendor.pip_shims.shims import parse_requirements # Parse requirements.txt file with Pip's parser. # Pip requires a `PipSession` which is a subclass of requests.Session. @@ -1754,7 +1759,7 @@ def do_install( selective_upgrade=False, ): from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM - from notpip._internal.exceptions import PipError + from .vendor.pip_shims.shims import PipError requirements_directory = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" @@ -2212,10 +2217,6 @@ def _launch_windows_subprocess(script): command = system_which(script.command) options = {"universal_newlines": True} - env_strings = [ - vistir.compat.to_native_string("{0}: {1}".format(k, v)) for k, v in os.environ.items() - ] - click.echo(vistir.compat.to_native_string("\n".join(env_strings)), err=True) # Command not found, maybe this is a shell built-in? if not command: diff --git a/pipenv/patched/notpip/__init__.py b/pipenv/patched/notpip/__init__.py index 9227d0eadc..ae265fa7df 100644 --- a/pipenv/patched/notpip/__init__.py +++ b/pipenv/patched/notpip/__init__.py @@ -1 +1 @@ -__version__ = "18.0" +__version__ = "18.1" diff --git a/pipenv/patched/notpip/_internal/__init__.py b/pipenv/patched/notpip/_internal/__init__.py index dcd0937e25..6d223928fe 100644 --- a/pipenv/patched/notpip/_internal/__init__.py +++ b/pipenv/patched/notpip/_internal/__init__.py @@ -4,7 +4,6 @@ import locale import logging import os -import optparse import warnings import sys @@ -38,17 +37,12 @@ else: securetransport.inject_into_urllib3() -from pipenv.patched.notpip import __version__ -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.exceptions import CommandError, PipError -from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions, get_prog +from pipenv.patched.notpip._internal.cli.autocompletion import autocomplete +from pipenv.patched.notpip._internal.cli.main_parser import parse_command +from pipenv.patched.notpip._internal.commands import commands_dict +from pipenv.patched.notpip._internal.exceptions import PipError from pipenv.patched.notpip._internal.utils import deprecation from pipenv.patched.notpip._internal.vcs import git, mercurial, subversion, bazaar # noqa -from pipenv.patched.notpip._internal.baseparser import ( - ConfigOptionParser, UpdatingDefaultsHelpFormatter, -) -from pipenv.patched.notpip._internal.commands import get_summaries, get_similar_commands -from pipenv.patched.notpip._internal.commands import commands_dict from pipenv.patched.notpip._vendor.urllib3.exceptions import InsecureRequestWarning logger = logging.getLogger(__name__) @@ -57,232 +51,6 @@ warnings.filterwarnings("ignore", category=InsecureRequestWarning) -def autocomplete(): - """Command and option completion for the main option parser (and options) - and its subcommands (and options). - - Enable by sourcing one of the completion shell scripts (bash, zsh or fish). - """ - # Don't complete if user hasn't sourced bash_completion file. - if 'PIP_AUTO_COMPLETE' not in os.environ: - return - cwords = os.environ['COMP_WORDS'].split()[1:] - cword = int(os.environ['COMP_CWORD']) - try: - current = cwords[cword - 1] - except IndexError: - current = '' - - subcommands = [cmd for cmd, summary in get_summaries()] - options = [] - # subcommand - try: - subcommand_name = [w for w in cwords if w in subcommands][0] - except IndexError: - subcommand_name = None - - parser = create_main_parser() - # subcommand options - if subcommand_name: - # special case: 'help' subcommand has no options - if subcommand_name == 'help': - sys.exit(1) - # special case: list locally installed dists for show and uninstall - should_list_installed = ( - subcommand_name in ['show', 'uninstall'] and - not current.startswith('-') - ) - if should_list_installed: - installed = [] - lc = current.lower() - for dist in get_installed_distributions(local_only=True): - if dist.key.startswith(lc) and dist.key not in cwords[1:]: - installed.append(dist.key) - # if there are no dists installed, fall back to option completion - if installed: - for dist in installed: - print(dist) - sys.exit(1) - - subcommand = commands_dict[subcommand_name]() - - for opt in subcommand.parser.option_list_all: - if opt.help != optparse.SUPPRESS_HELP: - for opt_str in opt._long_opts + opt._short_opts: - options.append((opt_str, opt.nargs)) - - # filter out previously specified options from available options - prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] - options = [(x, v) for (x, v) in options if x not in prev_opts] - # filter options by current input - options = [(k, v) for k, v in options if k.startswith(current)] - # get completion type given cwords and available subcommand options - completion_type = get_path_completion_type( - cwords, cword, subcommand.parser.option_list_all, - ) - # get completion files and directories if ``completion_type`` is - # ````, ```` or ```` - if completion_type: - options = auto_complete_paths(current, completion_type) - options = ((opt, 0) for opt in options) - for option in options: - opt_label = option[0] - # append '=' to options which require args - if option[1] and option[0][:2] == "--": - opt_label += '=' - print(opt_label) - else: - # show main parser options only when necessary - - opts = [i.option_list for i in parser.option_groups] - opts.append(parser.option_list) - opts = (o for it in opts for o in it) - if current.startswith('-'): - for opt in opts: - if opt.help != optparse.SUPPRESS_HELP: - subcommands += opt._long_opts + opt._short_opts - else: - # get completion type given cwords and all available options - completion_type = get_path_completion_type(cwords, cword, opts) - if completion_type: - subcommands = auto_complete_paths(current, completion_type) - - print(' '.join([x for x in subcommands if x.startswith(current)])) - sys.exit(1) - - -def get_path_completion_type(cwords, cword, opts): - """Get the type of path completion (``file``, ``dir``, ``path`` or None) - - :param cwords: same as the environmental variable ``COMP_WORDS`` - :param cword: same as the environmental variable ``COMP_CWORD`` - :param opts: The available options to check - :return: path completion type (``file``, ``dir``, ``path`` or None) - """ - if cword < 2 or not cwords[cword - 2].startswith('-'): - return - for opt in opts: - if opt.help == optparse.SUPPRESS_HELP: - continue - for o in str(opt).split('/'): - if cwords[cword - 2].split('=')[0] == o: - if any(x in ('path', 'file', 'dir') - for x in opt.metavar.split('/')): - return opt.metavar - - -def auto_complete_paths(current, completion_type): - """If ``completion_type`` is ``file`` or ``path``, list all regular files - and directories starting with ``current``; otherwise only list directories - starting with ``current``. - - :param current: The word to be completed - :param completion_type: path completion type(`file`, `path` or `dir`)i - :return: A generator of regular files and/or directories - """ - directory, filename = os.path.split(current) - current_path = os.path.abspath(directory) - # Don't complete paths if they can't be accessed - if not os.access(current_path, os.R_OK): - return - filename = os.path.normcase(filename) - # list all files that start with ``filename`` - file_list = (x for x in os.listdir(current_path) - if os.path.normcase(x).startswith(filename)) - for f in file_list: - opt = os.path.join(current_path, f) - comp_file = os.path.normcase(os.path.join(directory, f)) - # complete regular files when there is not ```` after option - # complete directories when there is ````, ```` or - # ````after option - if completion_type != 'dir' and os.path.isfile(opt): - yield comp_file - elif os.path.isdir(opt): - yield os.path.join(comp_file, '') - - -def create_main_parser(): - parser_kw = { - 'usage': '\n%prog [options]', - 'add_help_option': False, - 'formatter': UpdatingDefaultsHelpFormatter(), - 'name': 'global', - 'prog': get_prog(), - } - - parser = ConfigOptionParser(**parser_kw) - parser.disable_interspersed_args() - - pip_pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - parser.version = 'pip %s from %s (python %s)' % ( - __version__, pip_pkg_dir, sys.version[:3], - ) - - # add the general options - gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) - parser.add_option_group(gen_opts) - - parser.main = True # so the help formatter knows - - # create command listing for description - command_summaries = get_summaries() - description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] - parser.description = '\n'.join(description) - - return parser - - -def parseopts(args): - parser = create_main_parser() - - # Note: parser calls disable_interspersed_args(), so the result of this - # call is to split the initial args into the general options before the - # subcommand and everything else. - # For example: - # args: ['--timeout=5', 'install', '--user', 'INITools'] - # general_options: ['--timeout==5'] - # args_else: ['install', '--user', 'INITools'] - general_options, args_else = parser.parse_args(args) - - # --version - if general_options.version: - sys.stdout.write(parser.version) - sys.stdout.write(os.linesep) - sys.exit() - - # pip || pip help -> print_help() - if not args_else or (args_else[0] == 'help' and len(args_else) == 1): - parser.print_help() - sys.exit() - - # the subcommand name - cmd_name = args_else[0] - - if cmd_name not in commands_dict: - guess = get_similar_commands(cmd_name) - - msg = ['unknown command "%s"' % cmd_name] - if guess: - msg.append('maybe you meant "%s"' % guess) - - raise CommandError(' - '.join(msg)) - - # all the args without the subcommand - cmd_args = args[:] - cmd_args.remove(cmd_name) - - return cmd_name, cmd_args - - -def check_isolated(args): - isolated = False - - if "--isolated" in args: - isolated = True - - return isolated - - def main(args=None): if args is None: args = sys.argv[1:] @@ -293,7 +61,7 @@ def main(args=None): autocomplete() try: - cmd_name, cmd_args = parseopts(args) + cmd_name, cmd_args = parse_command(args) except PipError as exc: sys.stderr.write("ERROR: %s" % exc) sys.stderr.write(os.linesep) @@ -306,5 +74,5 @@ def main(args=None): except locale.Error as e: # setlocale can apparently crash if locale are uninitialized logger.debug("Ignoring error %s when setting locale", e) - command = commands_dict[cmd_name](isolated=check_isolated(cmd_args)) + command = commands_dict[cmd_name](isolated=("--isolated" in cmd_args)) return command.main(cmd_args) diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py index 1d351b5c42..6d696fbdac 100644 --- a/pipenv/patched/notpip/_internal/build_env.py +++ b/pipenv/patched/notpip/_internal/build_env.py @@ -7,6 +7,8 @@ from distutils.sysconfig import get_python_lib from sysconfig import get_paths +from pipenv.patched.notpip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet + from pipenv.patched.notpip._internal.utils.misc import call_subprocess from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.utils.ui import open_spinner @@ -75,6 +77,20 @@ def restore_var(varname, old_value): def cleanup(self): self._temp_dir.cleanup() + def missing_requirements(self, reqs): + """Return a list of the requirements from reqs that are not present + """ + missing = [] + with self: + ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep)) + for req in reqs: + try: + if ws.find(Requirement.parse(req)) is None: + missing.append(req) + except VersionConflict: + missing.append(req) + return missing + def install_requirements(self, finder, requirements, message): args = [ sys.executable, '-m', 'pip', 'install', '--ignore-installed', diff --git a/pipenv/patched/notpip/_internal/cache.py b/pipenv/patched/notpip/_internal/cache.py index a3a28fd89e..d91b817039 100644 --- a/pipenv/patched/notpip/_internal/cache.py +++ b/pipenv/patched/notpip/_internal/cache.py @@ -8,9 +8,9 @@ from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal import index -from pipenv.patched.notpip._internal.compat import expanduser from pipenv.patched.notpip._internal.download import path_to_url +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.wheel import InvalidWheelFilename, Wheel @@ -22,7 +22,7 @@ class Cache(object): :param cache_dir: The root of the cache. - :param format_control: A pip.index.FormatControl object to limit + :param format_control: An object of FormatControl class to limit binaries being read from the cache. :param allowed_formats: which formats of files the cache should store. ('binary' and 'source' are the only allowed values) @@ -72,8 +72,8 @@ def _get_candidates(self, link, package_name): return [] canonical_name = canonicalize_name(package_name) - formats = index.fmt_ctl_formats( - self.format_control, canonical_name + formats = self.format_control.get_allowed_formats( + canonical_name ) if not self.allowed_formats.intersection(formats): return [] @@ -101,7 +101,7 @@ def _link_for_candidate(self, link, candidate): root = self.get_path_for_link(link) path = os.path.join(root, candidate) - return index.Link(path_to_url(path)) + return Link(path_to_url(path)) def cleanup(self): pass diff --git a/pipenv/patched/notpip/_internal/cli/__init__.py b/pipenv/patched/notpip/_internal/cli/__init__.py new file mode 100644 index 0000000000..e589bb917e --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/__init__.py @@ -0,0 +1,4 @@ +"""Subpackage containing all of pip's command line interface related code +""" + +# This file intentionally does not import submodules diff --git a/pipenv/patched/notpip/_internal/cli/autocompletion.py b/pipenv/patched/notpip/_internal/cli/autocompletion.py new file mode 100644 index 0000000000..15b560a1dc --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/autocompletion.py @@ -0,0 +1,152 @@ +"""Logic that powers autocompletion installed by ``pip completion``. +""" + +import optparse +import os +import sys + +from pipenv.patched.notpip._internal.cli.main_parser import create_main_parser +from pipenv.patched.notpip._internal.commands import commands_dict, get_summaries +from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions + + +def autocomplete(): + """Entry Point for completion of main and subcommand options. + """ + # Don't complete if user hasn't sourced bash_completion file. + if 'PIP_AUTO_COMPLETE' not in os.environ: + return + cwords = os.environ['COMP_WORDS'].split()[1:] + cword = int(os.environ['COMP_CWORD']) + try: + current = cwords[cword - 1] + except IndexError: + current = '' + + subcommands = [cmd for cmd, summary in get_summaries()] + options = [] + # subcommand + try: + subcommand_name = [w for w in cwords if w in subcommands][0] + except IndexError: + subcommand_name = None + + parser = create_main_parser() + # subcommand options + if subcommand_name: + # special case: 'help' subcommand has no options + if subcommand_name == 'help': + sys.exit(1) + # special case: list locally installed dists for show and uninstall + should_list_installed = ( + subcommand_name in ['show', 'uninstall'] and + not current.startswith('-') + ) + if should_list_installed: + installed = [] + lc = current.lower() + for dist in get_installed_distributions(local_only=True): + if dist.key.startswith(lc) and dist.key not in cwords[1:]: + installed.append(dist.key) + # if there are no dists installed, fall back to option completion + if installed: + for dist in installed: + print(dist) + sys.exit(1) + + subcommand = commands_dict[subcommand_name]() + + for opt in subcommand.parser.option_list_all: + if opt.help != optparse.SUPPRESS_HELP: + for opt_str in opt._long_opts + opt._short_opts: + options.append((opt_str, opt.nargs)) + + # filter out previously specified options from available options + prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] + options = [(x, v) for (x, v) in options if x not in prev_opts] + # filter options by current input + options = [(k, v) for k, v in options if k.startswith(current)] + # get completion type given cwords and available subcommand options + completion_type = get_path_completion_type( + cwords, cword, subcommand.parser.option_list_all, + ) + # get completion files and directories if ``completion_type`` is + # ````, ```` or ```` + if completion_type: + options = auto_complete_paths(current, completion_type) + options = ((opt, 0) for opt in options) + for option in options: + opt_label = option[0] + # append '=' to options which require args + if option[1] and option[0][:2] == "--": + opt_label += '=' + print(opt_label) + else: + # show main parser options only when necessary + + opts = [i.option_list for i in parser.option_groups] + opts.append(parser.option_list) + opts = (o for it in opts for o in it) + if current.startswith('-'): + for opt in opts: + if opt.help != optparse.SUPPRESS_HELP: + subcommands += opt._long_opts + opt._short_opts + else: + # get completion type given cwords and all available options + completion_type = get_path_completion_type(cwords, cword, opts) + if completion_type: + subcommands = auto_complete_paths(current, completion_type) + + print(' '.join([x for x in subcommands if x.startswith(current)])) + sys.exit(1) + + +def get_path_completion_type(cwords, cword, opts): + """Get the type of path completion (``file``, ``dir``, ``path`` or None) + + :param cwords: same as the environmental variable ``COMP_WORDS`` + :param cword: same as the environmental variable ``COMP_CWORD`` + :param opts: The available options to check + :return: path completion type (``file``, ``dir``, ``path`` or None) + """ + if cword < 2 or not cwords[cword - 2].startswith('-'): + return + for opt in opts: + if opt.help == optparse.SUPPRESS_HELP: + continue + for o in str(opt).split('/'): + if cwords[cword - 2].split('=')[0] == o: + if not opt.metavar or any( + x in ('path', 'file', 'dir') + for x in opt.metavar.split('/')): + return opt.metavar + + +def auto_complete_paths(current, completion_type): + """If ``completion_type`` is ``file`` or ``path``, list all regular files + and directories starting with ``current``; otherwise only list directories + starting with ``current``. + + :param current: The word to be completed + :param completion_type: path completion type(`file`, `path` or `dir`)i + :return: A generator of regular files and/or directories + """ + directory, filename = os.path.split(current) + current_path = os.path.abspath(directory) + # Don't complete paths if they can't be accessed + if not os.access(current_path, os.R_OK): + return + filename = os.path.normcase(filename) + # list all files that start with ``filename`` + file_list = (x for x in os.listdir(current_path) + if os.path.normcase(x).startswith(filename)) + for f in file_list: + opt = os.path.join(current_path, f) + comp_file = os.path.normcase(os.path.join(directory, f)) + # complete regular files when there is not ```` after option + # complete directories when there is ````, ```` or + # ````after option + if completion_type != 'dir' and os.path.isfile(opt): + yield comp_file + elif os.path.isdir(opt): + yield os.path.join(comp_file, '') diff --git a/pipenv/patched/notpip/_internal/basecommand.py b/pipenv/patched/notpip/_internal/cli/base_command.py similarity index 92% rename from pipenv/patched/notpip/_internal/basecommand.py rename to pipenv/patched/notpip/_internal/cli/base_command.py index 60199d5532..229831f238 100644 --- a/pipenv/patched/notpip/_internal/basecommand.py +++ b/pipenv/patched/notpip/_internal/cli/base_command.py @@ -7,10 +7,14 @@ import os import sys -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.baseparser import ( +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.parser import ( ConfigOptionParser, UpdatingDefaultsHelpFormatter, ) +from pipenv.patched.notpip._internal.cli.status_codes import ( + ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, + VIRTUALENV_NOT_FOUND, +) from pipenv.patched.notpip._internal.download import PipSession from pipenv.patched.notpip._internal.exceptions import ( BadCommand, CommandError, InstallationError, PreviousBuildDirError, @@ -18,12 +22,10 @@ ) from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.locations import running_under_virtualenv -from pipenv.patched.notpip._internal.req.req_file import parse_requirements -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement -from pipenv.patched.notpip._internal.status_codes import ( - ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, - VIRTUALENV_NOT_FOUND, +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, ) +from pipenv.patched.notpip._internal.req.req_file import parse_requirements from pipenv.patched.notpip._internal.utils.logging import setup_logging from pipenv.patched.notpip._internal.utils.misc import get_prog, normalize_path from pipenv.patched.notpip._internal.utils.outdated import pip_version_check @@ -168,12 +170,14 @@ def main(self, args): return UNKNOWN_ERROR finally: - # Check if we're using the latest version of pip available - skip_version_check = ( - options.disable_pip_version_check or - getattr(options, "no_index", False) + allow_version_check = ( + # Does this command have the index_group options? + hasattr(options, "no_index") and + # Is this command allowed to perform this check? + not (options.disable_pip_version_check or options.no_index) ) - if not skip_version_check: + # Check if we're using the latest version of pip available + if allow_version_check: session = self._build_session( options, retries=0, @@ -208,7 +212,7 @@ def populate_requirement_set(requirement_set, args, options, finder, requirement_set.add_requirement(req_to_add) for req in args: - req_to_add = InstallRequirement.from_line( + req_to_add = install_req_from_line( req, None, isolated=options.isolated_mode, wheel_cache=wheel_cache ) @@ -216,7 +220,7 @@ def populate_requirement_set(requirement_set, args, options, finder, requirement_set.add_requirement(req_to_add) for req in options.editables: - req_to_add = InstallRequirement.from_editable( + req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, wheel_cache=wheel_cache diff --git a/pipenv/patched/notpip/_internal/cmdoptions.py b/pipenv/patched/notpip/_internal/cli/cmdoptions.py similarity index 81% rename from pipenv/patched/notpip/_internal/cmdoptions.py rename to pipenv/patched/notpip/_internal/cli/cmdoptions.py index c25e769fb3..a075a67e44 100644 --- a/pipenv/patched/notpip/_internal/cmdoptions.py +++ b/pipenv/patched/notpip/_internal/cli/cmdoptions.py @@ -13,10 +13,9 @@ from functools import partial from optparse import SUPPRESS_HELP, Option, OptionGroup -from pipenv.patched.notpip._internal.index import ( - FormatControl, fmt_ctl_handle_mutual_exclude, fmt_ctl_no_binary, -) +from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.locations import USER_CACHE_DIR, src_prefix +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.models.index import PyPI from pipenv.patched.notpip._internal.utils.hashes import STRONG_HASHES from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -53,13 +52,52 @@ def getname(n): names = ["build_options", "global_options", "install_options"] if any(map(getname, names)): control = options.format_control - fmt_ctl_no_binary(control) + control.disallow_binaries() warnings.warn( 'Disabling all use of wheels due to the use of --build-options ' '/ --global-options / --install-options.', stacklevel=2, ) +def check_dist_restriction(options, check_target=False): + """Function for determining if custom platform options are allowed. + + :param options: The OptionParser options. + :param check_target: Whether or not to check if --target is being used. + """ + dist_restriction_set = any([ + options.python_version, + options.platform, + options.abi, + options.implementation, + ]) + + binary_only = FormatControl(set(), {':all:'}) + sdist_dependencies_allowed = ( + options.format_control != binary_only and + not options.ignore_dependencies + ) + + # Installations or downloads using dist restrictions must not combine + # source distributions and dist-specific wheels, as they are not + # gauranteed to be locally compatible. + if dist_restriction_set and sdist_dependencies_allowed: + raise CommandError( + "When restricting platform and interpreter constraints using " + "--python-version, --platform, --abi, or --implementation, " + "either --no-deps must be set, or --only-binary=:all: must be " + "set and --no-binary must not be set (or must be set to " + ":none:)." + ) + + if check_target: + if dist_restriction_set and not options.target_dir: + raise CommandError( + "Can not use any platform or abi specific options unless " + "installing via '--target'" + ) + + ########### # options # ########### @@ -365,24 +403,25 @@ def _get_format_control(values, option): def _handle_no_binary(option, opt_str, value, parser): - existing = getattr(parser.values, option.dest) - fmt_ctl_handle_mutual_exclude( + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( value, existing.no_binary, existing.only_binary, ) def _handle_only_binary(option, opt_str, value, parser): - existing = getattr(parser.values, option.dest) - fmt_ctl_handle_mutual_exclude( + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( value, existing.only_binary, existing.no_binary, ) def no_binary(): + format_control = FormatControl(set(), set()) return Option( "--no-binary", dest="format_control", action="callback", callback=_handle_no_binary, type="str", - default=FormatControl(set(), set()), + default=format_control, help="Do not use binary packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all binary packages, :none: to empty the set, or one or " @@ -393,10 +432,11 @@ def no_binary(): def only_binary(): + format_control = FormatControl(set(), set()) return Option( "--only-binary", dest="format_control", action="callback", callback=_handle_only_binary, type="str", - default=FormatControl(set(), set()), + default=format_control, help="Do not use source packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all source packages, :none: to empty the set, or one or " @@ -406,6 +446,61 @@ def only_binary(): ) +platform = partial( + Option, + '--platform', + dest='platform', + metavar='platform', + default=None, + help=("Only use wheels compatible with . " + "Defaults to the platform of the running system."), +) + + +python_version = partial( + Option, + '--python-version', + dest='python_version', + metavar='python_version', + default=None, + help=("Only use wheels compatible with Python " + "interpreter version . If not specified, then the " + "current system interpreter minor version is used. A major " + "version (e.g. '2') can be specified to match all " + "minor revs of that major version. A minor version " + "(e.g. '34') can also be specified."), +) + + +implementation = partial( + Option, + '--implementation', + dest='implementation', + metavar='implementation', + default=None, + help=("Only use wheels compatible with Python " + "implementation , e.g. 'pp', 'jy', 'cp', " + " or 'ip'. If not specified, then the current " + "interpreter implementation is used. Use 'py' to force " + "implementation-agnostic wheels."), +) + + +abi = partial( + Option, + '--abi', + dest='abi', + metavar='abi', + default=None, + help=("Only use wheels compatible with Python " + "abi , e.g. 'pypy_41'. If not specified, then the " + "current interpreter abi tag is used. Generally " + "you will need to specify --implementation, " + "--platform, and --python-version when using " + "this option."), +) + + def prefer_binary(): return Option( "--prefer-binary", @@ -501,7 +596,7 @@ def prefer_binary(): '--no-clean', action='store_true', default=False, - help="Don't clean up build directories)." + help="Don't clean up build directories." ) # type: Any pre = partial( diff --git a/pipenv/patched/notpip/_internal/cli/main_parser.py b/pipenv/patched/notpip/_internal/cli/main_parser.py new file mode 100644 index 0000000000..abe2f69e71 --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/main_parser.py @@ -0,0 +1,96 @@ +"""A single place for constructing and exposing the main parser +""" + +import os +import sys + +from pipenv.patched.notpip import __version__ +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.parser import ( + ConfigOptionParser, UpdatingDefaultsHelpFormatter, +) +from pipenv.patched.notpip._internal.commands import ( + commands_dict, get_similar_commands, get_summaries, +) +from pipenv.patched.notpip._internal.exceptions import CommandError +from pipenv.patched.notpip._internal.utils.misc import get_prog + +__all__ = ["create_main_parser", "parse_command"] + + +def create_main_parser(): + """Creates and returns the main parser for pip's CLI + """ + + parser_kw = { + 'usage': '\n%prog [options]', + 'add_help_option': False, + 'formatter': UpdatingDefaultsHelpFormatter(), + 'name': 'global', + 'prog': get_prog(), + } + + parser = ConfigOptionParser(**parser_kw) + parser.disable_interspersed_args() + + pip_pkg_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), "..", "..", + )) + parser.version = 'pip %s from %s (python %s)' % ( + __version__, pip_pkg_dir, sys.version[:3], + ) + + # add the general options + gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) + parser.add_option_group(gen_opts) + + parser.main = True # so the help formatter knows + + # create command listing for description + command_summaries = get_summaries() + description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] + parser.description = '\n'.join(description) + + return parser + + +def parse_command(args): + parser = create_main_parser() + + # Note: parser calls disable_interspersed_args(), so the result of this + # call is to split the initial args into the general options before the + # subcommand and everything else. + # For example: + # args: ['--timeout=5', 'install', '--user', 'INITools'] + # general_options: ['--timeout==5'] + # args_else: ['install', '--user', 'INITools'] + general_options, args_else = parser.parse_args(args) + + # --version + if general_options.version: + sys.stdout.write(parser.version) + sys.stdout.write(os.linesep) + sys.exit() + + # pip || pip help -> print_help() + if not args_else or (args_else[0] == 'help' and len(args_else) == 1): + parser.print_help() + sys.exit() + + # the subcommand name + cmd_name = args_else[0] + + if cmd_name not in commands_dict: + guess = get_similar_commands(cmd_name) + + msg = ['unknown command "%s"' % cmd_name] + if guess: + msg.append('maybe you meant "%s"' % guess) + + raise CommandError(' - '.join(msg)) + + # all the args without the subcommand + cmd_args = args[:] + cmd_args.remove(cmd_name) + + return cmd_name, cmd_args diff --git a/pipenv/patched/notpip/_internal/baseparser.py b/pipenv/patched/notpip/_internal/cli/parser.py similarity index 88% rename from pipenv/patched/notpip/_internal/baseparser.py rename to pipenv/patched/notpip/_internal/cli/parser.py index f82093bfad..2d2c8f4d7b 100644 --- a/pipenv/patched/notpip/_internal/baseparser.py +++ b/pipenv/patched/notpip/_internal/cli/parser.py @@ -9,8 +9,9 @@ from pipenv.patched.notpip._vendor.six import string_types -from pipenv.patched.notpip._internal.compat import get_terminal_size +from pipenv.patched.notpip._internal.cli.status_codes import UNKNOWN_ERROR from pipenv.patched.notpip._internal.configuration import Configuration, ConfigurationError +from pipenv.patched.notpip._internal.utils.compat import get_terminal_size logger = logging.getLogger(__name__) @@ -192,7 +193,14 @@ def _update_defaults(self, defaults): continue if option.action in ('store_true', 'store_false', 'count'): - val = strtobool(val) + try: + val = strtobool(val) + except ValueError: + error_msg = invalid_config_error_message( + option.action, key, val + ) + self.error(error_msg) + elif option.action == 'append': val = val.split() val = [self.check_default(option, key, v) for v in val] @@ -225,7 +233,7 @@ def get_default_values(self): try: self.config.load() except ConfigurationError as err: - self.exit(2, err.args[0]) + self.exit(UNKNOWN_ERROR, str(err)) defaults = self._update_defaults(self.defaults.copy()) # ours for option in self._get_all_options(): @@ -237,4 +245,17 @@ def get_default_values(self): def error(self, msg): self.print_usage(sys.stderr) - self.exit(2, "%s\n" % msg) + self.exit(UNKNOWN_ERROR, "%s\n" % msg) + + +def invalid_config_error_message(action, key, val): + """Returns a better error message when invalid configuration option + is provided.""" + if action in ('store_true', 'store_false'): + return ("{0} is not a valid value for {1} option, " + "please specify a boolean value like yes/no, " + "true/false or 1/0 instead.").format(val, key) + + return ("{0} is not a valid value for {1} option, " + "please specify a numerical value like 1/0 " + "instead.").format(val, key) diff --git a/pipenv/patched/notpip/_internal/status_codes.py b/pipenv/patched/notpip/_internal/cli/status_codes.py similarity index 100% rename from pipenv/patched/notpip/_internal/status_codes.py rename to pipenv/patched/notpip/_internal/cli/status_codes.py diff --git a/pipenv/patched/notpip/_internal/commands/__init__.py b/pipenv/patched/notpip/_internal/commands/__init__.py index 140c460936..a403c6f9a7 100644 --- a/pipenv/patched/notpip/_internal/commands/__init__.py +++ b/pipenv/patched/notpip/_internal/commands/__init__.py @@ -21,7 +21,7 @@ if MYPY_CHECK_RUNNING: from typing import List, Type # noqa: F401 - from pipenv.patched.notpip._internal.basecommand import Command # noqa: F401 + from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 commands_order = [ InstallCommand, diff --git a/pipenv/patched/notpip/_internal/commands/check.py b/pipenv/patched/notpip/_internal/commands/check.py index cd5ffb5f9f..adf4f5e793 100644 --- a/pipenv/patched/notpip/_internal/commands/check.py +++ b/pipenv/patched/notpip/_internal/commands/check.py @@ -1,6 +1,6 @@ import logging -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.operations.check import ( check_package_set, create_package_set_from_installed, ) diff --git a/pipenv/patched/notpip/_internal/commands/completion.py b/pipenv/patched/notpip/_internal/commands/completion.py index f4c31c1bb1..cb8a11a7ca 100644 --- a/pipenv/patched/notpip/_internal/commands/completion.py +++ b/pipenv/patched/notpip/_internal/commands/completion.py @@ -3,7 +3,7 @@ import sys import textwrap -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.utils.misc import get_prog BASE_COMPLETION = """ diff --git a/pipenv/patched/notpip/_internal/commands/configuration.py b/pipenv/patched/notpip/_internal/commands/configuration.py index 090109c52b..6c1dbdfdbd 100644 --- a/pipenv/patched/notpip/_internal/commands/configuration.py +++ b/pipenv/patched/notpip/_internal/commands/configuration.py @@ -2,11 +2,11 @@ import os import subprocess -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS from pipenv.patched.notpip._internal.configuration import Configuration, kinds from pipenv.patched.notpip._internal.exceptions import PipError from pipenv.patched.notpip._internal.locations import venv_config_file -from pipenv.patched.notpip._internal.status_codes import ERROR, SUCCESS from pipenv.patched.notpip._internal.utils.misc import get_prog logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/download.py b/pipenv/patched/notpip/_internal/commands/download.py index 63d91b0407..e5d871216f 100644 --- a/pipenv/patched/notpip/_internal/commands/download.py +++ b/pipenv/patched/notpip/_internal/commands/download.py @@ -3,10 +3,8 @@ import logging import os -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand -from pipenv.patched.notpip._internal.exceptions import CommandError -from pipenv.patched.notpip._internal.index import FormatControl +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker @@ -69,52 +67,10 @@ def __init__(self, *args, **kw): help=("Download packages into ."), ) - cmd_opts.add_option( - '--platform', - dest='platform', - metavar='platform', - default=None, - help=("Only download wheels compatible with . " - "Defaults to the platform of the running system."), - ) - - cmd_opts.add_option( - '--python-version', - dest='python_version', - metavar='python_version', - default=None, - help=("Only download wheels compatible with Python " - "interpreter version . If not specified, then the " - "current system interpreter minor version is used. A major " - "version (e.g. '2') can be specified to match all " - "minor revs of that major version. A minor version " - "(e.g. '34') can also be specified."), - ) - - cmd_opts.add_option( - '--implementation', - dest='implementation', - metavar='implementation', - default=None, - help=("Only download wheels compatible with Python " - "implementation , e.g. 'pp', 'jy', 'cp', " - " or 'ip'. If not specified, then the current " - "interpreter implementation is used. Use 'py' to force " - "implementation-agnostic wheels."), - ) - - cmd_opts.add_option( - '--abi', - dest='abi', - metavar='abi', - default=None, - help=("Only download wheels compatible with Python " - "abi , e.g. 'pypy_41'. If not specified, then the " - "current interpreter abi tag is used. Generally " - "you will need to specify --implementation, " - "--platform, and --python-version when using " - "this option."), - ) + cmd_opts.add_option(cmdoptions.platform()) + cmd_opts.add_option(cmdoptions.python_version()) + cmd_opts.add_option(cmdoptions.implementation()) + cmd_opts.add_option(cmdoptions.abi()) index_opts = cmdoptions.make_option_group( cmdoptions.index_group, @@ -135,25 +91,7 @@ def run(self, options, args): else: python_versions = None - dist_restriction_set = any([ - options.python_version, - options.platform, - options.abi, - options.implementation, - ]) - binary_only = FormatControl(set(), {':all:'}) - no_sdist_dependencies = ( - options.format_control != binary_only and - not options.ignore_dependencies - ) - if dist_restriction_set and no_sdist_dependencies: - raise CommandError( - "When restricting platform and interpreter constraints using " - "--python-version, --platform, --abi, or --implementation, " - "either --no-deps must be set, or --only-binary=:all: must be " - "set and --no-binary must not be set (or must be set to " - ":none:)." - ) + cmdoptions.check_dist_restriction(options) options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) diff --git a/pipenv/patched/notpip/_internal/commands/freeze.py b/pipenv/patched/notpip/_internal/commands/freeze.py index 6959b6abc3..343227bab7 100644 --- a/pipenv/patched/notpip/_internal/commands/freeze.py +++ b/pipenv/patched/notpip/_internal/commands/freeze.py @@ -2,11 +2,11 @@ import sys -from pipenv.patched.notpip._internal import index -from pipenv.patched.notpip._internal.basecommand import Command from pipenv.patched.notpip._internal.cache import WheelCache -from pipenv.patched.notpip._internal.compat import stdlib_pkgs +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.operations.freeze import freeze +from pipenv.patched.notpip._internal.utils.compat import stdlib_pkgs DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel'} @@ -71,7 +71,7 @@ def __init__(self, *args, **kw): self.parser.insert_option_group(0, self.cmd_opts) def run(self, options, args): - format_control = index.FormatControl(set(), set()) + format_control = FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) skip = set(stdlib_pkgs) if not options.freeze_all: diff --git a/pipenv/patched/notpip/_internal/commands/hash.py b/pipenv/patched/notpip/_internal/commands/hash.py index a86574dae0..183f11aef3 100644 --- a/pipenv/patched/notpip/_internal/commands/hash.py +++ b/pipenv/patched/notpip/_internal/commands/hash.py @@ -4,8 +4,8 @@ import logging import sys -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.status_codes import ERROR +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR from pipenv.patched.notpip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES from pipenv.patched.notpip._internal.utils.misc import read_chunks diff --git a/pipenv/patched/notpip/_internal/commands/help.py b/pipenv/patched/notpip/_internal/commands/help.py index aaf0f87e8b..f2c619659b 100644 --- a/pipenv/patched/notpip/_internal/commands/help.py +++ b/pipenv/patched/notpip/_internal/commands/help.py @@ -1,6 +1,7 @@ from __future__ import absolute_import -from pipenv.patched.notpip._internal.basecommand import SUCCESS, Command +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import SUCCESS from pipenv.patched.notpip._internal.exceptions import CommandError diff --git a/pipenv/patched/notpip/_internal/commands/install.py b/pipenv/patched/notpip/_internal/commands/install.py index ebdf07d72b..ddcb4759a5 100644 --- a/pipenv/patched/notpip/_internal/commands/install.py +++ b/pipenv/patched/notpip/_internal/commands/install.py @@ -9,9 +9,10 @@ from pipenv.patched.notpip._vendor import pkg_resources -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand from pipenv.patched.notpip._internal.cache import WheelCache +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand +from pipenv.patched.notpip._internal.cli.status_codes import ERROR from pipenv.patched.notpip._internal.exceptions import ( CommandError, InstallationError, PreviousBuildDirError, ) @@ -21,7 +22,6 @@ from pipenv.patched.notpip._internal.req import RequirementSet, install_given_reqs from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker from pipenv.patched.notpip._internal.resolve import Resolver -from pipenv.patched.notpip._internal.status_codes import ERROR from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ( ensure_dir, get_installed_version, @@ -83,6 +83,11 @@ def __init__(self, *args, **kw): '. Use --upgrade to replace existing packages in ' 'with new versions.' ) + cmd_opts.add_option(cmdoptions.platform()) + cmd_opts.add_option(cmdoptions.python_version()) + cmd_opts.add_option(cmdoptions.implementation()) + cmd_opts.add_option(cmdoptions.abi()) + cmd_opts.add_option( '--user', dest='use_user_site', @@ -204,7 +209,6 @@ def __init__(self, *args, **kw): def run(self, options, args): cmdoptions.check_install_build_global(options) - upgrade_strategy = "to-satisfy-only" if options.upgrade: upgrade_strategy = options.upgrade_strategy @@ -212,6 +216,13 @@ def run(self, options, args): if options.build_dir: options.build_dir = os.path.abspath(options.build_dir) + cmdoptions.check_dist_restriction(options, check_target=True) + + if options.python_version: + python_versions = [options.python_version] + else: + python_versions = None + options.src_dir = os.path.abspath(options.src_dir) install_options = options.install_options or [] if options.use_user_site: @@ -246,7 +257,14 @@ def run(self, options, args): global_options = options.global_options or [] with self._build_session(options) as session: - finder = self._build_package_finder(options, session) + finder = self._build_package_finder( + options=options, + session=session, + platform=options.platform, + python_versions=python_versions, + abi=options.abi, + implementation=options.implementation, + ) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) @@ -266,6 +284,7 @@ def run(self, options, args): ) as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, + check_supported_wheels=not options.target_dir, ) try: diff --git a/pipenv/patched/notpip/_internal/commands/list.py b/pipenv/patched/notpip/_internal/commands/list.py index 99aee99fed..577c0b5f2b 100644 --- a/pipenv/patched/notpip/_internal/commands/list.py +++ b/pipenv/patched/notpip/_internal/commands/list.py @@ -6,8 +6,8 @@ from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import zip_longest -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.cmdoptions import index_group, make_option_group +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.utils.misc import ( @@ -102,7 +102,9 @@ def __init__(self, *args, **kw): help='Include editable package from output.', default=True, ) - index_opts = make_option_group(index_group, self.parser) + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, self.parser + ) self.parser.insert_option_group(0, index_opts) self.parser.insert_option_group(0, cmd_opts) diff --git a/pipenv/patched/notpip/_internal/commands/search.py b/pipenv/patched/notpip/_internal/commands/search.py index ac111c1487..986208f998 100644 --- a/pipenv/patched/notpip/_internal/commands/search.py +++ b/pipenv/patched/notpip/_internal/commands/search.py @@ -11,12 +11,12 @@ # why we ignore the type on this import from pipenv.patched.notpip._vendor.six.moves import xmlrpc_client # type: ignore -from pipenv.patched.notpip._internal.basecommand import SUCCESS, Command -from pipenv.patched.notpip._internal.compat import get_terminal_size +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS from pipenv.patched.notpip._internal.download import PipXmlrpcTransport from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.models.index import PyPI -from pipenv.patched.notpip._internal.status_codes import NO_MATCHES_FOUND +from pipenv.patched.notpip._internal.utils.compat import get_terminal_size from pipenv.patched.notpip._internal.utils.logging import indent_log logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/show.py b/pipenv/patched/notpip/_internal/commands/show.py index 8de6b6b8c5..3fd24482f8 100644 --- a/pipenv/patched/notpip/_internal/commands/show.py +++ b/pipenv/patched/notpip/_internal/commands/show.py @@ -7,8 +7,8 @@ from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.status_codes import ERROR, SUCCESS +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/uninstall.py b/pipenv/patched/notpip/_internal/commands/uninstall.py index 45a0eba52f..cf6a511c35 100644 --- a/pipenv/patched/notpip/_internal/commands/uninstall.py +++ b/pipenv/patched/notpip/_internal/commands/uninstall.py @@ -2,9 +2,10 @@ from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.exceptions import InstallationError -from pipenv.patched.notpip._internal.req import InstallRequirement, parse_requirements +from pipenv.patched.notpip._internal.req import parse_requirements +from pipenv.patched.notpip._internal.req.constructors import install_req_from_line from pipenv.patched.notpip._internal.utils.misc import protect_pip_from_modification_on_windows @@ -47,7 +48,7 @@ def run(self, options, args): with self._build_session(options) as session: reqs_to_uninstall = {} for name in args: - req = InstallRequirement.from_line( + req = install_req_from_line( name, isolated=options.isolated_mode, ) if req.name: diff --git a/pipenv/patched/notpip/_internal/commands/wheel.py b/pipenv/patched/notpip/_internal/commands/wheel.py index c04d58ed21..08d695abe0 100644 --- a/pipenv/patched/notpip/_internal/commands/wheel.py +++ b/pipenv/patched/notpip/_internal/commands/wheel.py @@ -4,9 +4,9 @@ import logging import os -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand from pipenv.patched.notpip._internal.cache import WheelCache +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand from pipenv.patched.notpip._internal.exceptions import CommandError, PreviousBuildDirError from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet diff --git a/pipenv/patched/notpip/_internal/configuration.py b/pipenv/patched/notpip/_internal/configuration.py index 3df185f741..3c02c95547 100644 --- a/pipenv/patched/notpip/_internal/configuration.py +++ b/pipenv/patched/notpip/_internal/configuration.py @@ -18,7 +18,9 @@ from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import configparser -from pipenv.patched.notpip._internal.exceptions import ConfigurationError +from pipenv.patched.notpip._internal.exceptions import ( + ConfigurationError, ConfigurationFileCouldNotBeLoaded, +) from pipenv.patched.notpip._internal.locations import ( legacy_config_file, new_config_file, running_under_virtualenv, site_config_files, venv_config_file, @@ -289,11 +291,16 @@ def _construct_parser(self, fname): try: parser.read(fname) except UnicodeDecodeError: - raise ConfigurationError(( - "ERROR: " - "Configuration file contains invalid %s characters.\n" - "Please fix your configuration, located at %s\n" - ) % (locale.getpreferredencoding(False), fname)) + # See https://github.com/pypa/pip/issues/4963 + raise ConfigurationFileCouldNotBeLoaded( + reason="contains invalid {} characters".format( + locale.getpreferredencoding(False) + ), + fname=fname, + ) + except configparser.Error as error: + # See https://github.com/pypa/pip/issues/4893 + raise ConfigurationFileCouldNotBeLoaded(error=error) return parser def _load_environment_vars(self): diff --git a/pipenv/patched/notpip/_internal/download.py b/pipenv/patched/notpip/_internal/download.py index 06a456449e..8f4c38f527 100644 --- a/pipenv/patched/notpip/_internal/download.py +++ b/pipenv/patched/notpip/_internal/download.py @@ -323,7 +323,7 @@ def cert_verify(self, conn, url, verify, cert): conn.ca_certs = None -class PipSession(Session): +class PipSession(requests.Session): timeout = None @@ -753,7 +753,7 @@ def _copy_dist_from_dir(link_path, location): # build an sdist setup_py = 'setup.py' - sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] + sdist_args = [sys.executable] sdist_args.append('-c') sdist_args.append(SETUPTOOLS_SHIM % setup_py) sdist_args.append('sdist') diff --git a/pipenv/patched/notpip/_internal/exceptions.py b/pipenv/patched/notpip/_internal/exceptions.py index 43595977db..2eadcf2840 100644 --- a/pipenv/patched/notpip/_internal/exceptions.py +++ b/pipenv/patched/notpip/_internal/exceptions.py @@ -247,3 +247,22 @@ def hash_then_or(hash_name): class UnsupportedPythonVersion(InstallationError): """Unsupported python version according to Requires-Python package metadata.""" + + +class ConfigurationFileCouldNotBeLoaded(ConfigurationError): + """When there are errors while loading a configuration file + """ + + def __init__(self, reason="could not be loaded", fname=None, error=None): + super(ConfigurationFileCouldNotBeLoaded, self).__init__(error) + self.reason = reason + self.fname = fname + self.error = error + + def __str__(self): + if self.fname is not None: + message_part = " in {}.".format(self.fname) + else: + assert self.error is not None + message_part = ".\n{}\n".format(self.error.message) + return "Configuration file {}{}".format(self.reason, message_part) diff --git a/pipenv/patched/notpip/_internal/index.py b/pipenv/patched/notpip/_internal/index.py index 426880e987..b4b023739f 100644 --- a/pipenv/patched/notpip/_internal/index.py +++ b/pipenv/patched/notpip/_internal/index.py @@ -20,24 +20,27 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._internal.compat import ipaddress from pipenv.patched.notpip._internal.download import HAS_TLS, is_url, path_to_url, url_to_path from pipenv.patched.notpip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, InvalidWheelFilename, UnsupportedWheel, ) +from pipenv.patched.notpip._internal.models.candidate import InstallationCandidate +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.models.index import PyPI +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.pep425tags import get_supported +from pipenv.patched.notpip._internal.utils.compat import ipaddress from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, cached_property, normalize_path, - remove_auth_from_url, splitext, + ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, normalize_path, + remove_auth_from_url, ) from pipenv.patched.notpip._internal.utils.packaging import check_requires_python from pipenv.patched.notpip._internal.wheel import Wheel, wheel_ext -__all__ = ['FormatControl', 'fmt_ctl_handle_mutual_exclude', 'PackageFinder'] +__all__ = ['FormatControl', 'PackageFinder'] SECURE_ORIGINS = [ @@ -56,46 +59,120 @@ logger = logging.getLogger(__name__) -class InstallationCandidate(object): +def _get_content_type(url, session): + """Get the Content-Type of the given url, using a HEAD request""" + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) + if scheme not in {'http', 'https'}: + # FIXME: some warning or something? + # assertion error? + return '' - def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) - self.requires_python = requires_python + resp = session.head(url, allow_redirects=True) + resp.raise_for_status() - def __repr__(self): - return "".format( - self.project, self.version, self.location, - ) + return resp.headers.get("Content-Type", "") - def __hash__(self): - return hash(self._key) - def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) +def _handle_get_page_fail(link, reason, url, meth=None): + if meth is None: + meth = logger.debug + meth("Could not fetch URL %s: %s - skipping", link, reason) - def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) - def __eq__(self, other): - return self._compare(other, lambda s, o: s == o) +def _get_html_page(link, session=None): + if session is None: + raise TypeError( + "_get_html_page() missing 1 required keyword argument: 'session'" + ) - def __ge__(self, other): - return self._compare(other, lambda s, o: s >= o) + url = link.url + url = url.split('#', 1)[0] - def __gt__(self, other): - return self._compare(other, lambda s, o: s > o) + # Check for VCS schemes that do not support lookup as web pages. + from pipenv.patched.notpip._internal.vcs import VcsSupport + for scheme in VcsSupport.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + logger.debug('Cannot look at %s URL %s', scheme, link) + return None - def __ne__(self, other): - return self._compare(other, lambda s, o: s != o) + try: + filename = link.filename + for bad_ext in ARCHIVE_EXTENSIONS: + if filename.endswith(bad_ext): + content_type = _get_content_type(url, session=session) + if content_type.lower().startswith('text/html'): + break + else: + logger.debug( + 'Skipping page %s because of Content-Type: %s', + link, + content_type, + ) + return - def _compare(self, other, method): - if not isinstance(other, InstallationCandidate): - return NotImplemented + logger.debug('Getting page %s', url) + + # Tack index.html onto file:// URLs that point to directories + (scheme, netloc, path, params, query, fragment) = \ + urllib_parse.urlparse(url) + if (scheme == 'file' and + os.path.isdir(urllib_request.url2pathname(path))): + # add trailing slash if not present so urljoin doesn't trim + # final segment + if not url.endswith('/'): + url += '/' + url = urllib_parse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s', url) + + resp = session.get( + url, + headers={ + "Accept": "text/html", + # We don't want to blindly returned cached data for + # /simple/, because authors generally expecting that + # twine upload && pip install will function, but if + # they've done a pip install in the last ~10 minutes + # it won't. Thus by setting this to zero we will not + # blindly use any cached data, however the benefit of + # using max-age=0 instead of no-cache, is that we will + # still support conditional requests, so we will still + # minimize traffic sent in cases where the page hasn't + # changed at all, we will just always incur the round + # trip for the conditional GET now instead of only + # once per 10 minutes. + # For more information, please see pypa/pip#5670. + "Cache-Control": "max-age=0", + }, + ) + resp.raise_for_status() - return method(self._key, other._key) + # The check for archives above only works if the url ends with + # something that looks like an archive. However that is not a + # requirement of an url. Unless we issue a HEAD request on every + # url we cannot know ahead of time for sure if something is HTML + # or not. However we can check after we've downloaded it. + content_type = resp.headers.get('Content-Type', 'unknown') + if not content_type.lower().startswith("text/html"): + logger.debug( + 'Skipping page %s because of Content-Type: %s', + link, + content_type, + ) + return + + inst = HTMLPage(resp.content, resp.url, resp.headers) + except requests.HTTPError as exc: + _handle_get_page_fail(link, exc, url) + except SSLError as exc: + reason = "There was a problem confirming the ssl certificate: " + reason += str(exc) + _handle_get_page_fail(link, reason, url, meth=logger.info) + except requests.ConnectionError as exc: + _handle_get_page_fail(link, "connection error: %s" % exc, url) + except requests.Timeout: + _handle_get_page_fail(link, "timed out", url) + else: + return inst class PackageFinder(object): @@ -210,15 +287,15 @@ def get_formatted_locations(self): return "\n".join(lines) def add_dependency_links(self, links): - # # FIXME: this shouldn't be global list this, it should only - # # apply to requirements of the package that specifies the - # # dependency_links value - # # FIXME: also, we should track comes_from (i.e., use Link) + # FIXME: this shouldn't be global list this, it should only + # apply to requirements of the package that specifies the + # dependency_links value + # FIXME: also, we should track comes_from (i.e., use Link) if self.process_dependency_links: deprecated( "Dependency Links processing has been deprecated and will be " "removed in a future release.", - replacement=None, + replacement="PEP 508 URL dependencies", gone_in="18.2", issue=4187, ) @@ -242,6 +319,7 @@ def get_extras_links(links): return extras + @staticmethod def _sort_locations(locations, expand_dir=False): """ @@ -467,7 +545,7 @@ def find_all_candidates(self, project_name): logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) - formats = fmt_ctl_formats(self.format_control, canonical_name) + formats = self.format_control.get_allowed_formats(canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links @@ -483,7 +561,7 @@ def find_all_candidates(self, project_name): continue with indent_log(): page_versions.extend( - self._package_versions(page.links, search) + self._package_versions(page.iter_links(), search) ) dependency_versions = self._package_versions( @@ -626,7 +704,9 @@ def _get_pages(self, locations, project_name): try: page = self._get_page(location) - except requests.HTTPError as e: + except requests.HTTPError: + continue + if page is None: continue yield page @@ -743,7 +823,7 @@ def _link_package_versions(self, link, search, ignore_compatibility=True): return InstallationCandidate(search.supplied, version, link, link.requires_python) def _get_page(self, link): - return HTMLPage.get_page(link, session=self.session) + return _get_html_page(link, session=self.session) def egg_info_matches( @@ -763,7 +843,7 @@ def egg_info_matches( return None if search_name is None: full_match = match.group(0) - return full_match[full_match.index('-'):] + return full_match.split('-', 1)[-1] name = match.group(0).lower() # To match the "safe" name that pkg_resources creates: name = name.replace('_', '-') @@ -775,377 +855,71 @@ def egg_info_matches( return None -class HTMLPage(object): - """Represents one page, along with its URL""" - - def __init__(self, content, url, headers=None): - # Determine if we have any encoding information in our headers - encoding = None - if headers and "Content-Type" in headers: - content_type, params = cgi.parse_header(headers["Content-Type"]) +def _determine_base_url(document, page_url): + """Determine the HTML document's base URL. - if "charset" in params: - encoding = params['charset'] + This looks for a ```` tag in the HTML document. If present, its href + attribute denotes the base URL of anchor tags in the document. If there is + no such tag (or if it does not have a valid href attribute), the HTML + file's URL is used as the base URL. - self.content = content - self.parsed = html5lib.parse( - self.content, - transport_encoding=encoding, - namespaceHTMLElements=False, - ) - self.url = url - self.headers = headers - - def __str__(self): - return self.url + :param document: An HTML document representation. The current + implementation expects the result of ``html5lib.parse()``. + :param page_url: The URL of the HTML document. + """ + for base in document.findall(".//base"): + href = base.get("href") + if href is not None: + return href + return page_url - @classmethod - def get_page(cls, link, skip_archives=True, session=None): - if session is None: - raise TypeError( - "get_page() missing 1 required keyword argument: 'session'" - ) - url = link.url - url = url.split('#', 1)[0] +def _get_encoding_from_headers(headers): + """Determine if we have any encoding information in our headers. + """ + if headers and "Content-Type" in headers: + content_type, params = cgi.parse_header(headers["Content-Type"]) + if "charset" in params: + return params['charset'] + return None - # Check for VCS schemes that do not support lookup as web pages. - from pipenv.patched.notpip._internal.vcs import VcsSupport - for scheme in VcsSupport.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %s URL %s', scheme, link) - return None - try: - if skip_archives: - filename = link.filename - for bad_ext in ARCHIVE_EXTENSIONS: - if filename.endswith(bad_ext): - content_type = cls._get_content_type( - url, session=session, - ) - if content_type.lower().startswith('text/html'): - break - else: - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - logger.debug('Getting page %s', url) - - # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = \ - urllib_parse.urlparse(url) - if (scheme == 'file' and - os.path.isdir(urllib_request.url2pathname(path))): - # add trailing slash if not present so urljoin doesn't trim - # final segment - if not url.endswith('/'): - url += '/' - url = urllib_parse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s', url) - - resp = session.get( - url, - headers={ - "Accept": "text/html", - "Cache-Control": "max-age=600", - }, - ) - resp.raise_for_status() - - # The check for archives above only works if the url ends with - # something that looks like an archive. However that is not a - # requirement of an url. Unless we issue a HEAD request on every - # url we cannot know ahead of time for sure if something is HTML - # or not. However we can check after we've downloaded it. - content_type = resp.headers.get('Content-Type', 'unknown') - if not content_type.lower().startswith("text/html"): - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return +_CLEAN_LINK_RE = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) - inst = cls(resp.content, resp.url, resp.headers) - except requests.HTTPError as exc: - cls._handle_fail(link, exc, url) - except SSLError as exc: - reason = "There was a problem confirming the ssl certificate: " - reason += str(exc) - cls._handle_fail(link, reason, url, meth=logger.info) - except requests.ConnectionError as exc: - cls._handle_fail(link, "connection error: %s" % exc, url) - except requests.Timeout: - cls._handle_fail(link, "timed out", url) - else: - return inst - @staticmethod - def _handle_fail(link, reason, url, meth=None): - if meth is None: - meth = logger.debug +def _clean_link(url): + """Makes sure a link is fully encoded. That is, if a ' ' shows up in + the link, it will be rewritten to %20 (while not over-quoting + % or other characters).""" + return _CLEAN_LINK_RE.sub(lambda match: '%%%2x' % ord(match.group(0)), url) - meth("Could not fetch URL %s: %s - skipping", link, reason) - @staticmethod - def _get_content_type(url, session): - """Get the Content-Type of the given url, using a HEAD request""" - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) - if scheme not in {'http', 'https'}: - # FIXME: some warning or something? - # assertion error? - return '' - - resp = session.head(url, allow_redirects=True) - resp.raise_for_status() +class HTMLPage(object): + """Represents one page, along with its URL""" - return resp.headers.get("Content-Type", "") + def __init__(self, content, url, headers=None): + self.content = content + self.url = url + self.headers = headers - @cached_property - def base_url(self): - bases = [ - x for x in self.parsed.findall(".//base") - if x.get("href") is not None - ] - if bases and bases[0].get("href"): - return bases[0].get("href") - else: - return self.url + def __str__(self): + return self.url - @property - def links(self): + def iter_links(self): """Yields all links in the page""" - for anchor in self.parsed.findall(".//a"): + document = html5lib.parse( + self.content, + transport_encoding=_get_encoding_from_headers(self.headers), + namespaceHTMLElements=False, + ) + base_url = _determine_base_url(document, self.url) + for anchor in document.findall(".//a"): if anchor.get("href"): href = anchor.get("href") - url = self.clean_link( - urllib_parse.urljoin(self.base_url, href) - ) + url = _clean_link(urllib_parse.urljoin(base_url, href)) pyrequire = anchor.get('data-requires-python') pyrequire = unescape(pyrequire) if pyrequire else None - yield Link(url, self, requires_python=pyrequire) - - _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) - - def clean_link(self, url): - """Makes sure a link is fully encoded. That is, if a ' ' shows up in - the link, it will be rewritten to %20 (while not over-quoting - % or other characters).""" - return self._clean_re.sub( - lambda match: '%%%2x' % ord(match.group(0)), url) - - -class Link(object): - - def __init__(self, url, comes_from=None, requires_python=None): - """ - Object representing a parsed link from https://pypi.org/simple/* - - url: - url of the resource pointed to (href of the link) - comes_from: - instance of HTMLPage where the link was found, or string. - requires_python: - String containing the `Requires-Python` metadata field, specified - in PEP 345. This may be specified by a data-requires-python - attribute in the HTML link tag, as described in PEP 503. - """ - - # url can be a UNC windows share - if url.startswith('\\\\'): - url = path_to_url(url) - - self.url = url - self.comes_from = comes_from - self.requires_python = requires_python if requires_python else None - - def __str__(self): - if self.requires_python: - rp = ' (requires-python:%s)' % self.requires_python - else: - rp = '' - if self.comes_from: - return '%s (from %s)%s' % (self.url, self.comes_from, rp) - else: - return str(self.url) - - def __repr__(self): - return '' % self - - def __eq__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url == other.url - - def __ne__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url != other.url - - def __lt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url < other.url - - def __le__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url <= other.url - - def __gt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url > other.url - - def __ge__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url >= other.url - - def __hash__(self): - return hash(self.url) - - @property - def filename(self): - _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) - name = posixpath.basename(path.rstrip('/')) or netloc - name = urllib_parse.unquote(name) - assert name, ('URL %r produced no filename' % self.url) - return name - - @property - def scheme(self): - return urllib_parse.urlsplit(self.url)[0] - - @property - def netloc(self): - return urllib_parse.urlsplit(self.url)[1] - - @property - def path(self): - return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) - - def splitext(self): - return splitext(posixpath.basename(self.path.rstrip('/'))) - - @property - def ext(self): - return self.splitext()[1] - - @property - def url_without_fragment(self): - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) - return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) - - _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') - - @property - def egg_fragment(self): - match = self._egg_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') - - @property - def subdirectory_fragment(self): - match = self._subdirectory_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _hash_re = re.compile( - r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' - ) - - @property - def hash(self): - match = self._hash_re.search(self.url) - if match: - return match.group(2) - return None - - @property - def hash_name(self): - match = self._hash_re.search(self.url) - if match: - return match.group(1) - return None - - @property - def show_url(self): - return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) - - @property - def is_wheel(self): - return self.ext == wheel_ext - - @property - def is_artifact(self): - """ - Determines if this points to an actual artifact (e.g. a tarball) or if - it points to an "abstract" thing like a path or a VCS location. - """ - from pipenv.patched.notpip._internal.vcs import vcs - - if self.scheme in vcs.all_schemes: - return False - - return True - - -FormatControl = namedtuple('FormatControl', 'no_binary only_binary') -"""This object has two fields, no_binary and only_binary. - -If a field is falsy, it isn't set. If it is {':all:'}, it should match all -packages except those listed in the other field. Only one field can be set -to {':all:'} at a time. The rest of the time exact package name matches -are listed, with any given package only showing up in one field at a time. -""" - - -def fmt_ctl_handle_mutual_exclude(value, target, other): - new = value.split(',') - while ':all:' in new: - other.clear() - target.clear() - target.add(':all:') - del new[:new.index(':all:') + 1] - if ':none:' not in new: - # Without a none, we want to discard everything as :all: covers it - return - for name in new: - if name == ':none:': - target.clear() - continue - name = canonicalize_name(name) - other.discard(name) - target.add(name) - - -def fmt_ctl_formats(fmt_ctl, canonical_name): - result = {"binary", "source"} - if canonical_name in fmt_ctl.only_binary: - result.discard('source') - elif canonical_name in fmt_ctl.no_binary: - result.discard('binary') - elif ':all:' in fmt_ctl.only_binary: - result.discard('source') - elif ':all:' in fmt_ctl.no_binary: - result.discard('binary') - return frozenset(result) - - -def fmt_ctl_no_binary(fmt_ctl): - fmt_ctl_handle_mutual_exclude( - ':all:', fmt_ctl.no_binary, fmt_ctl.only_binary, - ) + yield Link(url, self.url, requires_python=pyrequire) Search = namedtuple('Search', 'supplied canonical formats') diff --git a/pipenv/patched/notpip/_internal/locations.py b/pipenv/patched/notpip/_internal/locations.py index 29c6db7962..3c7d5bd82a 100644 --- a/pipenv/patched/notpip/_internal/locations.py +++ b/pipenv/patched/notpip/_internal/locations.py @@ -10,8 +10,8 @@ from distutils import sysconfig as distutils_sysconfig from distutils.command.install import SCHEME_KEYS # type: ignore -from pipenv.patched.notpip._internal.compat import WINDOWS, expanduser from pipenv.patched.notpip._internal.utils import appdirs +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser # Application Directories USER_CACHE_DIR = appdirs.user_cache_dir("pip") diff --git a/pipenv/patched/notpip/_internal/models/candidate.py b/pipenv/patched/notpip/_internal/models/candidate.py new file mode 100644 index 0000000000..9627589eb0 --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/candidate.py @@ -0,0 +1,24 @@ +from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version + +from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin + + +class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + + def __init__(self, project, version, location, requires_python=None): + self.project = project + self.version = parse_version(version) + self.location = location + self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), + defining_class=InstallationCandidate + ) + + def __repr__(self): + return "".format( + self.project, self.version, self.location, + ) diff --git a/pipenv/patched/notpip/_internal/models/format_control.py b/pipenv/patched/notpip/_internal/models/format_control.py new file mode 100644 index 0000000000..caad3cba75 --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/format_control.py @@ -0,0 +1,62 @@ +from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name + + +class FormatControl(object): + """A helper class for controlling formats from which packages are installed. + If a field is falsy, it isn't set. If it is {':all:'}, it should match all + packages except those listed in the other field. Only one field can be set + to {':all:'} at a time. The rest of the time exact package name matches + are listed, with any given package only showing up in one field at a time. + """ + def __init__(self, no_binary=None, only_binary=None): + self.no_binary = set() if no_binary is None else no_binary + self.only_binary = set() if only_binary is None else only_binary + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "{}({}, {})".format( + self.__class__.__name__, + self.no_binary, + self.only_binary + ) + + @staticmethod + def handle_mutual_excludes(value, target, other): + new = value.split(',') + while ':all:' in new: + other.clear() + target.clear() + target.add(':all:') + del new[:new.index(':all:') + 1] + # Without a none, we want to discard everything as :all: covers it + if ':none:' not in new: + return + for name in new: + if name == ':none:': + target.clear() + continue + name = canonicalize_name(name) + other.discard(name) + target.add(name) + + def get_allowed_formats(self, canonical_name): + result = {"binary", "source"} + if canonical_name in self.only_binary: + result.discard('source') + elif canonical_name in self.no_binary: + result.discard('binary') + elif ':all:' in self.only_binary: + result.discard('source') + elif ':all:' in self.no_binary: + result.discard('binary') + return frozenset(result) + + def disallow_binaries(self): + self.handle_mutual_excludes( + ':all:', self.no_binary, self.only_binary, + ) diff --git a/pipenv/patched/notpip/_internal/models/index.py b/pipenv/patched/notpip/_internal/models/index.py index f9e8489492..0983fc9c37 100644 --- a/pipenv/patched/notpip/_internal/models/index.py +++ b/pipenv/patched/notpip/_internal/models/index.py @@ -1,15 +1,29 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -class Index(object): - def __init__(self, url): +class PackageIndex(object): + """Represents a Package Index and provides easier access to endpoints + """ + + def __init__(self, url, file_storage_domain): + super(PackageIndex, self).__init__() self.url = url self.netloc = urllib_parse.urlsplit(url).netloc - self.simple_url = self.url_to_path('simple') - self.pypi_url = self.url_to_path('pypi') + self.simple_url = self._url_for_path('simple') + self.pypi_url = self._url_for_path('pypi') + + # This is part of a temporary hack used to block installs of PyPI + # packages which depend on external urls only necessary until PyPI can + # block such packages themselves + self.file_storage_domain = file_storage_domain - def url_to_path(self, path): + def _url_for_path(self, path): return urllib_parse.urljoin(self.url, path) -PyPI = Index('https://pypi.org/') +PyPI = PackageIndex( + 'https://pypi.org/', file_storage_domain='files.pythonhosted.org' +) +TestPyPI = PackageIndex( + 'https://test.pypi.org/', file_storage_domain='test-files.pythonhosted.org' +) diff --git a/pipenv/patched/notpip/_internal/models/link.py b/pipenv/patched/notpip/_internal/models/link.py new file mode 100644 index 0000000000..686af1d0a0 --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/link.py @@ -0,0 +1,141 @@ +import posixpath +import re + +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse + +from pipenv.patched.notpip._internal.download import path_to_url +from pipenv.patched.notpip._internal.utils.misc import splitext +from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin +from pipenv.patched.notpip._internal.wheel import wheel_ext + + +class Link(KeyBasedCompareMixin): + """Represents a parsed link from a Package Index's simple URL + """ + + def __init__(self, url, comes_from=None, requires_python=None): + """ + url: + url of the resource pointed to (href of the link) + comes_from: + instance of HTMLPage where the link was found, or string. + requires_python: + String containing the `Requires-Python` metadata field, specified + in PEP 345. This may be specified by a data-requires-python + attribute in the HTML link tag, as described in PEP 503. + """ + + # url can be a UNC windows share + if url.startswith('\\\\'): + url = path_to_url(url) + + self.url = url + self.comes_from = comes_from + self.requires_python = requires_python if requires_python else None + + super(Link, self).__init__( + key=(self.url), + defining_class=Link + ) + + def __str__(self): + if self.requires_python: + rp = ' (requires-python:%s)' % self.requires_python + else: + rp = '' + if self.comes_from: + return '%s (from %s)%s' % (self.url, self.comes_from, rp) + else: + return str(self.url) + + def __repr__(self): + return '' % self + + @property + def filename(self): + _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) + name = posixpath.basename(path.rstrip('/')) or netloc + name = urllib_parse.unquote(name) + assert name, ('URL %r produced no filename' % self.url) + return name + + @property + def scheme(self): + return urllib_parse.urlsplit(self.url)[0] + + @property + def netloc(self): + return urllib_parse.urlsplit(self.url)[1] + + @property + def path(self): + return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) + + def splitext(self): + return splitext(posixpath.basename(self.path.rstrip('/'))) + + @property + def ext(self): + return self.splitext()[1] + + @property + def url_without_fragment(self): + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) + return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) + + _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') + + @property + def egg_fragment(self): + match = self._egg_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') + + @property + def subdirectory_fragment(self): + match = self._subdirectory_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _hash_re = re.compile( + r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' + ) + + @property + def hash(self): + match = self._hash_re.search(self.url) + if match: + return match.group(2) + return None + + @property + def hash_name(self): + match = self._hash_re.search(self.url) + if match: + return match.group(1) + return None + + @property + def show_url(self): + return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + + @property + def is_wheel(self): + return self.ext == wheel_ext + + @property + def is_artifact(self): + """ + Determines if this points to an actual artifact (e.g. a tarball) or if + it points to an "abstract" thing like a path or a VCS location. + """ + from pipenv.patched.notpip._internal.vcs import vcs + + if self.scheme in vcs.all_schemes: + return False + + return True diff --git a/pipenv/patched/notpip/_internal/operations/freeze.py b/pipenv/patched/notpip/_internal/operations/freeze.py index 532989bcbd..b18b98e46f 100644 --- a/pipenv/patched/notpip/_internal/operations/freeze.py +++ b/pipenv/patched/notpip/_internal/operations/freeze.py @@ -10,11 +10,13 @@ from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError from pipenv.patched.notpip._internal.exceptions import InstallationError -from pipenv.patched.notpip._internal.req import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, +) from pipenv.patched.notpip._internal.req.req_file import COMMENT_RE from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.misc import ( - dist_is_editable, get_installed_distributions, + dist_is_editable, get_installed_distributions, make_vcs_requirement_url, ) logger = logging.getLogger(__name__) @@ -99,13 +101,13 @@ def freeze( line = line[2:].strip() else: line = line[len('--editable'):].strip().lstrip('=') - line_req = InstallRequirement.from_editable( + line_req = install_req_from_editable( line, isolated=isolated, wheel_cache=wheel_cache, ) else: - line_req = InstallRequirement.from_line( + line_req = install_req_from_line( COMMENT_RE.sub('', line).strip(), isolated=isolated, wheel_cache=wheel_cache, @@ -166,7 +168,13 @@ def __init__(self, name, req, editable, comments=()): _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') @classmethod - def from_dist(cls, dist, dependency_links): + def _init_args_from_dist(cls, dist, dependency_links): + """ + Compute and return arguments (req, editable, comments) to pass to + FrozenRequirement.__init__(). + + This method is for use in FrozenRequirement.from_dist(). + """ location = os.path.normcase(os.path.abspath(dist.location)) comments = [] from pipenv.patched.notpip._internal.vcs import vcs, get_src_requirement @@ -231,12 +239,15 @@ def from_dist(cls, dist, dependency_links): else: rev = '{%s}' % date_match.group(1) editable = True - req = '%s@%s#egg=%s' % ( - svn_location, - rev, - cls.egg_name(dist) - ) - return cls(dist.project_name, req, editable, comments) + egg_name = cls.egg_name(dist) + req = make_vcs_requirement_url(svn_location, rev, egg_name) + + return (req, editable, comments) + + @classmethod + def from_dist(cls, dist, dependency_links): + args = cls._init_args_from_dist(dist, dependency_links) + return cls(dist.project_name, *args) @staticmethod def egg_name(dist): diff --git a/pipenv/patched/notpip/_internal/operations/prepare.py b/pipenv/patched/notpip/_internal/operations/prepare.py index 9ebc3ebdc8..d61270b4c9 100644 --- a/pipenv/patched/notpip/_internal/operations/prepare.py +++ b/pipenv/patched/notpip/_internal/operations/prepare.py @@ -7,7 +7,6 @@ from pipenv.patched.notpip._vendor import pkg_resources, requests from pipenv.patched.notpip._internal.build_env import BuildEnvironment -from pipenv.patched.notpip._internal.compat import expanduser from pipenv.patched.notpip._internal.download import ( is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path, ) @@ -15,6 +14,7 @@ DirectoryUrlHashUnsupported, HashUnpinned, InstallationError, PreviousBuildDirError, VcsHashUnsupported, ) +from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.hashes import MissingHashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import display_path, normalize_path, rmtree @@ -65,7 +65,7 @@ def dist(self, finder): """Return a setuptools Dist object.""" raise NotImplementedError(self.dist) - def prep_for_dist(self, finder): + def prep_for_dist(self, finder, build_isolation): """Ensure that we can get a Dist for this requirement.""" raise NotImplementedError(self.dist) @@ -93,35 +93,35 @@ def dist(self, finder): return dist def prep_for_dist(self, finder, build_isolation): - # Before calling "setup.py egg_info", we need to set-up the build - # environment. - build_requirements = self.req.get_pep_518_info() - should_isolate = build_isolation and build_requirements is not None + # Prepare for building. We need to: + # 1. Load pyproject.toml (if it exists) + # 2. Set up the build environment - if should_isolate: - # Haven't implemented PEP 517 yet, so spew a warning about it if - # build-requirements don't include setuptools and wheel. - missing_requirements = {'setuptools', 'wheel'} - { - pkg_resources.Requirement(r).key for r in build_requirements - } - if missing_requirements: - logger.warning( - "Missing build requirements in pyproject.toml for %s.", - self.req, - ) - logger.warning( - "This version of pip does not implement PEP 517 so it " - "cannot build a wheel without %s.", - " and ".join(map(repr, sorted(missing_requirements))) - ) + self.req.load_pyproject_toml() + should_isolate = self.req.use_pep517 and build_isolation + if should_isolate: # Isolate in a BuildEnvironment and install the build-time # requirements. self.req.build_env = BuildEnvironment() self.req.build_env.install_requirements( - finder, build_requirements, + finder, self.req.pyproject_requires, "Installing build dependencies" ) + missing = [] + if self.req.requirements_to_check: + check = self.req.requirements_to_check + missing = self.req.build_env.missing_requirements(check) + if missing: + logger.warning( + "Missing build requirements in pyproject.toml for %s.", + self.req, + ) + logger.warning( + "The project does not specify a build backend, and pip " + "cannot fall back to setuptools without %s.", + " and ".join(map(repr, sorted(missing))) + ) try: self.req.run_egg_info() @@ -136,7 +136,7 @@ class Installed(DistAbstraction): def dist(self, finder): return self.req.satisfied_by - def prep_for_dist(self, finder): + def prep_for_dist(self, finder, build_isolation): pass diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index 4205f6e0ba..182c1c8859 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -10,7 +10,12 @@ import warnings from collections import OrderedDict -import pipenv.patched.notpip._internal.utils.glibc +try: + import pipenv.patched.notpip._internal.utils.glibc +except ImportError: + import pipenv.patched.notpip.utils.glibc + +from pipenv.patched.notpip._internal.utils.compat import get_extension_suffixes logger = logging.getLogger(__name__) @@ -252,10 +257,9 @@ def get_supported(versions=None, noarch=False, platform=None, abis[0:0] = [abi] abi3s = set() - import imp - for suffix in imp.get_suffixes(): - if suffix[0].startswith('.abi'): - abi3s.add(suffix[0].split('.', 2)[1]) + for suffix in get_extension_suffixes(): + if suffix.startswith('.abi'): + abi3s.add(suffix.split('.', 2)[1]) abis.extend(sorted(list(abi3s))) diff --git a/pipenv/patched/notpip/_internal/pyproject.py b/pipenv/patched/notpip/_internal/pyproject.py new file mode 100644 index 0000000000..a47e0f0542 --- /dev/null +++ b/pipenv/patched/notpip/_internal/pyproject.py @@ -0,0 +1,144 @@ +from __future__ import absolute_import + +import io +import os + +from pipenv.patched.notpip._vendor import pytoml, six + +from pipenv.patched.notpip._internal.exceptions import InstallationError + + +def _is_list_of_str(obj): + return ( + isinstance(obj, list) and + all(isinstance(item, six.string_types) for item in obj) + ) + + +def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): + """Load the pyproject.toml file. + + Parameters: + use_pep517 - Has the user requested PEP 517 processing? None + means the user hasn't explicitly specified. + pyproject_toml - Location of the project's pyproject.toml file + setup_py - Location of the project's setup.py file + req_name - The name of the requirement we're processing (for + error reporting) + + Returns: + None if we should use the legacy code path, otherwise a tuple + ( + requirements from pyproject.toml, + name of PEP 517 backend, + requirements we should check are installed after setting + up the build environment + ) + """ + has_pyproject = os.path.isfile(pyproject_toml) + has_setup = os.path.isfile(setup_py) + + if has_pyproject: + with io.open(pyproject_toml, encoding="utf-8") as f: + pp_toml = pytoml.load(f) + build_system = pp_toml.get("build-system") + else: + build_system = None + + # The following cases must use PEP 517 + # We check for use_pep517 equalling False because that + # means the user explicitly requested --no-use-pep517 + if has_pyproject and not has_setup: + if use_pep517 is False: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project does not have a setup.py" + ) + use_pep517 = True + elif build_system and "build-backend" in build_system: + if use_pep517 is False: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project specifies a build backend of {} " + "in pyproject.toml".format( + build_system["build-backend"] + ) + ) + use_pep517 = True + + # If we haven't worked out whether to use PEP 517 yet, + # and the user hasn't explicitly stated a preference, + # we do so if the project has a pyproject.toml file. + elif use_pep517 is None: + use_pep517 = has_pyproject + + # At this point, we know whether we're going to use PEP 517. + assert use_pep517 is not None + + # If we're using the legacy code path, there is nothing further + # for us to do here. + if not use_pep517: + return None + + if build_system is None: + # Either the user has a pyproject.toml with no build-system + # section, or the user has no pyproject.toml, but has opted in + # explicitly via --use-pep517. + # In the absence of any explicit backend specification, we + # assume the setuptools backend, and require wheel and a version + # of setuptools that supports that backend. + build_system = { + "requires": ["setuptools>=38.2.5", "wheel"], + "build-backend": "setuptools.build_meta", + } + + # If we're using PEP 517, we have build system information (either + # from pyproject.toml, or defaulted by the code above). + # Note that at this point, we do not know if the user has actually + # specified a backend, though. + assert build_system is not None + + # Ensure that the build-system section in pyproject.toml conforms + # to PEP 518. + error_template = ( + "{package} has a pyproject.toml file that does not comply " + "with PEP 518: {reason}" + ) + + # Specifying the build-system table but not the requires key is invalid + if "requires" not in build_system: + raise InstallationError( + error_template.format(package=req_name, reason=( + "it has a 'build-system' table but not " + "'build-system.requires' which is mandatory in the table" + )) + ) + + # Error out if requires is not a list of strings + requires = build_system["requires"] + if not _is_list_of_str(requires): + raise InstallationError(error_template.format( + package=req_name, + reason="'build-system.requires' is not a list of strings.", + )) + + backend = build_system.get("build-backend") + check = [] + if backend is None: + # If the user didn't specify a backend, we assume they want to use + # the setuptools backend. But we can't be sure they have included + # a version of setuptools which supplies the backend, or wheel + # (which is neede by the backend) in their requirements. So we + # make a note to check that those requirements are present once + # we have set up the environment. + # TODO: Review this - it's quite a lot of work to check for a very + # specific case. The problem is, that case is potentially quite + # common - projects that adopted PEP 518 early for the ability to + # specify requirements to execute setup.py, but never considered + # needing to mention the build tools themselves. The original PEP + # 518 code had a similar check (but implemented in a different + # way). + backend = "setuptools.build_meta" + check = ["setuptools>=38.2.5", "wheel"] + + return (requires, backend, check) diff --git a/pipenv/patched/notpip/_internal/req/constructors.py b/pipenv/patched/notpip/_internal/req/constructors.py new file mode 100644 index 0000000000..9fe28d88a0 --- /dev/null +++ b/pipenv/patched/notpip/_internal/req/constructors.py @@ -0,0 +1,298 @@ +"""Backing implementation for InstallRequirement's various constructors + +The idea here is that these formed a major chunk of InstallRequirement's size +so, moving them and support code dedicated to them outside of that class +helps creates for better understandability for the rest of the code. + +These are meant to be used elsewhere within pip to create instances of +InstallRequirement. +""" + +import logging +import os +import re +import traceback + +from pipenv.patched.notpip._vendor.packaging.markers import Marker +from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pipenv.patched.notpip._vendor.packaging.specifiers import Specifier +from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError, parse_requirements + +from pipenv.patched.notpip._internal.download import ( + is_archive_file, is_url, path_to_url, url_to_path, +) +from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.models.index import PyPI, TestPyPI +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.utils.misc import is_installable_dir +from pipenv.patched.notpip._internal.vcs import vcs +from pipenv.patched.notpip._internal.wheel import Wheel + +__all__ = [ + "install_req_from_editable", "install_req_from_line", + "parse_editable" +] + +logger = logging.getLogger(__name__) +operators = Specifier._operators.keys() + + +def _strip_extras(path): + m = re.match(r'^(.+)(\[[^\]]+\])$', path) + extras = None + if m: + path_no_extras = m.group(1) + extras = m.group(2) + else: + path_no_extras = path + + return path_no_extras, extras + + +def parse_editable(editable_req): + """Parses an editable requirement into: + - a requirement name + - an URL + - extras + - editable options + Accepted requirements: + svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir + .[some_extra] + """ + + url = editable_req + + # If a file path is specified with extras, strip off the extras. + url_no_extras, extras = _strip_extras(url) + + if os.path.isdir(url_no_extras): + if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): + raise InstallationError( + "Directory %r is not installable. File 'setup.py' not found." % + url_no_extras + ) + # Treating it as code that has already been checked out + url_no_extras = path_to_url(url_no_extras) + + if url_no_extras.lower().startswith('file:'): + package_name = Link(url_no_extras).egg_fragment + if extras: + return ( + package_name, + url_no_extras, + Requirement("placeholder" + extras.lower()).extras, + ) + else: + return package_name, url_no_extras, None + + for version_control in vcs: + if url.lower().startswith('%s:' % version_control): + url = '%s+%s' % (version_control, url) + break + + if '+' not in url: + raise InstallationError( + '%s should either be a path to a local project or a VCS url ' + 'beginning with svn+, git+, hg+, or bzr+' % + editable_req + ) + + vc_type = url.split('+', 1)[0].lower() + + if not vcs.get_backend(vc_type): + error_message = 'For --editable=%s only ' % editable_req + \ + ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ + ' is currently supported' + raise InstallationError(error_message) + + package_name = Link(url).egg_fragment + if not package_name: + raise InstallationError( + "Could not detect requirement name for '%s', please specify one " + "with #egg=your_package_name" % editable_req + ) + return package_name, url, None + + +def deduce_helpful_msg(req): + """Returns helpful msg in case requirements file does not exist, + or cannot be parsed. + + :params req: Requirements file path + """ + msg = "" + if os.path.exists(req): + msg = " It does exist." + # Try to parse and check if it is a requirements file. + try: + with open(req, 'r') as fp: + # parse first line only + next(parse_requirements(fp.read())) + msg += " The argument you provided " + \ + "(%s) appears to be a" % (req) + \ + " requirements file. If that is the" + \ + " case, use the '-r' flag to install" + \ + " the packages specified within it." + except RequirementParseError: + logger.debug("Cannot parse '%s' as requirements \ + file" % (req), exc_info=1) + else: + msg += " File '%s' does not exist." % (req) + return msg + + +# ---- The actual constructors follow ---- + + +def install_req_from_editable( + editable_req, comes_from=None, isolated=False, options=None, + wheel_cache=None, constraint=False +): + name, url, extras_override = parse_editable(editable_req) + if url.startswith('file:'): + source_dir = url_to_path(url) + else: + source_dir = None + + if name is not None: + try: + req = Requirement(name) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % name) + else: + req = None + return InstallRequirement( + req, comes_from, source_dir=source_dir, + editable=True, + link=Link(url), + constraint=constraint, + isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + extras=extras_override or (), + ) + + +def install_req_from_line( + name, comes_from=None, isolated=False, options=None, wheel_cache=None, + constraint=False +): + """Creates an InstallRequirement from a name, which might be a + requirement, directory containing 'setup.py', filename, or URL. + """ + if is_url(name): + marker_sep = '; ' + else: + marker_sep = ';' + if marker_sep in name: + name, markers = name.split(marker_sep, 1) + markers = markers.strip() + if not markers: + markers = None + else: + markers = Marker(markers) + else: + markers = None + name = name.strip() + req = None + path = os.path.normpath(os.path.abspath(name)) + link = None + extras = None + + if is_url(name): + link = Link(name) + else: + p, extras = _strip_extras(path) + looks_like_dir = os.path.isdir(p) and ( + os.path.sep in name or + (os.path.altsep is not None and os.path.altsep in name) or + name.startswith('.') + ) + if looks_like_dir: + if not is_installable_dir(p): + raise InstallationError( + "Directory %r is not installable. Neither 'setup.py' " + "nor 'pyproject.toml' found." % name + ) + link = Link(path_to_url(p)) + elif is_archive_file(p): + if not os.path.isfile(p): + logger.warning( + 'Requirement %r looks like a filename, but the ' + 'file does not exist', + name + ) + link = Link(path_to_url(p)) + + # it's a local file, dir, or url + if link: + # Handle relative file URLs + if link.scheme == 'file' and re.search(r'\.\./', link.url): + link = Link( + path_to_url(os.path.normpath(os.path.abspath(link.path)))) + # wheel file + if link.is_wheel: + wheel = Wheel(link.filename) # can raise InvalidWheelFilename + req = "%s==%s" % (wheel.name, wheel.version) + else: + # set the req to the egg fragment. when it's not there, this + # will become an 'unnamed' requirement + req = link.egg_fragment + + # a requirement specifier + else: + req = name + + if extras: + extras = Requirement("placeholder" + extras.lower()).extras + else: + extras = () + if req is not None: + try: + req = Requirement(req) + except InvalidRequirement: + if os.path.sep in req: + add_msg = "It looks like a path." + add_msg += deduce_helpful_msg(req) + elif '=' in req and not any(op in req for op in operators): + add_msg = "= is not a valid operator. Did you mean == ?" + else: + add_msg = traceback.format_exc() + raise InstallationError( + "Invalid requirement: '%s'\n%s" % (req, add_msg) + ) + + return InstallRequirement( + req, comes_from, link=link, markers=markers, + isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + constraint=constraint, + extras=extras, + ) + + +def install_req_from_req( + req, comes_from=None, isolated=False, wheel_cache=None +): + try: + req = Requirement(req) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % req) + + domains_not_allowed = [ + PyPI.file_storage_domain, + TestPyPI.file_storage_domain, + ] + if req.url and comes_from.link.netloc in domains_not_allowed: + # Explicitly disallow pypi packages that depend on external urls + raise InstallationError( + "Packages installed from PyPI cannot depend on packages " + "which are not also hosted on PyPI.\n" + "%s depends on %s " % (comes_from.name, req) + ) + + return InstallRequirement( + req, comes_from, isolated=isolated, wheel_cache=wheel_cache + ) diff --git a/pipenv/patched/notpip/_internal/req/req_file.py b/pipenv/patched/notpip/_internal/req/req_file.py index 66a580224a..5f23cd3ac4 100644 --- a/pipenv/patched/notpip/_internal/req/req_file.py +++ b/pipenv/patched/notpip/_internal/req/req_file.py @@ -13,10 +13,12 @@ from pipenv.patched.notpip._vendor.six.moves import filterfalse from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal import cmdoptions +from pipenv.patched.notpip._internal.cli import cmdoptions from pipenv.patched.notpip._internal.download import get_file_content from pipenv.patched.notpip._internal.exceptions import RequirementsFileParseError -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, +) __all__ = ['parse_requirements'] @@ -151,7 +153,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, for dest in SUPPORTED_OPTIONS_REQ_DEST: if dest in opts.__dict__ and opts.__dict__[dest]: req_options[dest] = opts.__dict__[dest] - yield InstallRequirement.from_line( + yield install_req_from_line( args_str, line_comes_from, constraint=constraint, isolated=isolated, options=req_options, wheel_cache=wheel_cache ) @@ -159,7 +161,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # yield an editable requirement elif opts.editables: isolated = options.isolated_mode if options else False - yield InstallRequirement.from_editable( + yield install_req_from_editable( opts.editables[0], comes_from=line_comes_from, constraint=constraint, isolated=isolated, wheel_cache=wheel_cache ) diff --git a/pipenv/patched/notpip/_internal/req/req_install.py b/pipenv/patched/notpip/_internal/req/req_install.py index a9c642c0e9..3f32892cf5 100644 --- a/pipenv/patched/notpip/_internal/req/req_install.py +++ b/pipenv/patched/notpip/_internal/req/req_install.py @@ -1,66 +1,46 @@ from __future__ import absolute_import -import io import logging import os -import re import shutil import sys import sysconfig -import traceback import zipfile from distutils.util import change_root -from email.parser import FeedParser # type: ignore -from pipenv.patched.notpip._vendor import pkg_resources, pytoml, six -from pipenv.patched.notpip._vendor.packaging import specifiers -from pipenv.patched.notpip._vendor.packaging.markers import Marker -from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pipenv.patched.notpip._vendor import pkg_resources, six +from pipenv.patched.notpip._vendor.packaging.requirements import Requirement from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.packaging.version import Version from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version -from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError, parse_requirements +from pipenv.patched.notpip._vendor.pep517.wrappers import Pep517HookCaller from pipenv.patched.notpip._internal import wheel from pipenv.patched.notpip._internal.build_env import NoOpBuildEnvironment -from pipenv.patched.notpip._internal.compat import native_str -from pipenv.patched.notpip._internal.download import ( - is_archive_file, is_url, path_to_url, url_to_path, -) from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml from pipenv.patched.notpip._internal.req.req_uninstall import UninstallPathSet +from pipenv.patched.notpip._internal.utils.compat import native_str from pipenv.patched.notpip._internal.utils.hashes import Hashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( _make_build_dir, ask_path_exists, backup_dir, call_subprocess, display_path, dist_in_site_packages, dist_in_usersite, ensure_dir, - get_installed_version, is_installable_dir, read_text_file, rmtree, + get_installed_version, rmtree, ) +from pipenv.patched.notpip._internal.utils.packaging import get_metadata from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.utils.ui import open_spinner from pipenv.patched.notpip._internal.vcs import vcs -from pipenv.patched.notpip._internal.wheel import Wheel, move_wheel_files +from pipenv.patched.notpip._internal.wheel import move_wheel_files logger = logging.getLogger(__name__) -operators = specifiers.Specifier._operators.keys() - - -def _strip_extras(path): - m = re.match(r'^(.+)(\[[^\]]+\])$', path) - extras = None - if m: - path_no_extras = m.group(1) - extras = m.group(2) - else: - path_no_extras = path - - return path_no_extras, extras - class InstallRequirement(object): """ @@ -87,7 +67,6 @@ def __init__(self, req, comes_from, source_dir=None, editable=False, if link is not None: self.link = self.original_link = link else: - from pipenv.patched.notpip._internal.index import Link self.link = self.original_link = req and req.url and Link(req.url) if extras: @@ -128,155 +107,32 @@ def __init__(self, req, comes_from, source_dir=None, editable=False, self.isolated = isolated self.build_env = NoOpBuildEnvironment() - # Constructors - # TODO: Move these out of this class into custom methods. - @classmethod - def from_editable(cls, editable_req, comes_from=None, isolated=False, - options=None, wheel_cache=None, constraint=False): - from pipenv.patched.notpip._internal.index import Link - - name, url, extras_override = parse_editable(editable_req) - if url.startswith('file:'): - source_dir = url_to_path(url) - else: - source_dir = None - - if name is not None: - try: - req = Requirement(name) - except InvalidRequirement: - raise InstallationError("Invalid requirement: '%s'" % name) - else: - req = None - return cls( - req, comes_from, source_dir=source_dir, - editable=True, - link=Link(url), - constraint=constraint, - isolated=isolated, - options=options if options else {}, - wheel_cache=wheel_cache, - extras=extras_override or (), - ) - - @classmethod - def from_req(cls, req, comes_from=None, isolated=False, wheel_cache=None): - try: - req = Requirement(req) - except InvalidRequirement: - raise InstallationError("Invalid requirement: '%s'" % req) - if req.url: - raise InstallationError( - "Direct url requirement (like %s) are not allowed for " - "dependencies" % req - ) - return cls(req, comes_from, isolated=isolated, wheel_cache=wheel_cache) - - @classmethod - def from_line( - cls, name, comes_from=None, isolated=False, options=None, - wheel_cache=None, constraint=False): - """Creates an InstallRequirement from a name, which might be a - requirement, directory containing 'setup.py', filename, or URL. - """ - from pipenv.patched.notpip._internal.index import Link + # The static build requirements (from pyproject.toml) + self.pyproject_requires = None - if is_url(name): - marker_sep = '; ' - else: - marker_sep = ';' - if marker_sep in name: - name, markers = name.split(marker_sep, 1) - markers = markers.strip() - if not markers: - markers = None - else: - markers = Marker(markers) - else: - markers = None - name = name.strip() - req = None - path = os.path.normpath(os.path.abspath(name)) - link = None - extras = None - - if is_url(name): - link = Link(name) - else: - p, extras = _strip_extras(path) - looks_like_dir = os.path.isdir(p) and ( - os.path.sep in name or - (os.path.altsep is not None and os.path.altsep in name) or - name.startswith('.') - ) - if looks_like_dir: - if not is_installable_dir(p): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' " - "not found." % name - ) - link = Link(path_to_url(p)) - elif is_archive_file(p): - if not os.path.isfile(p): - logger.warning( - 'Requirement %r looks like a filename, but the ' - 'file does not exist', - name - ) - link = Link(path_to_url(p)) - - # it's a local file, dir, or url - if link: - # Handle relative file URLs - if link.scheme == 'file' and re.search(r'\.\./', link.url): - link = Link( - path_to_url(os.path.normpath(os.path.abspath(link.path)))) - # wheel file - if link.is_wheel: - wheel = Wheel(link.filename) # can raise InvalidWheelFilename - req = "%s==%s" % (wheel.name, wheel.version) - else: - # set the req to the egg fragment. when it's not there, this - # will become an 'unnamed' requirement - req = link.egg_fragment + # Build requirements that we will check are available + # TODO: We don't do this for --no-build-isolation. Should we? + self.requirements_to_check = [] - # a requirement specifier - else: - req = name + # The PEP 517 backend we should use to build the project + self.pep517_backend = None - if extras: - extras = Requirement("placeholder" + extras.lower()).extras - else: - extras = () - if req is not None: - try: - req = Requirement(req) - except InvalidRequirement: - if os.path.sep in req: - add_msg = "It looks like a path." - add_msg += deduce_helpful_msg(req) - elif '=' in req and not any(op in req for op in operators): - add_msg = "= is not a valid operator. Did you mean == ?" - else: - add_msg = traceback.format_exc() - raise InstallationError( - "Invalid requirement: '%s'\n%s" % (req, add_msg)) - return cls( - req, comes_from, link=link, markers=markers, - isolated=isolated, - options=options if options else {}, - wheel_cache=wheel_cache, - constraint=constraint, - extras=extras, - ) + # Are we using PEP 517 for this requirement? + # After pyproject.toml has been loaded, the only valid values are True + # and False. Before loading, None is valid (meaning "use the default"). + # Setting an explicit value before loading pyproject.toml is supported, + # but after loading this flag should be treated as read only. + self.use_pep517 = None def __str__(self): if self.req: s = str(self.req) if self.link: s += ' from %s' % self.link.url + elif self.link: + s = self.link.url else: - s = self.link.url if self.link else None + s = '' if self.satisfied_by is not None: s += ' in %s' % display_path(self.satisfied_by.location) if self.comes_from: @@ -429,7 +285,7 @@ def _correct_build_location(self): package is not available until we run egg_info, so the build_location will return a temporary directory and store the _ideal_build_dir. - This is only called by self.egg_info_path to fix the temporary build + This is only called by self.run_egg_info to fix the temporary build directory. """ if self.source_dir is not None: @@ -557,48 +413,29 @@ def pyproject_toml(self): return pp_toml - def get_pep_518_info(self): - """Get PEP 518 build-time requirements. + def load_pyproject_toml(self): + """Load the pyproject.toml file. - Returns the list of the packages required to build the project, - specified as per PEP 518 within the package. If `pyproject.toml` is not - present, returns None to signify not using the same. + After calling this routine, all of the attributes related to PEP 517 + processing for this requirement have been set. In particular, the + use_pep517 attribute can be used to determine whether we should + follow the PEP 517 or legacy (setup.py) code path. """ - # If pyproject.toml does not exist, don't do anything. - if not os.path.isfile(self.pyproject_toml): - return None - - error_template = ( - "{package} has a pyproject.toml file that does not comply " - "with PEP 518: {reason}" + pep517_data = load_pyproject_toml( + self.use_pep517, + self.pyproject_toml, + self.setup_py, + str(self) ) - with io.open(self.pyproject_toml, encoding="utf-8") as f: - pp_toml = pytoml.load(f) - - # If there is no build-system table, just use setuptools and wheel. - if "build-system" not in pp_toml: - return ["setuptools", "wheel"] - - # Specifying the build-system table but not the requires key is invalid - build_system = pp_toml["build-system"] - if "requires" not in build_system: - raise InstallationError( - error_template.format(package=self, reason=( - "it has a 'build-system' table but not " - "'build-system.requires' which is mandatory in the table" - )) - ) - - # Error out if it's not a list of strings - requires = build_system["requires"] - if not _is_list_of_str(requires): - raise InstallationError(error_template.format( - package=self, - reason="'build-system.requires' is not a list of strings.", - )) - - return requires + if pep517_data is None: + self.use_pep517 = False + else: + self.use_pep517 = True + requires, backend, check = pep517_data + self.requirements_to_check = check + self.pyproject_requires = requires + self.pep517_backend = Pep517HookCaller(self.setup_py_dir, backend) def run_egg_info(self): assert self.source_dir @@ -615,7 +452,8 @@ def run_egg_info(self): with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py - base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + base_cmd = [sys_executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] @@ -636,20 +474,20 @@ def run_egg_info(self): command_desc='python setup.py egg_info') if not self.req: - if isinstance(parse_version(self.pkg_info()["Version"]), Version): + if isinstance(parse_version(self.metadata["Version"]), Version): op = "==" else: op = "===" self.req = Requirement( "".join([ - self.pkg_info()["Name"], + self.metadata["Name"], op, - self.pkg_info()["Version"], + self.metadata["Version"], ]) ) self._correct_build_location() else: - metadata_name = canonicalize_name(self.pkg_info()["Name"]) + metadata_name = canonicalize_name(self.metadata["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( 'Running setup.py (path:%s) egg_info for package %s ' @@ -659,19 +497,8 @@ def run_egg_info(self): ) self.req = Requirement(metadata_name) - def egg_info_data(self, filename): - if self.satisfied_by is not None: - if not self.satisfied_by.has_metadata(filename): - return None - return self.satisfied_by.get_metadata(filename) - assert self.source_dir - filename = self.egg_info_path(filename) - if not os.path.exists(filename): - return None - data = read_text_file(filename) - return data - - def egg_info_path(self, filename): + @property + def egg_info_path(self): if self._egg_info_path is None: if self.editable: base = self.source_dir @@ -709,8 +536,7 @@ def egg_info_path(self, filename): if not filenames: raise InstallationError( - "Files/directories (from %s) not found in %s" - % (filename, base) + "Files/directories not found in %s" % base ) # if we have more than one match, we pick the toplevel one. This # can easily be the case if there is a dist folder which contains @@ -721,24 +547,18 @@ def egg_info_path(self, filename): (os.path.altsep and x.count(os.path.altsep) or 0) ) self._egg_info_path = os.path.join(base, filenames[0]) - return os.path.join(self._egg_info_path, filename) + return self._egg_info_path - def pkg_info(self): - p = FeedParser() - data = self.egg_info_data('PKG-INFO') - if not data: - logger.warning( - 'No PKG-INFO file found in %s', - display_path(self.egg_info_path('PKG-INFO')), - ) - p.feed(data or '') - return p.close() + @property + def metadata(self): + if not hasattr(self, '_metadata'): + self._metadata = get_metadata(self.get_dist()) - _requirements_section_re = re.compile(r'\[(.*?)\]') + return self._metadata def get_dist(self): """Return a pkg_resources.Distribution built from self.egg_info_path""" - egg_info = self.egg_info_path('').rstrip(os.path.sep) + egg_info = self.egg_info_path.rstrip(os.path.sep) base_dir = os.path.dirname(egg_info) metadata = pkg_resources.PathMetadata(base_dir, egg_info) dist_name = os.path.splitext(os.path.basename(egg_info))[0] @@ -750,7 +570,7 @@ def get_dist(self): def assert_source_matches_version(self): assert self.source_dir - version = self.pkg_info()['version'] + version = self.metadata['version'] if self.req.specifier and version not in self.req.specifier: logger.warning( 'Requested %s, but installing version %s', @@ -794,10 +614,11 @@ def install_editable(self, install_options, with indent_log(): # FIXME: should we do --install-headers here too? + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: call_subprocess( [ - os.environ.get('PIP_PYTHON_PATH', sys.executable), + sys_executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + @@ -877,7 +698,7 @@ def _clean_zip_name(self, name, prefix): # only used by archive. def archive(self, build_dir): assert self.source_dir create_archive = True - archive_name = '%s-%s.zip' % (self.name, self.pkg_info()["version"]) + archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) archive_path = os.path.join(build_dir, archive_name) if os.path.exists(archive_path): response = ask_path_exists( @@ -1015,7 +836,8 @@ def prepend_root(path): def get_install_args(self, global_options, record_filename, root, prefix, pycompile): - install_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable), "-u"] + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + install_args = [sys_executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ @@ -1039,104 +861,3 @@ def get_install_args(self, global_options, record_filename, root, prefix, py_ver_str, self.name)] return install_args - - -def parse_editable(editable_req): - """Parses an editable requirement into: - - a requirement name - - an URL - - extras - - editable options - Accepted requirements: - svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir - .[some_extra] - """ - - from pipenv.patched.notpip._internal.index import Link - - url = editable_req - - # If a file path is specified with extras, strip off the extras. - url_no_extras, extras = _strip_extras(url) - - if os.path.isdir(url_no_extras): - if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' not found." % - url_no_extras - ) - # Treating it as code that has already been checked out - url_no_extras = path_to_url(url_no_extras) - - if url_no_extras.lower().startswith('file:'): - package_name = Link(url_no_extras).egg_fragment - if extras: - return ( - package_name, - url_no_extras, - Requirement("placeholder" + extras.lower()).extras, - ) - else: - return package_name, url_no_extras, None - - for version_control in vcs: - if url.lower().startswith('%s:' % version_control): - url = '%s+%s' % (version_control, url) - break - - if '+' not in url: - raise InstallationError( - '%s should either be a path to a local project or a VCS url ' - 'beginning with svn+, git+, hg+, or bzr+' % - editable_req - ) - - vc_type = url.split('+', 1)[0].lower() - - if not vcs.get_backend(vc_type): - error_message = 'For --editable=%s only ' % editable_req + \ - ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ - ' is currently supported' - raise InstallationError(error_message) - - package_name = Link(url).egg_fragment - if not package_name: - raise InstallationError( - "Could not detect requirement name for '%s', please specify one " - "with #egg=your_package_name" % editable_req - ) - return package_name, url, None - - -def deduce_helpful_msg(req): - """Returns helpful msg in case requirements file does not exist, - or cannot be parsed. - - :params req: Requirements file path - """ - msg = "" - if os.path.exists(req): - msg = " It does exist." - # Try to parse and check if it is a requirements file. - try: - with open(req, 'r') as fp: - # parse first line only - next(parse_requirements(fp.read())) - msg += " The argument you provided " + \ - "(%s) appears to be a" % (req) + \ - " requirements file. If that is the" + \ - " case, use the '-r' flag to install" + \ - " the packages specified within it." - except RequirementParseError: - logger.debug("Cannot parse '%s' as requirements \ - file" % (req), exc_info=1) - else: - msg += " File '%s' does not exist." % (req) - return msg - - -def _is_list_of_str(obj): - return ( - isinstance(obj, list) and - all(isinstance(item, six.string_types) for item in obj) - ) diff --git a/pipenv/patched/notpip/_internal/req/req_set.py b/pipenv/patched/notpip/_internal/req/req_set.py index 2c54c85a9f..a65851ffe8 100644 --- a/pipenv/patched/notpip/_internal/req/req_set.py +++ b/pipenv/patched/notpip/_internal/req/req_set.py @@ -12,19 +12,22 @@ class RequirementSet(object): - def __init__(self, require_hashes=False, ignore_compatibility=True): + def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): """Create a RequirementSet. """ self.requirements = OrderedDict() self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels + if ignore_compatibility: + self.check_supported_wheels = False + self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False # Mapping of alias: real_name self.requirement_aliases = {} self.unnamed_requirements = [] self.successfully_downloaded = [] self.reqs_to_cleanup = [] - self.ignore_compatibility = ignore_compatibility def __str__(self): reqs = [req for req in self.requirements.values() @@ -56,17 +59,22 @@ def add_requirement(self, install_req, parent_req_name=None, requirement is applicable and has just been added. """ name = install_req.name + + # If the markers do not match, ignore this requirement. if not install_req.match_markers(extras_requested): - logger.info("Ignoring %s: markers '%s' don't match your " - "environment", install_req.name, - install_req.markers) + logger.info( + "Ignoring %s: markers '%s' don't match your environment", + name, install_req.markers, + ) return [], None - # This check has to come after we filter requirements with the - # environment markers. + # If the wheel is not supported, raise an error. + # Should check this after filtering out based on environment markers to + # allow specifying different wheels based on the environment/OS, in a + # single requirements file. if install_req.link and install_req.link.is_wheel: wheel = Wheel(install_req.link.filename) - if not wheel.supported() and not self.ignore_compatibility: + if self.check_supported_wheels and not wheel.supported(): raise InstallationError( "%s is not a supported wheel on this platform." % wheel.filename @@ -78,59 +86,73 @@ def add_requirement(self, install_req, parent_req_name=None, "a non direct req should have a parent" ) + # Unnamed requirements are scanned again and the requirement won't be + # added as a dependency until after scanning. if not name: # url or path requirement w/o an egg fragment self.unnamed_requirements.append(install_req) return [install_req], None - else: - try: - existing_req = self.get_requirement(name) - except KeyError: - existing_req = None - if (parent_req_name is None and existing_req and not - existing_req.constraint and - existing_req.extras == install_req.extras and not - existing_req.req.specifier == install_req.req.specifier): - raise InstallationError( - 'Double requirement given: %s (already in %s, name=%r)' - % (install_req, existing_req, name)) - if not existing_req: - # Add requirement - self.requirements[name] = install_req - # FIXME: what about other normalizations? E.g., _ vs. -? - if name.lower() != name: - self.requirement_aliases[name.lower()] = name - result = [install_req] - else: - # Assume there's no need to scan, and that we've already - # encountered this for scanning. - result = [] - if not install_req.constraint and existing_req.constraint: - if (install_req.link and not (existing_req.link and - install_req.link.path == existing_req.link.path)): - self.reqs_to_cleanup.append(install_req) - raise InstallationError( - "Could not satisfy constraints for '%s': " - "installation from path or url cannot be " - "constrained to a version" % name, - ) - # If we're now installing a constraint, mark the existing - # object for real installation. - existing_req.constraint = False - existing_req.extras = tuple( - sorted(set(existing_req.extras).union( - set(install_req.extras)))) - logger.debug("Setting %s extras to: %s", - existing_req, existing_req.extras) - # And now we need to scan this. - result = [existing_req] - # Canonicalise to the already-added object for the backref - # check below. - install_req = existing_req - - # We return install_req here to allow for the caller to add it to - # the dependency information for the parent package. - return result, install_req + + try: + existing_req = self.get_requirement(name) + except KeyError: + existing_req = None + + has_conflicting_requirement = ( + parent_req_name is None and + existing_req and + not existing_req.constraint and + existing_req.extras == install_req.extras and + existing_req.req.specifier != install_req.req.specifier + ) + if has_conflicting_requirement: + raise InstallationError( + "Double requirement given: %s (already in %s, name=%r)" + % (install_req, existing_req, name) + ) + + # When no existing requirement exists, add the requirement as a + # dependency and it will be scanned again after. + if not existing_req: + self.requirements[name] = install_req + # FIXME: what about other normalizations? E.g., _ vs. -? + if name.lower() != name: + self.requirement_aliases[name.lower()] = name + # We'd want to rescan this requirements later + return [install_req], install_req + + # Assume there's no need to scan, and that we've already + # encountered this for scanning. + if install_req.constraint or not existing_req.constraint: + return [], existing_req + + does_not_satisfy_constraint = ( + install_req.link and + not ( + existing_req.link and + install_req.link.path == existing_req.link.path + ) + ) + if does_not_satisfy_constraint: + self.reqs_to_cleanup.append(install_req) + raise InstallationError( + "Could not satisfy constraints for '%s': " + "installation from path or url cannot be " + "constrained to a version" % name, + ) + # If we're now installing a constraint, mark the existing + # object for real installation. + existing_req.constraint = False + existing_req.extras = tuple(sorted( + set(existing_req.extras) | set(install_req.extras) + )) + logger.debug( + "Setting %s extras to: %s", + existing_req, existing_req.extras, + ) + # Return the existing requirement for addition to the parent and + # scanning again. + return [existing_req], existing_req def has_requirement(self, project_name): name = project_name.lower() @@ -152,7 +174,7 @@ def get_requirement(self, project_name): return self.requirements[name] if name in self.requirement_aliases: return self.requirements[self.requirement_aliases[name]] - # raise KeyError("No project with the name %r" % project_name) + pass def cleanup_files(self): """Clean up files, remove builds.""" diff --git a/pipenv/patched/notpip/_internal/req/req_uninstall.py b/pipenv/patched/notpip/_internal/req/req_uninstall.py index 3ccd3265b6..4cd15d8446 100644 --- a/pipenv/patched/notpip/_internal/req/req_uninstall.py +++ b/pipenv/patched/notpip/_internal/req/req_uninstall.py @@ -9,9 +9,9 @@ from pipenv.patched.notpip._vendor import pkg_resources -from pipenv.patched.notpip._internal.compat import WINDOWS, cache_from_source, uses_pycache from pipenv.patched.notpip._internal.exceptions import UninstallationError from pipenv.patched.notpip._internal.locations import bin_py, bin_user +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, cache_from_source, uses_pycache from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local, @@ -120,6 +120,8 @@ def compress_for_output_listing(paths): folders.add(os.path.dirname(path)) files.add(path) + _normcased_files = set(map(os.path.normcase, files)) + folders = compact(folders) # This walks the tree using os.walk to not miss extra folders @@ -130,8 +132,9 @@ def compress_for_output_listing(paths): if fname.endswith(".pyc"): continue - file_ = os.path.normcase(os.path.join(dirpath, fname)) - if os.path.isfile(file_) and file_ not in files: + file_ = os.path.join(dirpath, fname) + if (os.path.isfile(file_) and + os.path.normcase(file_) not in _normcased_files): # We are skipping this file. Add it to the set. will_skip.add(file_) diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py index 461a2bb879..b0d096f9d2 100644 --- a/pipenv/patched/notpip/_internal/resolve.py +++ b/pipenv/patched/notpip/_internal/resolve.py @@ -18,7 +18,7 @@ BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, UnsupportedPythonVersion, ) -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import install_req_from_req from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import dist_in_usersite, ensure_dir from pipenv.patched.notpip._internal.utils.packaging import check_dist_requires_python @@ -249,9 +249,6 @@ def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=F # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) - if ignore_requires_python or self.ignore_requires_python: - self.ignore_compatibility = True - if req_to_install.constraint or req_to_install.prepared: return [] @@ -267,7 +264,7 @@ def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=F try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_compatibility: + if self.ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -281,7 +278,7 @@ def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=F more_reqs = [] def add_req(subreq, extras_requested): - sub_install_req = InstallRequirement.from_req( + sub_install_req = install_req_from_req( str(subreq), req_to_install, isolated=self.isolated, @@ -303,10 +300,10 @@ def add_req(subreq, extras_requested): # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) - # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -338,7 +335,7 @@ def add_req(subreq, extras_requested): for available in available_requested: if hasattr(dist, '_DistInfoDistribution__dep_map'): for req in dist._DistInfoDistribution__dep_map[available]: - req = InstallRequirement.from_req( + req = install_req_from_req( str(req), req_to_install, isolated=self.isolated, diff --git a/pipenv/patched/notpip/_internal/utils/appdirs.py b/pipenv/patched/notpip/_internal/utils/appdirs.py index 291de7a95d..e8e145266e 100644 --- a/pipenv/patched/notpip/_internal/utils/appdirs.py +++ b/pipenv/patched/notpip/_internal/utils/appdirs.py @@ -9,7 +9,7 @@ from pipenv.patched.notpip._vendor.six import PY2, text_type -from pipenv.patched.notpip._internal.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser def user_cache_dir(appname): diff --git a/pipenv/patched/notpip/_internal/compat.py b/pipenv/patched/notpip/_internal/utils/compat.py similarity index 96% rename from pipenv/patched/notpip/_internal/compat.py rename to pipenv/patched/notpip/_internal/utils/compat.py index 6e51e32a73..483bfdc89f 100644 --- a/pipenv/patched/notpip/_internal/compat.py +++ b/pipenv/patched/notpip/_internal/utils/compat.py @@ -25,6 +25,7 @@ __all__ = [ "ipaddress", "uses_pycache", "console_to_str", "native_str", "get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size", + "get_extension_suffixes", ] @@ -160,6 +161,18 @@ def get_path_uid(path): return file_uid +if sys.version_info >= (3, 4): + from importlib.machinery import EXTENSION_SUFFIXES + + def get_extension_suffixes(): + return EXTENSION_SUFFIXES +else: + from imp import get_suffixes + + def get_extension_suffixes(): + return [suffix[0] for suffix in get_suffixes()] + + def expanduser(path): """ Expand ~ and ~user constructions. diff --git a/pipenv/patched/notpip/_internal/utils/filesystem.py b/pipenv/patched/notpip/_internal/utils/filesystem.py index 91976486dc..e8d6a2bb4c 100644 --- a/pipenv/patched/notpip/_internal/utils/filesystem.py +++ b/pipenv/patched/notpip/_internal/utils/filesystem.py @@ -1,7 +1,7 @@ import os import os.path -from pipenv.patched.notpip._internal.compat import get_path_uid +from pipenv.patched.notpip._internal.utils.compat import get_path_uid def check_path_owner(path): diff --git a/pipenv/patched/notpip/_internal/utils/logging.py b/pipenv/patched/notpip/_internal/utils/logging.py index 257a62346c..576c4fa098 100644 --- a/pipenv/patched/notpip/_internal/utils/logging.py +++ b/pipenv/patched/notpip/_internal/utils/logging.py @@ -5,7 +5,7 @@ import logging.handlers import os -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.misc import ensure_dir try: diff --git a/pipenv/patched/notpip/_internal/utils/misc.py b/pipenv/patched/notpip/_internal/utils/misc.py index e254f3d436..45e5204c16 100644 --- a/pipenv/patched/notpip/_internal/utils/misc.py +++ b/pipenv/patched/notpip/_internal/utils/misc.py @@ -26,14 +26,14 @@ from pipenv.patched.notpip._vendor.six.moves import input from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal.compat import ( - WINDOWS, console_to_str, expanduser, stdlib_pkgs, -) from pipenv.patched.notpip._internal.exceptions import CommandError, InstallationError from pipenv.patched.notpip._internal.locations import ( running_under_virtualenv, site_packages, user_site, virtualenv_no_global, write_delete_marker_file, ) +from pipenv.patched.notpip._internal.utils.compat import ( + WINDOWS, console_to_str, expanduser, stdlib_pkgs, +) if PY2: from io import BytesIO as StringIO @@ -96,7 +96,7 @@ def get_prog(): try: prog = os.path.basename(sys.argv[0]) if prog in ('__main__.py', '-c'): - return "%s -m pip" % os.environ.get('PIP_PYTHON_PATH', sys.executable) + return "%s -m pip" % sys.executable else: return prog except (AttributeError, TypeError, IndexError): @@ -187,12 +187,16 @@ def format_size(bytes): def is_installable_dir(path): - """Return True if `path` is a directory containing a setup.py file.""" + """Is path is a directory containing setup.py or pyproject.toml? + """ if not os.path.isdir(path): return False setup_py = os.path.join(path, 'setup.py') if os.path.isfile(setup_py): return True + pyproject_toml = os.path.join(path, 'pyproject.toml') + if os.path.isfile(pyproject_toml): + return True return False @@ -852,6 +856,44 @@ def enum(*sequential, **named): return type('Enum', (), enums) +def make_vcs_requirement_url(repo_url, rev, egg_project_name, subdir=None): + """ + Return the URL for a VCS requirement. + + Args: + repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + """ + req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) + if subdir: + req += '&subdirectory={}'.format(subdir) + + return req + + +def split_auth_from_netloc(netloc): + """ + Parse out and remove the auth information from a netloc. + + Returns: (netloc, (username, password)). + """ + if '@' not in netloc: + return netloc, (None, None) + + # Split from the right because that's how urllib.parse.urlsplit() + # behaves if more than one @ is present (which can be checked using + # the password attribute of urlsplit()'s return value). + auth, netloc = netloc.rsplit('@', 1) + if ':' in auth: + # Split from the left because that's how urllib.parse.urlsplit() + # behaves if more than one : is present (which again can be checked + # using the password attribute of the return value) + user_pass = tuple(auth.split(':', 1)) + else: + user_pass = auth, None + + return netloc, user_pass + + def remove_auth_from_url(url): # Return a copy of url with 'username:password@' removed. # username/pass params are passed to subversion through flags @@ -859,12 +901,11 @@ def remove_auth_from_url(url): # parsed url purl = urllib_parse.urlsplit(url) - stripped_netloc = \ - purl.netloc.split('@')[-1] + netloc, user_pass = split_auth_from_netloc(purl.netloc) # stripped url url_pieces = ( - purl.scheme, stripped_netloc, purl.path, purl.query, purl.fragment + purl.scheme, netloc, purl.path, purl.query, purl.fragment ) surl = urllib_parse.urlunsplit(url_pieces) return surl diff --git a/pipenv/patched/notpip/_internal/utils/models.py b/pipenv/patched/notpip/_internal/utils/models.py new file mode 100644 index 0000000000..d5cb80a7cb --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/models.py @@ -0,0 +1,40 @@ +"""Utilities for defining models +""" + +import operator + + +class KeyBasedCompareMixin(object): + """Provides comparision capabilities that is based on a key + """ + + def __init__(self, key, defining_class): + self._compare_key = key + self._defining_class = defining_class + + def __hash__(self): + return hash(self._compare_key) + + def __lt__(self, other): + return self._compare(other, operator.__lt__) + + def __le__(self, other): + return self._compare(other, operator.__le__) + + def __gt__(self, other): + return self._compare(other, operator.__gt__) + + def __ge__(self, other): + return self._compare(other, operator.__ge__) + + def __eq__(self, other): + return self._compare(other, operator.__eq__) + + def __ne__(self, other): + return self._compare(other, operator.__ne__) + + def _compare(self, other, method): + if not isinstance(other, self._defining_class): + return NotImplemented + + return method(self._compare_key, other._compare_key) diff --git a/pipenv/patched/notpip/_internal/utils/outdated.py b/pipenv/patched/notpip/_internal/utils/outdated.py index 6133e6fd0a..f8b1fe04bb 100644 --- a/pipenv/patched/notpip/_internal/utils/outdated.py +++ b/pipenv/patched/notpip/_internal/utils/outdated.py @@ -9,8 +9,8 @@ from pipenv.patched.notpip._vendor import lockfile, pkg_resources from pipenv.patched.notpip._vendor.packaging import version as packaging_version -from pipenv.patched.notpip._internal.compat import WINDOWS from pipenv.patched.notpip._internal.index import PackageFinder +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ensure_dir, get_installed_version @@ -22,16 +22,25 @@ class SelfCheckState(object): def __init__(self, cache_dir): - self.statefile_path = os.path.join(cache_dir, "selfcheck.json") + self.state = {} + self.statefile_path = None - # Load the existing state - try: - with open(self.statefile_path) as statefile: - self.state = json.load(statefile)[sys.prefix] - except (IOError, ValueError, KeyError): - self.state = {} + # Try to load the existing state + if cache_dir: + self.statefile_path = os.path.join(cache_dir, "selfcheck.json") + try: + with open(self.statefile_path) as statefile: + self.state = json.load(statefile)[sys.prefix] + except (IOError, ValueError, KeyError): + # Explicitly suppressing exceptions, since we don't want to + # error out if the cache file is invalid. + pass def save(self, pypi_version, current_time): + # If we do not have a path to cache in, don't bother saving. + if not self.statefile_path: + return + # Check to make sure that we own the directory if not check_path_owner(os.path.dirname(self.statefile_path)): return diff --git a/pipenv/patched/notpip/_internal/utils/packaging.py b/pipenv/patched/notpip/_internal/utils/packaging.py index 1354774374..d1e8ecaa5b 100644 --- a/pipenv/patched/notpip/_internal/utils/packaging.py +++ b/pipenv/patched/notpip/_internal/utils/packaging.py @@ -8,6 +8,7 @@ from pipenv.patched.notpip._vendor.packaging import specifiers, version from pipenv.patched.notpip._internal import exceptions +from pipenv.patched.notpip._internal.utils.misc import display_path logger = logging.getLogger(__name__) @@ -35,22 +36,31 @@ def check_requires_python(requires_python): def get_metadata(dist): if (isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata('METADATA')): - return dist.get_metadata('METADATA') + metadata = dist.get_metadata('METADATA') elif dist.has_metadata('PKG-INFO'): - return dist.get_metadata('PKG-INFO') + metadata = dist.get_metadata('PKG-INFO') + else: + logger.warning("No metadata found in %s", display_path(dist.location)) + metadata = '' - -def check_dist_requires_python(dist, absorb=True): - metadata = get_metadata(dist) feed_parser = FeedParser() feed_parser.feed(metadata) - pkg_info_dict = feed_parser.close() + return feed_parser.close() + + +def check_dist_requires_python(dist, absorb=True): + pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') - if not absorb: + if absorb: return requires_python try: if not check_requires_python(requires_python): - return requires_python + raise exceptions.UnsupportedPythonVersion( + "%s requires Python '%s' but the running Python is %s" % ( + dist.project_name, + requires_python, + '.'.join(map(str, sys.version_info[:3])),) + ) except specifiers.InvalidSpecifier as e: logger.warning( "Package %s has an invalid Requires-Python entry %s - %s", diff --git a/pipenv/patched/notpip/_internal/utils/ui.py b/pipenv/patched/notpip/_internal/utils/ui.py index b96863cc44..6eebd17d54 100644 --- a/pipenv/patched/notpip/_internal/utils/ui.py +++ b/pipenv/patched/notpip/_internal/utils/ui.py @@ -15,7 +15,7 @@ from pipenv.patched.notpip._vendor.progress.helpers import HIDE_CURSOR, SHOW_CURSOR, WritelnMixin from pipenv.patched.notpip._vendor.progress.spinner import Spinner -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.logging import get_indentation from pipenv.patched.notpip._internal.utils.misc import format_size from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/pipenv/patched/notpip/_internal/vcs/__init__.py b/pipenv/patched/notpip/_internal/vcs/__init__.py index 146f2829d3..5aeac6330e 100644 --- a/pipenv/patched/notpip/_internal/vcs/__init__.py +++ b/pipenv/patched/notpip/_internal/vcs/__init__.py @@ -17,7 +17,7 @@ if MYPY_CHECK_RUNNING: from typing import Dict, Optional, Tuple # noqa: F401 - from pipenv.patched.notpip._internal.basecommand import Command # noqa: F401 + from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 __all__ = ['vcs', 'get_src_requirement'] @@ -200,12 +200,6 @@ def _is_local_repository(self, repo): drive, tail = os.path.splitdrive(repo) return repo.startswith(os.path.sep) or drive - # See issue #1083 for why this method was introduced: - # https://github.com/pypa/pip/issues/1083 - def translate_egg_surname(self, surname): - # For example, Django has branches of the form "stable/1.7.x". - return surname.replace('/', '_') - def export(self, location): """ Export the repository at the url to the destination location @@ -213,51 +207,65 @@ def export(self, location): """ raise NotImplementedError - def get_url_rev(self, url): + def get_netloc_and_auth(self, netloc, scheme): + """ + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Args: + netloc: the original repository URL netloc. + scheme: the repository URL's scheme without the vcs prefix. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). """ - Returns the correct repository URL and revision by parsing the given - repository URL + return netloc, (None, None) + + def get_url_rev_and_auth(self, url): + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). """ - error_message = ( - "Sorry, '%s' is a malformed VCS url. " - "The format is +://, " - "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp" - ) - assert '+' in url, error_message % url - url = url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + if '+' not in scheme: + raise ValueError( + "Sorry, {!r} is a malformed VCS url. " + "The format is +://, " + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + ) + # Remove the vcs prefix. + scheme = scheme.split('+', 1)[1] + netloc, user_pass = self.get_netloc_and_auth(netloc, scheme) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev + return url, rev, user_pass - def get_url_rev_args(self, url): + def make_rev_args(self, username, password): """ - Return the URL and RevOptions "extra arguments" to use in obtain(), - as a tuple (url, extra_args). + Return the RevOptions "extra arguments" to use in obtain(). """ - return url, [] + return [] def get_url_rev_options(self, url): """ Return the URL and RevOptions object to use in obtain() and in some cases export(), as a tuple (url, rev_options). """ - url, rev = self.get_url_rev(url) - url, extra_args = self.get_url_rev_args(url) + url, rev, user_pass = self.get_url_rev_and_auth(url) + username, password = user_pass + extra_args = self.make_rev_args(username, password) rev_options = self.make_rev_options(rev, extra_args=extra_args) return url, rev_options - def get_info(self, location): - """ - Returns (url, revision), where both are strings - """ - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - return self.get_url(location), self.get_revision(location) - def normalize_url(self, url): """ Normalize a URL for comparison by unquoting it and removing any @@ -291,7 +299,7 @@ def switch(self, dest, url, rev_options): """ raise NotImplementedError - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): """ Update an already-existing repo to the given ``rev_options``. @@ -341,7 +349,7 @@ def obtain(self, dest): self.repo_name, rev_display, ) - self.update(dest, rev_options) + self.update(dest, url, rev_options) else: logger.info('Skipping because already up-to-date.') return @@ -421,8 +429,6 @@ def get_src_requirement(self, dist, location): def get_url(self, location): """ Return the url used at location - - This is used in get_info() and obtain(). """ raise NotImplementedError diff --git a/pipenv/patched/notpip/_internal/vcs/bazaar.py b/pipenv/patched/notpip/_internal/vcs/bazaar.py index b2664cd842..890448edc6 100644 --- a/pipenv/patched/notpip/_internal/vcs/bazaar.py +++ b/pipenv/patched/notpip/_internal/vcs/bazaar.py @@ -6,7 +6,9 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import display_path, rmtree +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, rmtree, +) from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -62,16 +64,16 @@ def fetch_new(self, dest, url, rev_options): def switch(self, dest, url, rev_options): self.run_command(['switch', url], cwd=dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): cmd_args = ['pull', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev = super(Bazaar, self).get_url_rev(url) + url, rev, user_pass = super(Bazaar, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'bzr+' + url - return url, rev + return url, rev, user_pass def get_url(self, location): urls = self.run_command(['info'], show_stdout=False, cwd=location) @@ -98,9 +100,9 @@ def get_src_requirement(self, dist, location): return None if not repo.lower().startswith('bzr:'): repo = 'bzr+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] current_rev = self.get_revision(location) - return '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) + egg_project_name = dist.egg_name().split('-', 1)[0] + return make_vcs_requirement_url(repo, current_rev, egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/git.py b/pipenv/patched/notpip/_internal/vcs/git.py index ef2dd908b0..3db561444c 100644 --- a/pipenv/patched/notpip/_internal/vcs/git.py +++ b/pipenv/patched/notpip/_internal/vcs/git.py @@ -8,9 +8,9 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._internal.compat import samefile from pipenv.patched.notpip._internal.exceptions import BadCommand -from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.compat import samefile +from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -77,6 +77,20 @@ def get_git_version(self): version = '.'.join(version.split('.')[:3]) return parse_version(version) + def get_branch(self, location): + """ + Return the current branch, or None if HEAD isn't at a branch + (e.g. detached HEAD). + """ + args = ['rev-parse', '--abbrev-ref', 'HEAD'] + output = self.run_command(args, show_stdout=False, cwd=location) + branch = output.strip() + + if branch == 'HEAD': + return None + + return branch + def export(self, location): """Export the Git repository at the url to the destination location""" if not location.endswith('/'): @@ -91,8 +105,8 @@ def export(self, location): def get_revision_sha(self, dest, rev): """ - Return a commit hash for the given revision if it names a remote - branch or tag. Otherwise, return None. + Return (sha_or_none, is_branch), where sha_or_none is a commit hash + if the revision names a remote branch or tag, otherwise None. Args: dest: the repository directory. @@ -115,22 +129,30 @@ def get_revision_sha(self, dest, rev): branch_ref = 'refs/remotes/origin/{}'.format(rev) tag_ref = 'refs/tags/{}'.format(rev) - return refs.get(branch_ref) or refs.get(tag_ref) + sha = refs.get(branch_ref) + if sha is not None: + return (sha, True) - def check_rev_options(self, dest, rev_options): - """Check the revision options before checkout. + sha = refs.get(tag_ref) - Returns a new RevOptions object for the SHA1 of the branch or tag - if found. + return (sha, False) + + def resolve_revision(self, dest, url, rev_options): + """ + Resolve a revision to a new RevOptions object with the SHA1 of the + branch, tag, or ref if found. Args: rev_options: a RevOptions object. """ rev = rev_options.arg_rev - sha = self.get_revision_sha(dest, rev) + sha, is_branch = self.get_revision_sha(dest, rev) if sha is not None: - return rev_options.make_new(sha) + rev_options = rev_options.make_new(sha) + rev_options.branch_name = rev if is_branch else None + + return rev_options # Do not show a warning for the common case of something that has # the form of a Git commit hash. @@ -139,6 +161,19 @@ def check_rev_options(self, dest, rev_options): "Did not find branch or tag '%s', assuming revision or ref.", rev, ) + + if not rev.startswith('refs/'): + return rev_options + + # If it looks like a ref, we have to fetch it explicitly. + self.run_command( + ['fetch', '-q', url] + rev_options.to_args(), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + sha = self.get_revision(dest, rev='FETCH_HEAD') + rev_options = rev_options.make_new(sha) + return rev_options def is_commit_id_equal(self, dest, name): @@ -164,20 +199,22 @@ def fetch_new(self, dest, url, rev_options): if rev_options.rev: # Then a specific revision was requested. - rev_options = self.check_rev_options(dest, rev_options) - # Only do a checkout if the current commit id doesn't match - # the requested revision. - if not self.is_commit_id_equal(dest, rev_options.rev): - rev = rev_options.rev - # Only fetch the revision if it's a ref - if rev.startswith('refs/'): - self.run_command( - ['fetch', '-q', url] + rev_options.to_args(), - cwd=dest, - ) - # Change the revision to the SHA of the ref we fetched - rev = 'FETCH_HEAD' - self.run_command(['checkout', '-q', rev], cwd=dest) + rev_options = self.resolve_revision(dest, url, rev_options) + branch_name = getattr(rev_options, 'branch_name', None) + if branch_name is None: + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + cmd_args = ['checkout', '-q'] + rev_options.to_args() + self.run_command(cmd_args, cwd=dest) + elif self.get_branch(dest) != branch_name: + # Then a specific branch was requested, and that branch + # is not yet checked out. + track_branch = 'origin/{}'.format(branch_name) + cmd_args = [ + 'checkout', '-b', branch_name, '--track', track_branch, + ] + self.run_command(cmd_args, cwd=dest) #: repo may contain submodules self.update_submodules(dest) @@ -189,7 +226,7 @@ def switch(self, dest, url, rev_options): self.update_submodules(dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): # First fetch changes from the default remote if self.get_git_version() >= parse_version('1.9.0'): # fetch tags in addition to everything else @@ -197,7 +234,7 @@ def update(self, dest, rev_options): else: self.run_command(['fetch', '-q'], cwd=dest) # Then reset to wanted revision (maybe even origin/master) - rev_options = self.check_rev_options(dest, rev_options) + rev_options = self.resolve_revision(dest, url, rev_options) cmd_args = ['reset', '--hard', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) #: update submodules @@ -218,9 +255,11 @@ def get_url(self, location): url = found_remote.split(' ')[1] return url.strip() - def get_revision(self, location): + def get_revision(self, location, rev=None): + if rev is None: + rev = 'HEAD' current_rev = self.run_command( - ['rev-parse', 'HEAD'], show_stdout=False, cwd=location, + ['rev-parse', rev], show_stdout=False, cwd=location, ) return current_rev.strip() @@ -255,17 +294,15 @@ def get_src_requirement(self, dist, location): repo = self.get_url(location) if not repo.lower().startswith('git:'): repo = 'git+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None current_rev = self.get_revision(location) - req = '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) - subdirectory = self._get_subdirectory(location) - if subdirectory: - req += '&subdirectory=' + subdirectory + egg_project_name = dist.egg_name().split('-', 1)[0] + subdir = self._get_subdirectory(location) + req = make_vcs_requirement_url(repo, current_rev, egg_project_name, + subdir=subdir) + return req - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): """ Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. That's required because although they use SSH they sometimes don't @@ -275,12 +312,12 @@ def get_url_rev(self, url): if '://' not in url: assert 'file:' not in url url = url.replace('git+', 'git+ssh://') - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) url = url.replace('ssh://', '') else: - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) - return url, rev + return url, rev, user_pass def update_submodules(self, location): if not os.path.exists(os.path.join(location, '.gitmodules')): diff --git a/pipenv/patched/notpip/_internal/vcs/mercurial.py b/pipenv/patched/notpip/_internal/vcs/mercurial.py index a143e765f9..d76d47f387 100644 --- a/pipenv/patched/notpip/_internal/vcs/mercurial.py +++ b/pipenv/patched/notpip/_internal/vcs/mercurial.py @@ -6,7 +6,7 @@ from pipenv.patched.notpip._vendor.six.moves import configparser from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -59,7 +59,7 @@ def switch(self, dest, url, rev_options): cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): self.run_command(['pull', '-q'], cwd=dest) cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) @@ -88,11 +88,10 @@ def get_src_requirement(self, dist, location): repo = self.get_url(location) if not repo.lower().startswith('hg:'): repo = 'hg+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None current_rev_hash = self.get_revision_hash(location) - return '%s@%s#egg=%s' % (repo, current_rev_hash, egg_project_name) + egg_project_name = dist.egg_name().split('-', 1)[0] + return make_vcs_requirement_url(repo, current_rev_hash, + egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/subversion.py b/pipenv/patched/notpip/_internal/vcs/subversion.py index 5adbdaa33b..f3c3db4d95 100644 --- a/pipenv/patched/notpip/_internal/vcs/subversion.py +++ b/pipenv/patched/notpip/_internal/vcs/subversion.py @@ -4,17 +4,15 @@ import os import re -from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse - -from pipenv.patched.notpip._internal.index import Link +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log -from pipenv.patched.notpip._internal.utils.misc import display_path, remove_auth_from_url, rmtree +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, rmtree, split_auth_from_netloc, +) from pipenv.patched.notpip._internal.vcs import VersionControl, vcs _svn_xml_url_re = re.compile('url="([^"]+)"') _svn_rev_re = re.compile(r'committed-rev="(\d+)"') -_svn_url_re = re.compile(r'URL: (.+)') -_svn_revision_re = re.compile(r'Revision: (.+)') _svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') _svn_info_xml_url_re = re.compile(r'(.*)') @@ -31,34 +29,6 @@ class Subversion(VersionControl): def get_base_rev_args(self, rev): return ['-r', rev] - def get_info(self, location): - """Returns (url, revision), where both are strings""" - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - output = self.run_command( - ['info', location], - show_stdout=False, - extra_environ={'LANG': 'C'}, - ) - match = _svn_url_re.search(output) - if not match: - logger.warning( - 'Cannot determine URL of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return None, None - url = match.group(1).strip() - match = _svn_revision_re.search(output) - if not match: - logger.warning( - 'Cannot determine revision of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return url, None - return url, match.group(1) - def export(self, location): """Export the svn repository at the url to the destination location""" url, rev_options = self.get_url_rev_options(self.url) @@ -87,7 +57,7 @@ def switch(self, dest, url, rev_options): cmd_args = ['switch'] + rev_options.to_args() + [url, dest] self.run_command(cmd_args) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): cmd_args = ['update'] + rev_options.to_args() + [dest] self.run_command(cmd_args) @@ -132,18 +102,34 @@ def get_revision(self, location): revision = max(revision, localrev) return revision - def get_url_rev(self, url): + def get_netloc_and_auth(self, netloc, scheme): + """ + This override allows the auth information to be passed to svn via the + --username and --password options instead of via the URL. + """ + if scheme == 'ssh': + # The --username and --password options can't be used for + # svn+ssh URLs, so keep the auth information in the URL. + return super(Subversion, self).get_netloc_and_auth( + netloc, scheme) + + return split_auth_from_netloc(netloc) + + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev = super(Subversion, self).get_url_rev(url) + url, rev, user_pass = super(Subversion, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'svn+' + url - return url, rev + return url, rev, user_pass - def get_url_rev_args(self, url): - extra_args = get_rev_options_args(url) - url = remove_auth_from_url(url) + def make_rev_args(self, username, password): + extra_args = [] + if username: + extra_args += ['--username', username] + if password: + extra_args += ['--password', password] - return url, extra_args + return extra_args def get_url(self, location): # In cases where the source is in a subdirectory, not alongside @@ -213,42 +199,15 @@ def get_src_requirement(self, dist, location): repo = self.get_url(location) if repo is None: return None + repo = 'svn+' + repo + rev = self.get_revision(location) # FIXME: why not project name? egg_project_name = dist.egg_name().split('-', 1)[0] - rev = self.get_revision(location) - return 'svn+%s@%s#egg=%s' % (repo, rev, egg_project_name) + return make_vcs_requirement_url(repo, rev, egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" return False -def get_rev_options_args(url): - """ - Return the extra arguments to pass to RevOptions. - """ - r = urllib_parse.urlsplit(url) - if hasattr(r, 'username'): - # >= Python-2.5 - username, password = r.username, r.password - else: - netloc = r[1] - if '@' in netloc: - auth = netloc.split('@')[0] - if ':' in auth: - username, password = auth.split(':', 1) - else: - username, password = auth, None - else: - username, password = None, None - - extra_args = [] - if username: - extra_args += ['--username', username] - if password: - extra_args += ['--password', password] - - return extra_args - - vcs.register(Subversion) diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py index 14ec00143c..6df5a3a3bb 100644 --- a/pipenv/patched/notpip/_internal/wheel.py +++ b/pipenv/patched/notpip/_internal/wheel.py @@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): ] # If an executable sits with sys.executable, we don't warn for it. # This covers the case of venv invocations without activating the venv. - not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) + executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) + not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) warn_for = { parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() if os.path.normcase(parent_dir) not in not_warn_dirs @@ -475,7 +476,7 @@ def _get_script_text(entry): if warn_script_location: msg = message_about_scripts_not_on_PATH(generated_console_scripts) if msg is not None: - logger.warn(msg) + logger.warning(msg) if len(gui) > 0: generated.extend( @@ -500,16 +501,19 @@ def _get_script_text(entry): with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) writer = csv.writer(record_out) + outrows = [] for row in reader: row[0] = installed.pop(row[0], row[0]) if row[0] in changed: row[1], row[2] = rehash(row[0]) - writer.writerow(row) + outrows.append(tuple(row)) for f in generated: digest, length = rehash(f) - writer.writerow((normpath(f, lib_dir), digest, length)) + outrows.append((normpath(f, lib_dir), digest, length)) for f in installed: - writer.writerow((installed[f], '', '')) + outrows.append((installed[f], '', '')) + for row in sorted(outrows): + writer.writerow(row) shutil.move(temp_record, record) @@ -664,8 +668,9 @@ def _base_setup_args(self, req): # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. + executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) return [ - os.environ.get('PIP_PYTHON_PATH', sys.executable), '-u', '-c', + executable_loc, '-u', '-c', SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) @@ -710,6 +715,7 @@ def build(self, requirements, session, autobuilding=False): :return: True if all the wheels built correctly. """ from pipenv.patched.notpip._internal import index + from pipenv.patched.notpip._internal.models.link import Link building_is_possible = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir @@ -717,6 +723,7 @@ def build(self, requirements, session, autobuilding=False): assert building_is_possible buildset = [] + format_control = self.finder.format_control for req in requirements: if req.constraint: continue @@ -740,8 +747,7 @@ def build(self, requirements, session, autobuilding=False): if index.egg_info_matches(base, None, link) is None: # E.g. local directory. Build wheel just for this run. ephem_cache = True - if "binary" not in index.fmt_ctl_formats( - self.finder.format_control, + if "binary" not in format_control.get_allowed_formats( canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " @@ -802,7 +808,7 @@ def build(self, requirements, session, autobuilding=False): self.preparer.build_dir ) # Update the link for this. - req.link = index.Link(path_to_url(wheel_file)) + req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( diff --git a/pipenv/patched/notpip/_vendor/certifi/__init__.py b/pipenv/patched/notpip/_vendor/certifi/__init__.py index 0c4963ef60..aa329fbb4b 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__init__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.04.16" +__version__ = "2018.08.24" diff --git a/pipenv/patched/notpip/_vendor/certifi/__main__.py b/pipenv/patched/notpip/_vendor/certifi/__main__.py index 5f1da0dd0c..983ed0f9b2 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__main__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__main__.py @@ -1,2 +1,2 @@ -from certifi import where +from pipenv.patched.notpip._vendor.certifi import where print(where()) diff --git a/pipenv/patched/notpip/_vendor/certifi/cacert.pem b/pipenv/patched/notpip/_vendor/certifi/cacert.pem index 2713f541c4..85de024e71 100644 --- a/pipenv/patched/notpip/_vendor/certifi/cacert.pem +++ b/pipenv/patched/notpip/_vendor/certifi/cacert.pem @@ -3692,169 +3692,6 @@ lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR -----END CERTIFICATE----- -# Issuer: CN=Certplus Root CA G1 O=Certplus -# Subject: CN=Certplus Root CA G1 O=Certplus -# Label: "Certplus Root CA G1" -# Serial: 1491911565779898356709731176965615564637713 -# MD5 Fingerprint: 7f:09:9c:f7:d9:b9:5c:69:69:56:d5:37:3e:14:0d:42 -# SHA1 Fingerprint: 22:fd:d0:b7:fd:a2:4e:0d:ac:49:2c:a0:ac:a6:7b:6a:1f:e3:f7:66 -# SHA256 Fingerprint: 15:2a:40:2b:fc:df:2c:d5:48:05:4d:22:75:b3:9c:7f:ca:3e:c0:97:80:78:b0:f0:ea:76:e5:61:a6:c7:43:3e ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUA -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBa -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB -ANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHNr49a -iZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt -6kuJPKNxQv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP -0FG7Yn2ksYyy/yARujVjBYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f -6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTvLRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDE -EW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2z4QTd28n6v+WZxcIbekN -1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc4nBvCGrc -h2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCT -mehd4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV -4EJQeIQEQWGw9CEjjy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPO -WftwenMGE9nTdDckQQoRb5fc5+R+ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1Ud -DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSowcCbkahDFXxd -Bie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHYlwuBsTANBgkq -hkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh -66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7 -/SMNkPX0XtPGYX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BS -S7CTKtQ+FjPlnsZlFT5kOwQ/2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j -2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F6ALEUz65noe8zDUa3qHpimOHZR4R -Kttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilXCNQ314cnrUlZp5Gr -RHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWetUNy -6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEV -V/xuZDDCVRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5 -g4VCXA9DO2pJNdWY9BW/+mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl -++O/QmueD6i9a5jc2NvLi6Td11n0bt3+qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo= ------END CERTIFICATE----- - -# Issuer: CN=Certplus Root CA G2 O=Certplus -# Subject: CN=Certplus Root CA G2 O=Certplus -# Label: "Certplus Root CA G2" -# Serial: 1492087096131536844209563509228951875861589 -# MD5 Fingerprint: a7:ee:c4:78:2d:1b:ee:2d:b9:29:ce:d6:a7:96:32:31 -# SHA1 Fingerprint: 4f:65:8e:1f:e9:06:d8:28:02:e9:54:47:41:c9:54:25:5d:69:cc:1a -# SHA256 Fingerprint: 6c:c0:50:41:e6:44:5e:74:69:6c:4c:fb:c9:f8:0f:54:3b:7e:ab:bb:44:b4:ce:6f:78:7c:6a:99:71:c4:2f:17 ------BEGIN CERTIFICATE----- -MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABM0PW1aC3/BFGtat -93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uNAm8x -Ik0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0P -AQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwj -FNiPwyCrKGBZMB8GA1UdIwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqG -SM49BAMDA2gAMGUCMHD+sAvZ94OX7PNVHdTcswYO/jOYnYs5kGuUIe22113WTNch -p+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjlvPl5adytRSv3tjFzzAal -U5ORGpOucGpnutee5WEaXw== ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G1 O=OpenTrust -# Subject: CN=OpenTrust Root CA G1 O=OpenTrust -# Label: "OpenTrust Root CA G1" -# Serial: 1492036577811947013770400127034825178844775 -# MD5 Fingerprint: 76:00:cc:81:29:cd:55:5e:88:6a:7a:2e:f7:4d:39:da -# SHA1 Fingerprint: 79:91:e8:34:f7:e2:ee:dd:08:95:01:52:e9:55:2d:14:e9:58:d5:7e -# SHA256 Fingerprint: 56:c7:71:28:d9:8c:18:d9:1b:4c:fd:ff:bc:25:ee:91:03:d4:75:8e:a2:ab:ad:82:6a:90:f3:45:7d:46:0e:b4 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcxMB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7faYp6b -wiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX -/uMftk87ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR0 -77F9jAHiOH3BX2pfJLKOYheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGP -uY4zbGneWK2gDqdkVBFpRGZPTBKnjix9xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLx -p2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO9z0M+Yo0FMT7MzUj8czx -Kselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq3ywgsNw2 -TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+W -G+Oin6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPw -vFEVVJSmdz7QdFG9URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYY -EQRVzXR7z2FwefR7LFxckvzluFqrTJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUl0YhVyE1 -2jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/PxN3DlCPaTKbYw -DQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E -PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kf -gLMtMrpkZ2CvuVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbS -FXJfLkur1J1juONI5f6ELlgKn0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0 -V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLhX4SPgPL0DTatdrOjteFkdjpY3H1P -XlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80nR14SohWZ25g/4/I -i+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcmGS3t -TAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L91 -09S5zvE/bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/Ky -Pu1svf0OnWZzsD2097+o4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJ -AwSQiumPv+i2tCqjI40cHLI5kqiPAlxAOXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj -1oxx ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G2 O=OpenTrust -# Subject: CN=OpenTrust Root CA G2 O=OpenTrust -# Label: "OpenTrust Root CA G2" -# Serial: 1492012448042702096986875987676935573415441 -# MD5 Fingerprint: 57:24:b6:59:24:6b:ae:c8:fe:1c:0c:20:f2:c0:4e:eb -# SHA1 Fingerprint: 79:5f:88:60:c5:ab:7c:3d:92:e6:cb:f4:8d:e1:45:cd:11:ef:60:0b -# SHA256 Fingerprint: 27:99:58:29:fe:6a:75:15:c1:bf:e8:48:f9:c4:76:1d:b1:6c:22:59:29:25:7b:f4:0d:08:94:f2:9e:a8:ba:f2 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcyMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+Ntmh -/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78e -CbY2albz4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/6 -1UWY0jUJ9gNDlP7ZvyCVeYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fE -FY8ElggGQgT4hNYdvJGmQr5J1WqIP7wtUdGejeBSzFfdNTVY27SPJIjki9/ca1TS -gSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz3GIZ38i1MH/1PCZ1Eb3X -G7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj3CzMpSZy -YhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaH -vGOz9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4 -t/bQWVyJ98LVtZR00dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/ -gh7PU3+06yzbXfZqfUAkBXKJOAGTy3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUajn6QiL3 -5okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59M4PLuG53hq8w -DQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz -Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0 -nXGEL8pZ0keImUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qT -RmTFAHneIWv2V6CG1wZy7HBGS4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpT -wm+bREx50B1ws9efAvSyB7DH5fitIw6mVskpEndI2S9G/Tvw/HRwkqWOOAgfZDC2 -t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ6e18CL13zSdkzJTa -TkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97krgCf2 -o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU -3jg9CcCoSmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eA -iN1nE28daCSLT7d0geX0YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14f -WKGVyasvc0rQLW6aWQ9VGHgtPFGml4vmu7JwqkwR3v98KzfUetF3NI/n+UL3PIEM -S1IK ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G3 O=OpenTrust -# Subject: CN=OpenTrust Root CA G3 O=OpenTrust -# Label: "OpenTrust Root CA G3" -# Serial: 1492104908271485653071219941864171170455615 -# MD5 Fingerprint: 21:37:b4:17:16:92:7b:67:46:70:a9:96:d7:a8:13:24 -# SHA1 Fingerprint: 6e:26:64:f3:56:bf:34:55:bf:d1:93:3f:7c:01:de:d8:13:da:8a:a6 -# SHA256 Fingerprint: b7:c3:62:31:70:6e:81:07:8c:36:7c:b8:96:19:8f:1e:32:08:dd:92:69:49:dd:8f:57:09:a4:10:f7:5b:62:92 ------BEGIN CERTIFICATE----- -MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAx -CzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5U -cnVzdCBSb290IENBIEczMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFow -QDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwUT3Bl -blRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARK7liuTcpm -3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5Bta1d -oYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4G -A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5 -DMlv4VBN0BBY3JWIbTAfBgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAK -BggqhkjOPQQDAwNpADBmAjEAj6jcnboMBBf6Fek9LykBl7+BFjNAk2z8+e2AcG+q -j9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta3U1fJAuwACEl74+nBCZx -4nxp5V2a+EEfOzmTk51V6s2N8fvB ------END CERTIFICATE----- - # Issuer: CN=ISRG Root X1 O=Internet Security Research Group # Subject: CN=ISRG Root X1 O=Internet Security Research Group # Label: "ISRG Root X1" @@ -4398,3 +4235,66 @@ MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== -----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GC CA" +# Serial: 44084345621038548146064804565436152554 +# MD5 Fingerprint: a9:d6:b9:2d:2f:93:64:f8:a5:69:ca:91:e9:68:07:23 +# SHA1 Fingerprint: e0:11:84:5e:34:de:be:88:81:b9:9c:f6:16:26:d1:96:1f:c3:b9:31 +# SHA256 Fingerprint: 85:60:f9:1c:36:24:da:ba:95:70:b5:fe:a0:db:e3:6f:f1:1a:83:23:be:94:86:85:4f:b3:f3:4a:55:71:19:8d +-----BEGIN CERTIFICATE----- +MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw +CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91 +bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg +Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ +BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu +ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS +b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni +eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W +p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T +rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV +57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg +Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 +-----END CERTIFICATE----- diff --git a/pipenv/patched/notpip/_vendor/packaging/__about__.py b/pipenv/patched/notpip/_vendor/packaging/__about__.py index 4255c5b553..21fc6ce3e7 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__about__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__about__.py @@ -12,10 +12,10 @@ __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "17.1" +__version__ = "18.0" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2016 %s" % __author__ +__copyright__ = "Copyright 2014-2018 %s" % __author__ diff --git a/pipenv/patched/notpip/_vendor/packaging/requirements.py b/pipenv/patched/notpip/_vendor/packaging/requirements.py index 0da25914fe..5ec5d74a80 100644 --- a/pipenv/patched/notpip/_vendor/packaging/requirements.py +++ b/pipenv/patched/notpip/_vendor/packaging/requirements.py @@ -92,16 +92,16 @@ def __init__(self, requirement_string): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement( - "Invalid requirement, parse error at \"{0!r}\"".format( - requirement_string[e.loc:e.loc + 8])) + raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( + requirement_string[e.loc:e.loc + 8], e.msg + )) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) if not (parsed_url.scheme and parsed_url.netloc) or ( not parsed_url.scheme and not parsed_url.netloc): - raise InvalidRequirement("Invalid URL given") + raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: self.url = None diff --git a/pipenv/patched/notpip/_vendor/packaging/specifiers.py b/pipenv/patched/notpip/_vendor/packaging/specifiers.py index 9b6353f052..4c798999d0 100644 --- a/pipenv/patched/notpip/_vendor/packaging/specifiers.py +++ b/pipenv/patched/notpip/_vendor/packaging/specifiers.py @@ -503,7 +503,7 @@ def _compare_greater_than(self, prospective, spec): return False # Ensure that we do not allow a local version of the version mentioned - # in the specifier, which is techincally greater than, to match. + # in the specifier, which is technically greater than, to match. if prospective.local is not None: if Version(prospective.base_version) == Version(spec.base_version): return False diff --git a/pipenv/vendor/pathlib2.LICENSE.rst b/pipenv/patched/notpip/_vendor/pep517/LICENSE similarity index 83% rename from pipenv/vendor/pathlib2.LICENSE.rst rename to pipenv/patched/notpip/_vendor/pep517/LICENSE index ddb51b8af8..b0ae9dbc26 100644 --- a/pipenv/vendor/pathlib2.LICENSE.rst +++ b/pipenv/patched/notpip/_vendor/pep517/LICENSE @@ -1,7 +1,6 @@ The MIT License (MIT) -Copyright (c) 2014 Matthias C. M. Troffaes -Copyright (c) 2012-2014 Antoine Pitrou and contributors +Copyright (c) 2017 Thomas Kluyver Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -10,14 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/patched/notpip/_vendor/pep517/__init__.py b/pipenv/patched/notpip/_vendor/pep517/__init__.py new file mode 100644 index 0000000000..8beedea479 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/__init__.py @@ -0,0 +1,4 @@ +"""Wrappers to build Python packages using PEP 517 hooks +""" + +__version__ = '0.2' diff --git a/pipenv/patched/notpip/_vendor/pep517/_in_process.py b/pipenv/patched/notpip/_vendor/pep517/_in_process.py new file mode 100644 index 0000000000..baa14d381a --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/_in_process.py @@ -0,0 +1,182 @@ +"""This is invoked in a subprocess to call the build backend hooks. + +It expects: +- Command line args: hook_name, control_dir +- Environment variable: PEP517_BUILD_BACKEND=entry.point:spec +- control_dir/input.json: + - {"kwargs": {...}} + +Results: +- control_dir/output.json + - {"return_val": ...} +""" +from glob import glob +from importlib import import_module +import os +from os.path import join as pjoin +import re +import shutil +import sys + +# This is run as a script, not a module, so it can't do a relative import +import compat + +def _build_backend(): + """Find and load the build backend""" + ep = os.environ['PEP517_BUILD_BACKEND'] + mod_path, _, obj_path = ep.partition(':') + obj = import_module(mod_path) + if obj_path: + for path_part in obj_path.split('.'): + obj = getattr(obj, path_part) + return obj + +def get_requires_for_build_wheel(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_wheel + except AttributeError: + return [] + else: + return hook(config_settings) + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings): + """Invoke optional prepare_metadata_for_build_wheel + + Implements a fallback by building a wheel if the hook isn't defined. + """ + backend = _build_backend() + try: + hook = backend.prepare_metadata_for_build_wheel + except AttributeError: + return _get_wheel_metadata_from_wheel(backend, metadata_directory, + config_settings) + else: + return hook(metadata_directory, config_settings) + +WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + +def _dist_info_files(whl_zip): + """Identify the .dist-info folder inside a wheel ZipFile.""" + res = [] + for path in whl_zip.namelist(): + m = re.match(r'[^/\\]+-[^/\\]+\.dist-info/', path) + if m: + res.append(path) + if res: + return res + raise Exception("No .dist-info folder found in wheel") + +def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings): + """Build a wheel and extract the metadata from it. + + Fallback for when the build backend does not define the 'get_wheel_metadata' + hook. + """ + from zipfile import ZipFile + whl_basename = backend.build_wheel(metadata_directory, config_settings) + with open(os.path.join(metadata_directory, WHEEL_BUILT_MARKER), 'wb'): + pass # Touch marker file + + whl_file = os.path.join(metadata_directory, whl_basename) + with ZipFile(whl_file) as zipf: + dist_info = _dist_info_files(zipf) + zipf.extractall(path=metadata_directory, members=dist_info) + return dist_info[0].split('/')[0] + +def _find_already_built_wheel(metadata_directory): + """Check for a wheel already built during the get_wheel_metadata hook. + """ + if not metadata_directory: + return None + metadata_parent = os.path.dirname(metadata_directory) + if not os.path.isfile(pjoin(metadata_parent, WHEEL_BUILT_MARKER)): + return None + + whl_files = glob(os.path.join(metadata_parent, '*.whl')) + if not whl_files: + print('Found wheel built marker, but no .whl files') + return None + if len(whl_files) > 1: + print('Found multiple .whl files; unspecified behaviour. ' + 'Will call build_wheel.') + return None + + # Exactly one .whl file + return whl_files[0] + +def build_wheel(wheel_directory, config_settings, metadata_directory=None): + """Invoke the mandatory build_wheel hook. + + If a wheel was already built in the prepare_metadata_for_build_wheel fallback, this + will copy it rather than rebuilding the wheel. + """ + prebuilt_whl = _find_already_built_wheel(metadata_directory) + if prebuilt_whl: + shutil.copy2(prebuilt_whl, wheel_directory) + return os.path.basename(prebuilt_whl) + + return _build_backend().build_wheel(wheel_directory, config_settings, + metadata_directory) + + +def get_requires_for_build_sdist(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_sdist + except AttributeError: + return [] + else: + return hook(config_settings) + +class _DummyException(Exception): + """Nothing should ever raise this exception""" + +class GotUnsupportedOperation(Exception): + """For internal use when backend raises UnsupportedOperation""" + +def build_sdist(sdist_directory, config_settings): + """Invoke the mandatory build_sdist hook.""" + backend = _build_backend() + try: + return backend.build_sdist(sdist_directory, config_settings) + except getattr(backend, 'UnsupportedOperation', _DummyException): + raise GotUnsupportedOperation + +HOOK_NAMES = { + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_wheel', + 'build_wheel', + 'get_requires_for_build_sdist', + 'build_sdist', +} + +def main(): + if len(sys.argv) < 3: + sys.exit("Needs args: hook_name, control_dir") + hook_name = sys.argv[1] + control_dir = sys.argv[2] + if hook_name not in HOOK_NAMES: + sys.exit("Unknown hook: %s" % hook_name) + hook = globals()[hook_name] + + hook_input = compat.read_json(pjoin(control_dir, 'input.json')) + + json_out = {'unsupported': False, 'return_val': None} + try: + json_out['return_val'] = hook(**hook_input['kwargs']) + except GotUnsupportedOperation: + json_out['unsupported'] = True + + compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + +if __name__ == '__main__': + main() diff --git a/pipenv/patched/notpip/_vendor/pep517/check.py b/pipenv/patched/notpip/_vendor/pep517/check.py new file mode 100644 index 0000000000..3dffd2e0aa --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/check.py @@ -0,0 +1,194 @@ +"""Check a project and backend by attempting to build using PEP 517 hooks. +""" +import argparse +import logging +import os +from os.path import isfile, join as pjoin +from pipenv.patched.notpip._vendor.pytoml import TomlError, load as toml_load +import shutil +from subprocess import CalledProcessError +import sys +import tarfile +from tempfile import mkdtemp +import zipfile + +from .colorlog import enable_colourful_output +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + +def check_build_sdist(hooks): + with BuildEnvironment() as env: + try: + env.pip_install(hooks.build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_sdist({}) + log.info('Got build requires: %s', reqs) + except: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build sdist in %s', td) + try: + try: + filename = hooks.build_sdist(td, {}) + log.info('build_sdist returned %r', filename) + except: + log.info('Failure in build_sdist', exc_info=True) + return False + + if not filename.endswith('.tar.gz'): + log.error("Filename %s doesn't have .tar.gz extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if tarfile.is_tarfile(path): + log.info("Output file is a tar file") + else: + log.error("Output file is not a tar file") + return False + + finally: + shutil.rmtree(td) + + return True + +def check_build_wheel(hooks): + with BuildEnvironment() as env: + try: + env.pip_install(hooks.build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_wheel({}) + log.info('Got build requires: %s', reqs) + except: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build wheel in %s', td) + try: + try: + filename = hooks.build_wheel(td, {}) + log.info('build_wheel returned %r', filename) + except: + log.info('Failure in build_wheel', exc_info=True) + return False + + if not filename.endswith('.whl'): + log.error("Filename %s doesn't have .whl extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if zipfile.is_zipfile(path): + log.info("Output file is a zip file") + else: + log.error("Output file is not a zip file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check(source_dir): + pyproject = pjoin(source_dir, 'pyproject.toml') + if isfile(pyproject): + log.info('Found pyproject.toml') + else: + log.error('Missing pyproject.toml') + return False + + try: + with open(pyproject) as f: + pyproject_data = toml_load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + log.info('Loaded pyproject.toml') + except (TomlError, KeyError): + log.error("Invalid pyproject.toml", exc_info=True) + return False + + hooks = Pep517HookCaller(source_dir, backend) + + sdist_ok = check_build_sdist(hooks) + wheel_ok = check_build_wheel(hooks) + + if not sdist_ok: + log.warning('Sdist checks failed; scroll up to see') + if not wheel_ok: + log.warning('Wheel checks failed') + + return sdist_ok + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument('source_dir', + help="A directory containing pyproject.toml") + args = ap.parse_args(argv) + + enable_colourful_output() + + ok = check(args.source_dir) + + if ok: + print(ansi('Checks passed', 'green')) + else: + print(ansi('Checks failed', 'red')) + sys.exit(1) + +ansi_codes = { + 'reset': '\x1b[0m', + 'bold': '\x1b[1m', + 'red': '\x1b[31m', + 'green': '\x1b[32m', +} +def ansi(s, attr): + if os.name != 'nt' and sys.stdout.isatty(): + return ansi_codes[attr] + str(s) + ansi_codes['reset'] + else: + return str(s) + +if __name__ == '__main__': + main() diff --git a/pipenv/patched/notpip/_vendor/pep517/colorlog.py b/pipenv/patched/notpip/_vendor/pep517/colorlog.py new file mode 100644 index 0000000000..26cf7480dd --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/colorlog.py @@ -0,0 +1,110 @@ +"""Nicer log formatting with colours. + +Code copied from Tornado, Apache licensed. +""" +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +try: + import curses +except ImportError: + curses = None + +def _stderr_supports_color(): + color = False + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): + try: + curses.setupterm() + if curses.tigetnum("colors") > 0: + color = True + except Exception: + pass + return color + +class LogFormatter(logging.Formatter): + """Log formatter with colour support + """ + DEFAULT_COLORS = { + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red + logging.CRITICAL: 1, + } + + def __init__(self, color=True, datefmt=None): + r""" + :arg bool color: Enables color support. + :arg string fmt: Log message format. + It will be applied to the attributes dict of log records. The + text between ``%(color)s`` and ``%(end_color)s`` will be colored + depending on the level if color support is on. + :arg dict colors: color mappings from logging level to terminal color + code + :arg string datefmt: Datetime format. + Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. + .. versionchanged:: 3.2 + Added ``fmt`` and ``datefmt`` arguments. + """ + logging.Formatter.__init__(self, datefmt=datefmt) + self._colors = {} + if color and _stderr_supports_color(): + # The curses module has some str/bytes confusion in + # python3. Until version 3.2.3, most methods return + # bytes, but only accept strings. In addition, we want to + # output these strings with the logging module, which + # works with unicode strings. The explicit calls to + # unicode() below are harmless in python2 but will do the + # right conversion in python 3. + fg_color = (curses.tigetstr("setaf") or + curses.tigetstr("setf") or "") + if (3, 0) < sys.version_info < (3, 2, 3): + fg_color = str(fg_color, "ascii") + + for levelno, code in self.DEFAULT_COLORS.items(): + self._colors[levelno] = str(curses.tparm(fg_color, code), "ascii") + self._normal = str(curses.tigetstr("sgr0"), "ascii") + + scr = curses.initscr() + self.termwidth = scr.getmaxyx()[1] + curses.endwin() + else: + self._normal = '' + # Default width is usually 80, but too wide is worse than too narrow + self.termwidth = 70 + + def formatMessage(self, record): + l = len(record.message) + right_text = '{initial}-{name}'.format(initial=record.levelname[0], + name=record.name) + if l + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (l + len(right_text))) + else: + space = ' ' + + if record.levelno in self._colors: + start_color = self._colors[record.levelno] + end_color = self._normal + else: + start_color = end_color = '' + + return record.message + space + start_color + right_text + end_color + +def enable_colourful_output(level=logging.INFO): + handler = logging.StreamHandler() + handler.setFormatter(LogFormatter()) + logging.root.addHandler(handler) + logging.root.setLevel(level) diff --git a/pipenv/patched/notpip/_vendor/pep517/compat.py b/pipenv/patched/notpip/_vendor/pep517/compat.py new file mode 100644 index 0000000000..01c66fc7e4 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/compat.py @@ -0,0 +1,23 @@ +"""Handle reading and writing JSON in UTF-8, on Python 3 and 2.""" +import json +import sys + +if sys.version_info[0] >= 3: + # Python 3 + def write_json(obj, path, **kwargs): + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, **kwargs) + + def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + +else: + # Python 2 + def write_json(obj, path, **kwargs): + with open(path, 'wb') as f: + json.dump(obj, f, encoding='utf-8', **kwargs) + + def read_json(path): + with open(path, 'rb') as f: + return json.load(f) diff --git a/pipenv/patched/notpip/_vendor/pep517/envbuild.py b/pipenv/patched/notpip/_vendor/pep517/envbuild.py new file mode 100644 index 0000000000..c54d358570 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/envbuild.py @@ -0,0 +1,150 @@ +"""Build wheels/sdists by installing build deps to a temporary environment. +""" + +import os +import logging +from pipenv.patched.notpip._vendor import pytoml +import shutil +from subprocess import check_call +import sys +from sysconfig import get_paths +from tempfile import mkdtemp + +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + +def _load_pyproject(source_dir): + with open(os.path.join(source_dir, 'pyproject.toml')) as f: + pyproject_data = pytoml.load(f) + buildsys = pyproject_data['build-system'] + return buildsys['requires'], buildsys['build-backend'] + + +class BuildEnvironment(object): + """Context manager to install build deps in a simple temporary environment + + Based on code I wrote for pip, which is MIT licensed. + """ + # Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) + # + # Permission is hereby granted, free of charge, to any person obtaining + # a copy of this software and associated documentation files (the + # "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to + # permit persons to whom the Software is furnished to do so, subject to + # the following conditions: + # + # The above copyright notice and this permission notice shall be + # included in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + path = None + + def __init__(self, cleanup=True): + self._cleanup = cleanup + + def __enter__(self): + self.path = mkdtemp(prefix='pep517-build-env-') + log.info('Temporary build environment: %s', self.path) + + self.save_path = os.environ.get('PATH', None) + self.save_pythonpath = os.environ.get('PYTHONPATH', None) + + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + install_dirs = get_paths(install_scheme, vars={ + 'base': self.path, + 'platbase': self.path, + }) + + scripts = install_dirs['scripts'] + if self.save_path: + os.environ['PATH'] = scripts + os.pathsep + self.save_path + else: + os.environ['PATH'] = scripts + os.pathsep + os.defpath + + if install_dirs['purelib'] == install_dirs['platlib']: + lib_dirs = install_dirs['purelib'] + else: + lib_dirs = install_dirs['purelib'] + os.pathsep + \ + install_dirs['platlib'] + if self.save_pythonpath: + os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ + self.save_pythonpath + else: + os.environ['PYTHONPATH'] = lib_dirs + + return self + + def pip_install(self, reqs): + """Install dependencies into this env by calling pip in a subprocess""" + if not reqs: + return + log.info('Calling pip to install %s', reqs) + check_call([sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs)) + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._cleanup and (self.path is not None) and os.path.isdir(self.path): + shutil.rmtree(self.path) + + if self.save_path is None: + os.environ.pop('PATH', None) + else: + os.environ['PATH'] = self.save_path + + if self.save_pythonpath is None: + os.environ.pop('PYTHONPATH', None) + else: + os.environ['PYTHONPATH'] = self.save_pythonpath + +def build_wheel(source_dir, wheel_dir, config_settings=None): + """Build a wheel from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str wheel_dir: Target directory to create wheel in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_wheel(config_settings) + env.pip_install(reqs) + return hooks.build_wheel(wheel_dir, config_settings) + + +def build_sdist(source_dir, sdist_dir, config_settings=None): + """Build an sdist from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str sdist_dir: Target directory to place sdist in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_sdist(config_settings) + env.pip_install(reqs) + return hooks.build_sdist(sdist_dir, config_settings) diff --git a/pipenv/patched/notpip/_vendor/pep517/wrappers.py b/pipenv/patched/notpip/_vendor/pep517/wrappers.py new file mode 100644 index 0000000000..28260f320d --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/wrappers.py @@ -0,0 +1,134 @@ +from contextlib import contextmanager +import os +from os.path import dirname, abspath, join as pjoin +import shutil +from subprocess import check_call +import sys +from tempfile import mkdtemp + +from . import compat + +_in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + +@contextmanager +def tempdir(): + td = mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + +class UnsupportedOperation(Exception): + """May be raised by build_sdist if the backend indicates that it can't.""" + +class Pep517HookCaller(object): + """A wrapper around a source directory to be built with a PEP 517 backend. + + source_dir : The path to the source directory, containing pyproject.toml. + backend : The build backend spec, as per PEP 517, from pyproject.toml. + """ + def __init__(self, source_dir, build_backend): + self.source_dir = abspath(source_dir) + self.build_backend = build_backend + + def get_requires_for_build_wheel(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["wheel >= 0.25", "setuptools"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_wheel', { + 'config_settings': config_settings + }) + + def prepare_metadata_for_build_wheel(self, metadata_directory, config_settings=None): + """Prepare a *.dist-info folder with metadata for this project. + + Returns the name of the newly created folder. + + If the build backend defines a hook with this name, it will be called + in a subprocess. If not, the backend will be asked to build a wheel, + and the dist-info extracted from that. + """ + return self._call_hook('prepare_metadata_for_build_wheel', { + 'metadata_directory': abspath(metadata_directory), + 'config_settings': config_settings, + }) + + def build_wheel(self, wheel_directory, config_settings=None, metadata_directory=None): + """Build a wheel from this project. + + Returns the name of the newly created file. + + In general, this will call the 'build_wheel' hook in the backend. + However, if that was previously called by + 'prepare_metadata_for_build_wheel', and the same metadata_directory is + used, the previously built wheel will be copied to wheel_directory. + """ + if metadata_directory is not None: + metadata_directory = abspath(metadata_directory) + return self._call_hook('build_wheel', { + 'wheel_directory': abspath(wheel_directory), + 'config_settings': config_settings, + 'metadata_directory': metadata_directory, + }) + + def get_requires_for_build_sdist(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["setuptools >= 26"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_sdist', { + 'config_settings': config_settings + }) + + def build_sdist(self, sdist_directory, config_settings=None): + """Build an sdist from this project. + + Returns the name of the newly created file. + + This calls the 'build_sdist' backend hook in a subprocess. + """ + return self._call_hook('build_sdist', { + 'sdist_directory': abspath(sdist_directory), + 'config_settings': config_settings, + }) + + + def _call_hook(self, hook_name, kwargs): + env = os.environ.copy() + + # On Python 2, pytoml returns Unicode values (which is correct) but the + # environment passed to check_call needs to contain string values. We + # convert here by encoding using ASCII (the backend can only contain + # letters, digits and _, . and : characters, and will be used as a + # Python identifier, so non-ASCII content is wrong on Python 2 in + # any case). + if sys.version_info[0] == 2: + build_backend = self.build_backend.encode('ASCII') + else: + build_backend = self.build_backend + + env['PEP517_BUILD_BACKEND'] = build_backend + with tempdir() as td: + compat.write_json({'kwargs': kwargs}, pjoin(td, 'input.json'), + indent=2) + + # Run the hook in a subprocess + check_call([sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, env=env) + + data = compat.read_json(pjoin(td, 'output.json')) + if data.get('unsupported'): + raise UnsupportedOperation + return data['return_val'] + diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py index ed57821d14..ac893b66c8 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py @@ -47,6 +47,11 @@ # Python 3.2 compatibility import imp as _imp +try: + FileExistsError +except NameError: + FileExistsError = OSError + from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import urllib, map, filter @@ -78,8 +83,11 @@ __import__('pipenv.patched.notpip._vendor.packaging.markers') -if (3, 0) < sys.version_info < (3, 3): - raise RuntimeError("Python 3.3 or later is required") +__metaclass__ = type + + +if (3, 0) < sys.version_info < (3, 4): + raise RuntimeError("Python 3.4 or later is required") if six.PY2: # Those builtin exceptions are only defined in Python 3 @@ -537,7 +545,7 @@ def resource_listdir(resource_name): """List of resource names in the directory (like ``os.listdir()``)""" -class WorkingSet(object): +class WorkingSet: """A collection of active distributions on sys.path (or a similar list)""" def __init__(self, entries=None): @@ -637,13 +645,12 @@ def iter_entry_points(self, group, name=None): distributions in the working set, otherwise only ones matching both `group` and `name` are yielded (in distribution order). """ - for dist in self: - entries = dist.get_entry_map(group) - if name is None: - for ep in entries.values(): - yield ep - elif name in entries: - yield entries[name] + return ( + entry + for dist in self + for entry in dist.get_entry_map(group).values() + if name is None or name == entry.name + ) def run_script(self, requires, script_name): """Locate distribution for `requires` and run `script_name` script""" @@ -944,7 +951,7 @@ def markers_pass(self, req, extras=None): return not req.marker or any(extra_evals) -class Environment(object): +class Environment: """Searchable snapshot of distributions on a search path""" def __init__( @@ -959,7 +966,7 @@ def __init__( `platform` is an optional string specifying the name of the platform that platform-specific distributions must be compatible with. If unspecified, it defaults to the current platform. `python` is an - optional string naming the desired version of Python (e.g. ``'3.3'``); + optional string naming the desired version of Python (e.g. ``'3.6'``); it defaults to the current version. You may explicitly set `platform` (and/or `python`) to ``None`` if you @@ -2087,7 +2094,12 @@ def _handle_ns(packageName, path_item): importer = get_importer(path_item) if importer is None: return None - loader = importer.find_module(packageName) + + # capture warnings due to #1111 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + loader = importer.find_module(packageName) + if loader is None: return None module = sys.modules.get(packageName) @@ -2132,12 +2144,13 @@ def position_in_sys_path(path): parts = path_parts[:-module_parts] return safe_sys_path_index(_normalize_cached(os.sep.join(parts))) - if not isinstance(orig_path, list): - # Is this behavior useful when module.__path__ is not a list? - return + new_path = sorted(orig_path, key=position_in_sys_path) + new_path = [_normalize_cached(p) for p in new_path] - orig_path.sort(key=position_in_sys_path) - module.__path__[:] = [_normalize_cached(p) for p in orig_path] + if isinstance(module.__path__, list): + module.__path__[:] = new_path + else: + module.__path__ = new_path def declare_namespace(packageName): @@ -2148,9 +2161,10 @@ def declare_namespace(packageName): if packageName in _namespace_packages: return - path, parent = sys.path, None - if '.' in packageName: - parent = '.'.join(packageName.split('.')[:-1]) + path = sys.path + parent, _, _ = packageName.rpartition('.') + + if parent: declare_namespace(parent) if parent not in _namespace_packages: __import__(parent) @@ -2161,7 +2175,7 @@ def declare_namespace(packageName): # Track what packages are namespaces, so when new path items are added, # they can be updated - _namespace_packages.setdefault(parent, []).append(packageName) + _namespace_packages.setdefault(parent or None, []).append(packageName) _namespace_packages.setdefault(packageName, []) for path_item in path: @@ -2279,7 +2293,7 @@ def yield_lines(strs): ).match -class EntryPoint(object): +class EntryPoint: """Object representing an advertised importable object""" def __init__(self, name, module_name, attrs=(), extras=(), dist=None): @@ -2433,7 +2447,7 @@ def is_version_line(line): return safe_version(value.strip()) or None -class Distribution(object): +class Distribution: """Wrap an actual or potential sys.path entry w/metadata""" PKG_INFO = 'PKG-INFO' @@ -3027,7 +3041,10 @@ def _bypass_ensure_directory(path): dirname, filename = split(path) if dirname and filename and not isdir(dirname): _bypass_ensure_directory(dirname) - mkdir(dirname, 0o755) + try: + mkdir(dirname, 0o755) + except FileExistsError: + pass def split_sections(s): diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py b/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py index 331a51bb0f..3a44fa1ca7 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py @@ -2,6 +2,8 @@ import errno import sys +from pipenv.patched.notpip._vendor import six + def _makedirs_31(path, exist_ok=False): try: @@ -15,8 +17,7 @@ def _makedirs_31(path, exist_ok=False): # and exists_ok considerations are disentangled. # See https://github.com/pypa/setuptools/pull/1083#issuecomment-315168663 needs_makedirs = ( - sys.version_info < (3, 2, 5) or - (3, 3) <= sys.version_info < (3, 3, 6) or + six.PY2 or (3, 4) <= sys.version_info < (3, 4, 1) ) makedirs = _makedirs_31 if needs_makedirs else os.makedirs diff --git a/pipenv/patched/notpip/_vendor/pyparsing.py b/pipenv/patched/notpip/_vendor/pyparsing.py index b5ba3d5f57..455d1151c8 100644 --- a/pipenv/patched/notpip/_vendor/pyparsing.py +++ b/pipenv/patched/notpip/_vendor/pyparsing.py @@ -1,6 +1,6 @@ # module pyparsing.py # -# Copyright (c) 2003-2016 Paul T. McGuire +# Copyright (c) 2003-2018 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -25,6 +25,7 @@ __doc__ = \ """ pyparsing module - Classes and methods to define and execute parsing grammars +============================================================================= The pyparsing module is an alternative approach to creating and executing simple grammars, vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you @@ -58,10 +59,23 @@ class names, and the use of '+', '|' and '^' operators. - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - quoted strings - embedded comments + + +Getting Started - +----------------- +Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +classes inherit from. Use the docstrings for examples of how to: + - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes + - construct character word-group expressions using the L{Word} class + - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes + - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones + - associate names with your parsed results using L{ParserElement.setResultsName} + - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} + - find more useful common expressions in the L{pyparsing_common} namespace class """ -__version__ = "2.2.0" -__versionTime__ = "06 Mar 2017 02:06 UTC" +__version__ = "2.2.1" +__versionTime__ = "18 Sep 2018 00:49 UTC" __author__ = "Paul McGuire " import string @@ -82,6 +96,15 @@ class names, and the use of '+', '|' and '^' operators. except ImportError: from threading import RLock +try: + # Python 3 + from collections.abc import Iterable + from collections.abc import MutableMapping +except ImportError: + # Python 2.7 + from collections import Iterable + from collections import MutableMapping + try: from collections import OrderedDict as _OrderedDict except ImportError: @@ -940,7 +963,7 @@ def __getnewargs__(self): def __dir__(self): return (dir(type(self)) + list(self.keys())) -collections.MutableMapping.register(ParseResults) +MutableMapping.register(ParseResults) def col (loc,strg): """Returns current column within a string, counting newlines as line separators. @@ -1025,11 +1048,11 @@ def extract_stack(limit=0): # special handling for Python 3.5.0 - extra deep call stack by 1 offset = -3 if system_version == (3,5,0) else -2 frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] - return [(frame_summary.filename, frame_summary.lineno)] + return [frame_summary[:2]] def extract_tb(tb, limit=0): frames = traceback.extract_tb(tb, limit=limit) frame_summary = frames[-1] - return [(frame_summary.filename, frame_summary.lineno)] + return [frame_summary[:2]] else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb @@ -1374,7 +1397,7 @@ def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ): else: preloc = loc tokensStart = preloc - if self.mayIndexError or loc >= len(instring): + if self.mayIndexError or preloc >= len(instring): try: loc,tokens = self.parseImpl( instring, preloc, doActions ) except IndexError: @@ -1408,7 +1431,6 @@ def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ): self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), modal=self.modalResults ) - if debugging: #~ print ("Matched",self,"->",retTokens.asList()) if (self.debugActions[1] ): @@ -3242,7 +3264,7 @@ def __init__( self, exprs, savelist = False ): if isinstance( exprs, basestring ): self.exprs = [ ParserElement._literalStringClass( exprs ) ] - elif isinstance( exprs, collections.Iterable ): + elif isinstance( exprs, Iterable ): exprs = list(exprs) # if sequence of strings provided, wrap with Literal if all(isinstance(expr, basestring) for expr in exprs): @@ -4393,7 +4415,7 @@ def traceParseAction(f): @traceParseAction def remove_duplicate_chars(tokens): - return ''.join(sorted(set(''.join(tokens))) + return ''.join(sorted(set(''.join(tokens)))) wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) @@ -4583,7 +4605,7 @@ def oneOf( strs, caseless=False, useRegex=True ): symbols = [] if isinstance(strs,basestring): symbols = strs.split() - elif isinstance(strs, collections.Iterable): + elif isinstance(strs, Iterable): symbols = list(strs) else: warnings.warn("Invalid argument to oneOf, expected string or iterable", @@ -4734,7 +4756,7 @@ def locatedExpr(expr): _escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1]) _escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16))) _escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8))) -_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1) | Regex(r"\w", re.UNICODE) +_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1) _charRange = Group(_singleChar + Suppress("-") + _singleChar) _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" diff --git a/pipenv/patched/notpip/_vendor/pytoml/parser.py b/pipenv/patched/notpip/_vendor/pytoml/parser.py index e03a03fbda..9f94e9230a 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/parser.py +++ b/pipenv/patched/notpip/_vendor/pytoml/parser.py @@ -223,8 +223,8 @@ def _p_key(s): _datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') _basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') -_litstr_re = re.compile(r"[^'\000-\037]*") -_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*") +_litstr_re = re.compile(r"[^'\000\010\012-\037]*") +_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") def _p_value(s, object_pairs_hook): pos = s.pos() diff --git a/pipenv/patched/notpip/_vendor/requests/__init__.py b/pipenv/patched/notpip/_vendor/requests/__init__.py index 8e7576c13f..af3d4b1de9 100644 --- a/pipenv/patched/notpip/_vendor/requests/__init__.py +++ b/pipenv/patched/notpip/_vendor/requests/__init__.py @@ -91,7 +91,7 @@ def _check_cryptography(cryptography_version): RequestsDependencyWarning) # Attempt to enable urllib3's SNI support, if possible -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS if not WINDOWS: try: from pipenv.patched.notpip._vendor.urllib3.contrib import pyopenssl diff --git a/pipenv/patched/notpip/_vendor/vendor.txt b/pipenv/patched/notpip/_vendor/vendor.txt index b9854e9ad1..9389dd947d 100644 --- a/pipenv/patched/notpip/_vendor/vendor.txt +++ b/pipenv/patched/notpip/_vendor/vendor.txt @@ -9,14 +9,15 @@ msgpack-python==0.5.6 lockfile==0.12.2 progress==1.4 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==17.1 -pyparsing==2.2.0 -pytoml==0.1.16 +packaging==18.0 +pep517==0.2 +pyparsing==2.2.1 +pytoml==0.1.19 retrying==1.3.3 requests==2.19.1 chardet==3.0.4 idna==2.7 urllib3==1.23 - certifi==2018.4.16 -setuptools==39.2.0 + certifi==2018.8.24 +setuptools==40.4.3 webencodings==0.5.1 diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index 3ad4c1e934..f73ed5a6e2 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -4,4 +4,4 @@ crayons==0.1.2 pipfile==0.0.2 pip-tools==3.1.0 prettytoml==0.3 -pip==18.0 +pip==18.1 diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index de9b4353d4..c466ef0409 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -1,49 +1,55 @@ # -*- coding=utf-8 -*- -import importlib -from pip_shims import pip_version -import pkg_resources +__all__ = [ + "InstallRequirement", + "parse_requirements", + "RequirementSet", + "user_cache_dir", + "FAVORITE_HASH", + "is_file_url", + "url_to_path", + "PackageFinder", + "FormatControl", + "Wheel", + "Command", + "cmdoptions", + "get_installed_distributions", + "PyPI", + "SafeFileCache", + "InstallationError", + "parse_version", + "pip_version", + "install_req_from_editable", + "install_req_from_line", + "user_cache_dir" +] -def do_import(module_path, subimport=None, old_path=None, vendored_name=None): - old_path = old_path or module_path - prefix = vendored_name if vendored_name else "pip" - prefixes = ["{0}._internal".format(prefix), "{0}".format(prefix)] - paths = [module_path, old_path] - search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] - package = subimport if subimport else None - for to_import in search_order: - if not subimport: - to_import, _, package = to_import.rpartition(".") - try: - imported = importlib.import_module(to_import) - except ImportError: - continue - else: - return getattr(imported, package) - - -InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name="notpip") -parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name="notpip") -RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name="notpip") -user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name="notpip") -FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name="notpip") -is_file_url = do_import('download', 'is_file_url', vendored_name="notpip") -url_to_path = do_import('download', 'url_to_path', vendored_name="notpip") -PackageFinder = do_import('index', 'PackageFinder', vendored_name="notpip") -FormatControl = do_import('index', 'FormatControl', vendored_name="notpip") -Wheel = do_import('wheel', 'Wheel', vendored_name="notpip") -Command = do_import('cli.base_command', 'Command', old_path='basecommand', vendored_name="notpip") -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions', vendored_name="notpip") -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name="notpip") -PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') -SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') -InstallationError = do_import('exceptions', 'InstallationError', vendored_name='notpip') +from pipenv.vendor.appdirs import user_cache_dir +from pip_shims.shims import ( + InstallRequirement, + parse_requirements, + RequirementSet, + FAVORITE_HASH, + is_file_url, + url_to_path, + PackageFinder, + FormatControl, + Wheel, + Command, + cmdoptions, + get_installed_distributions, + PyPI, + SafeFileCache, + InstallationError, + parse_version, + pip_version, +) # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('18.1'): +if parse_version(pip_version) < parse_version('18.1'): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - install_req_from_line = do_import('req.constructors', 'install_req_from_line', vendored_name="notpip") - install_req_from_editable = do_import('req.constructors', 'install_req_from_editable', vendored_name="notpip") - + from pip_shims.shims import ( + install_req_from_editable, install_req_from_line + ) diff --git a/pipenv/project.py b/pipenv/project.py index 280a6f8b06..dcf9b41794 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -639,8 +639,9 @@ def pipfile_is_empty(self): def create_pipfile(self, python=None): """Creates the Pipfile, filled with juicy defaults.""" - from .patched.notpip._internal import ConfigOptionParser - from .patched.notpip._internal.cmdoptions import make_option_group, index_group + from .vendor.pip_shims.shims import ( + ConfigOptionParser, make_option_group, index_group + ) config_parser = ConfigOptionParser(name=self.name) config_parser.add_option_group(make_option_group(index_group, config_parser)) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index b82088a22d..b5ea41d9ed 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -97,9 +97,16 @@ def main(): import warnings from pipenv.vendor.vistir.compat import ResourceWarning warnings.simplefilter("ignore", category=ResourceWarning) - from pipenv.vendor import colorama - colorama.init() + import io + import six + if six.PY3: + sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') + sys.stderr = io.TextIOWrapper(sys.stderr.buffer,encoding='utf8') + else: + from pipenv._compat import force_encoding + force_encoding() os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") + os.environ["PYTHONIOENCODING"] = str("utf-8") parser = get_parser() parsed, remaining = parser.parse_known_args() # sys.argv = remaining @@ -110,4 +117,6 @@ def main(): if __name__ == "__main__": _patch_path() + from pipenv.vendor import colorama + colorama.init() main() diff --git a/pipenv/utils.py b/pipenv/utils.py index 7c78de3455..893ed88f08 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -211,10 +211,10 @@ def actually_resolve_deps( pre, req_dir=None, ): - from .patched.notpip._internal import basecommand - from .patched.notpip._internal.req import parse_requirements - from .patched.notpip._internal.exceptions import DistributionNotFound - from .patched.notpip._vendor.requests.exceptions import HTTPError + from .vendor.pip_shims.shims import ( + Command, parse_requirements, DistributionNotFound + ) + from .vendor.requests.exceptions import HTTPError from pipenv.patched.piptools.resolver import Resolver from pipenv.patched.piptools.repositories.pypi import PyPIRepository from pipenv.patched.piptools.scripts.compile import get_pip_command @@ -223,7 +223,7 @@ def actually_resolve_deps( from .vendor.requirementslib.models.requirements import Requirement from .vendor.vistir.path import create_tracked_tempdir, create_tracked_tempfile - class PipCommand(basecommand.Command): + class PipCommand(Command): """Needed for pip-tools.""" name = "PipCommand" @@ -338,6 +338,7 @@ def venv_resolve_deps( from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor import delegator from . import resolver + from ._compat import decode_output import json if not deps: @@ -380,14 +381,14 @@ def venv_resolve_deps( break _out = c.subprocess.before if _out is not None: - _out = to_native_string("{0}".format(_out)) + _out = decode_output("{0}".format(_out)) out += _out - sp.text = to_native_string("Locking... {0}".format(_out[:100])) - if environments.is_verbose(): - if _out is not None: - sp._hide_cursor() - sp.write(_out.rstrip()) - sp._show_cursor() + sp.text = to_native_string("{0}".format(_out[:100])) + if environments.is_verbose(): + if _out is not None: + sp._hide_cursor() + sp.write(_out.rstrip()) + sp._show_cursor() c.block() if c.return_code != 0: sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( @@ -423,7 +424,7 @@ def resolve_deps( """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ - from .patched.notpip._vendor.requests.exceptions import ConnectionError + from .vendor.requests.exceptions import ConnectionError from .vendor.requirementslib.models.requirements import Requirement index_lookup = {} @@ -637,9 +638,8 @@ def is_editable(pipfile_entry): def is_installable_file(path): """Determine if a path can potentially be installed""" - from .patched.notpip._internal.utils.misc import is_installable_dir + from .vendor.pip_shims.shims import is_installable_dir, is_archive_file from .patched.notpip._internal.utils.packaging import specifiers - from .patched.notpip._internal.download import is_archive_file from ._compat import Path if hasattr(path, "keys") and any( @@ -691,7 +691,7 @@ def is_file(package): def pep440_version(version): """Normalize version to PEP 440 standards""" - from .patched.notpip._internal.index import parse_version + from .vendor.pip_shims.shims import parse_version # Use pip built-in version parser. return str(parse_version(version)) @@ -1149,7 +1149,7 @@ def translate_markers(pipfile_entry): """ if not isinstance(pipfile_entry, Mapping): raise TypeError("Entry is not a pipfile formatted mapping.") - from notpip._vendor.distlib.markers import DEFAULT_CONTEXT as marker_context + from .vendor.distlib.markers import DEFAULT_CONTEXT as marker_context from .vendor.packaging.markers import Marker from .vendor.vistir.misc import dedup diff --git a/pipenv/vendor/backports/__init__.py b/pipenv/vendor/backports/__init__.py index 3cd3096323..0c64b4c10b 100644 --- a/pipenv/vendor/backports/__init__.py +++ b/pipenv/vendor/backports/__init__.py @@ -1,7 +1,5 @@ -# See https://pypi.python.org/pypi/backports - -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) from . import weakref from . import enum from . import shutil_get_terminal_size +from . import functools_lru_cache diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py index 4bf72946d3..6ba28bbbc3 100644 --- a/pipenv/vendor/dotenv/main.py +++ b/pipenv/vendor/dotenv/main.py @@ -95,6 +95,13 @@ def set_as_environment_variables(self, override=False): for k, v in self.dict().items(): if k in os.environ and not override: continue + # With Python 2 on Windows, ensuree environment variables are + # system strings to avoid "TypeError: environment can only contain + # strings" in Python's subprocess module. + if sys.version_info.major < 3 and sys.platform == 'win32': + from pipenv.utils import fs_str + k = fs_str(k) + v = fs_str(v) os.environ[k] = v return True diff --git a/pipenv/vendor/modutil.LICENSE b/pipenv/vendor/modutil.LICENSE deleted file mode 100644 index f680f0717a..0000000000 --- a/pipenv/vendor/modutil.LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2018, Brett Cannon -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py index 6ce97aadaa..358cc33b77 100644 --- a/pipenv/vendor/passa/internals/dependencies.py +++ b/pipenv/vendor/passa/internals/dependencies.py @@ -216,7 +216,7 @@ def _read_requires_python(metadata): def _get_dependencies_from_pip(ireq, sources): - """Retrieves dependencies for the requirement from pip internals. + """Retrieves dependencies for the requirement from pipenv.patched.notpip internals. The current strategy is to try the followings in order, returning the first successful result. diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 70fb0d5870..22bf130fea 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -3,7 +3,7 @@ import sys -__version__ = '0.3.1' +__version__ = '0.3.2' from . import shims diff --git a/pipenv/vendor/pip_shims/shims.py b/pipenv/vendor/pip_shims/shims.py index 7b81a60858..34de6906b4 100644 --- a/pipenv/vendor/pip_shims/shims.py +++ b/pipenv/vendor/pip_shims/shims.py @@ -43,6 +43,9 @@ def __init__(self): "pip_shims.utils": utils } self.pip_version = getattr(self._modules["pip"], "__version__") + version_types = ["post", "pre", "dev", "rc"] + if any(post in self.pip_version.rsplit(".")[-1] for post in version_types): + self.pip_version, _, _ = self.pip_version.rpartition(".") self.parsed_pip_version = self._parse(self.pip_version) self._contextmanagers = ("RequirementTracker",) self._moves = { @@ -140,6 +143,7 @@ def __init__(self): ("wheel.WheelCache", "7", "9.0.3") ), "WheelBuilder": ("wheel.WheelBuilder", "7.0.0", "9999"), + "PyPI": ("models.index.PyPI", "7.0.0", "9999"), } def _ensure_methods(self, cls, classname, *methods): diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index d800f92662..60192826d3 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.5' +__version__ = '1.1.6' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 4a4c50da50..20b2c19692 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -27,6 +27,7 @@ path_is_known_executable, unnest, ) +from .python import PythonVersion @attr.s diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index a9d6105b2d..1595a963a7 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -14,6 +14,8 @@ from ..utils import ( ensure_path, optional_instance_of, + get_python_version, + filter_pythons, unnest, ) from .mixins import BaseFinder, BasePath diff --git a/pipenv/vendor/requests/LICENSE b/pipenv/vendor/requests/LICENSE index 2e68b82ecb..841c6023b9 100644 --- a/pipenv/vendor/requests/LICENSE +++ b/pipenv/vendor/requests/LICENSE @@ -4,7 +4,7 @@ Copyright 2018 Kenneth Reitz you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 3103580e55..d72542c4d7 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -132,7 +132,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pip format""" + """Cleans VCS uris from pipenv.patched.notpip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: diff --git a/pipenv/vendor/shutil_backports/__init__.py b/pipenv/vendor/shutil_backports/__init__.py deleted file mode 100644 index fa12816e27..0000000000 --- a/pipenv/vendor/shutil_backports/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -__title__ = "shutil_backports" -__version__ = "0.1.0" -__license__ = "MIT" -__author__ = "Christopher Rosell" -__copyright__ = "Copyright 2014 Christopher Rosell" - -__all__ = ["get_terminal_size"] - -from .get_terminal_size import * diff --git a/pipenv/vendor/shutil_backports/get_terminal_size.py b/pipenv/vendor/shutil_backports/get_terminal_size.py deleted file mode 100644 index f1336e5839..0000000000 --- a/pipenv/vendor/shutil_backports/get_terminal_size.py +++ /dev/null @@ -1,100 +0,0 @@ -"""This is a backport of shutil.get_terminal_size from Python 3.3. - -The original implementation is in C, but here we use the ctypes and -fcntl modules to create a pure Python version of os.get_terminal_size. -""" - -import os -import struct -import sys - -from collections import namedtuple - -__all__ = ["get_terminal_size"] - - -terminal_size = namedtuple("terminal_size", "columns lines") - -try: - from ctypes import windll, create_string_buffer - - _handles = { - 0: windll.kernel32.GetStdHandle(-10), - 1: windll.kernel32.GetStdHandle(-11), - 2: windll.kernel32.GetStdHandle(-12), - } - - def _get_terminal_size(fd): - columns = lines = 0 - - try: - handle = _handles[fd] - csbi = create_string_buffer(22) - res = windll.kernel32.GetConsoleScreenBufferInfo(handle, csbi) - if res: - res = struct.unpack("hhhhHhhhhhh", csbi.raw) - left, top, right, bottom = res[5:9] - columns = right - left + 1 - lines = bottom - top + 1 - except Exception: - pass - - return columns, lines - -except ImportError: - import fcntl - import termios - - def _get_terminal_size(fd): - try: - res = fcntl.ioctl(fd, termios.TIOCGWINSZ, b"\x00" * 4) - lines, columns = struct.unpack("hh", res) - except Exception: - columns = lines = 0 - - return columns, lines - - -def get_terminal_size(fallback=(80, 24)): - """Get the size of the terminal window. - - For each of the two dimensions, the environment variable, COLUMNS - and LINES respectively, is checked. If the variable is defined and - the value is a positive integer, it is used. - - When COLUMNS or LINES is not defined, which is the common case, - the terminal connected to sys.__stdout__ is queried - by invoking os.get_terminal_size. - - If the terminal size cannot be successfully queried, either because - the system doesn't support querying, or because we are not - connected to a terminal, the value given in fallback parameter - is used. Fallback defaults to (80, 24) which is the default - size used by many terminal emulators. - - The value returned is a named tuple of type os.terminal_size. - """ - # Attempt to use the environment first - try: - columns = int(os.environ["COLUMNS"]) - except (KeyError, ValueError): - columns = 0 - - try: - lines = int(os.environ["LINES"]) - except (KeyError, ValueError): - lines = 0 - - # Only query if necessary - if columns <= 0 or lines <= 0: - try: - columns, lines = _get_terminal_size(sys.__stdout__.fileno()) - except (NameError, OSError): - pass - - # Use fallback as last resort - if columns <= 0 and lines <= 0: - columns, lines = fallback - - return terminal_size(columns, lines) - diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 091f27d801..35a32eb6ce 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -42,11 +42,12 @@ shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 vistir==0.2.2 -pip-shims==0.3.1 +pip-shims==0.3.2 ptyprocess==0.6.0 enum34==1.1.6 yaspin==0.14.0 cerberus==1.2 git+https://github.com/sarugaku/passa.git@master#egg=passa cursor==1.2.0 +resolvelib==0.2.2 backports.functools_lru_cache==1.5 diff --git a/pipenv/vendor/vendor_pip.txt b/pipenv/vendor/vendor_pip.txt index b9854e9ad1..9389dd947d 100644 --- a/pipenv/vendor/vendor_pip.txt +++ b/pipenv/vendor/vendor_pip.txt @@ -9,14 +9,15 @@ msgpack-python==0.5.6 lockfile==0.12.2 progress==1.4 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==17.1 -pyparsing==2.2.0 -pytoml==0.1.16 +packaging==18.0 +pep517==0.2 +pyparsing==2.2.1 +pytoml==0.1.19 retrying==1.3.3 requests==2.19.1 chardet==3.0.4 idna==2.7 urllib3==1.23 - certifi==2018.4.16 -setuptools==39.2.0 + certifi==2018.8.24 +setuptools==40.4.3 webencodings==0.5.1 diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 4d8a31918f..59b97ca0e3 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -118,15 +118,19 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): """ from .spin import create_spinner - if nospin is False: - try: - import yaspin - except ImportError: + has_yaspin = False + try: + import yaspin + except ImportError: + if not nospin: raise RuntimeError( "Failed to import spinner! Reinstall vistir with command:" " pip install --upgrade vistir[spinner]" ) + else: + spinner_name = "" else: + has_yaspin = True spinner_name = "" if not start_text and nospin is False: start_text = "Running..." @@ -135,6 +139,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): text=start_text, handler_map=handler_map, nospin=nospin, + use_yaspin=has_yaspin ) as _spinner: yield _spinner diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 7d400a4cdb..7342bc97de 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -489,3 +489,36 @@ def chunked(n, iterable): locale_encoding = locale.getdefaultencoding()[1] or "ascii" except Exception: locale_encoding = "ascii" + + +def getpreferredencoding(): + import locale + # Borrowed from Invoke + # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) + _encoding = locale.getpreferredencoding(False) + if six.PY2 and not sys.platform == "win32": + _default_encoding = locale.getdefaultlocale()[1] + if _default_encoding is not None: + _encoding = _default_encoding + return _encoding + + +PREFERRED_ENCODING = getpreferredencoding() + + +def decode_for_output(output): + """Given a string, decode it for output to a terminal + + :param str output: A string to print to a terminal + :return: A re-encoded string using the preferred encoding + :rtype: str + """ + + if not isinstance(output, six.string_types): + return output + try: + output = output.encode(PREFERRED_ENCODING) + except AttributeError: + pass + output = output.decode(PREFERRED_ENCODING) + return output diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 57a90277be..6b6e498f3d 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -33,7 +33,8 @@ class DummySpinner(object): def __init__(self, text="", **kwargs): colorama.init() - self.text = to_native_string(text) + from .misc import decode_for_output + self.text = to_native_string(decode_for_output(text)) self.stdout = kwargs.get("stdout", sys.stdout) self.stderr = kwargs.get("stderr", sys.stderr) self.out_buff = StringIO() @@ -44,10 +45,11 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_val, traceback): - if not exc_type: - self.ok(text=None) - else: - self.write_err(traceback) + if exc_type: + import traceback + from .misc import decode_for_output + self.write_err(decode_for_output(traceback.format_exception(traceback))) + self._close_output_buffer() return False def __getattr__(self, k): @@ -60,33 +62,42 @@ def __getattr__(self, k): else: return retval + def _close_output_buffer(self): + if self.out_buff and not self.out_buff.closed: + try: + self.out_buff.close() + except Exception: + pass + def fail(self, exitcode=1, text="FAIL"): + from .misc import decode_for_output if text and text != "None": - self.write_err(text) - if self.out_buff: - self.out_buff.close() - raise SystemExit(exitcode, text) + self.write_err(decode_for_output(text)) + self._close_output_buffer() def ok(self, text="OK"): if text and text != "None": self.stderr.write(self.text) - if self.out_buff: - self.out_buff.close() + self._close_output_buffer() return 0 def write(self, text=None): + from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - self.stdout.write(to_native_string("\r")) - line = to_native_string("{0}\n".format(text)) + text = decode_for_output(text) + self.stdout.write(decode_for_output("\r")) + line = decode_for_output("{0}\n".format(text)) self.stdout.write(line) self.stdout.write(CLEAR_LINE) def write_err(self, text=None): + from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - self.stderr.write(to_native_string("\r")) - line = to_native_string("{0}\n".format(text)) + text = decode_for_output(text) + self.stderr.write(decode_for_output("\r")) + line = decode_for_output("{0}\n".format(text)) self.stderr.write(line) self.stderr.write(CLEAR_LINE) diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index d01fb98ef1..06b8b62116 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -16,6 +16,9 @@ import threading import time +import colorama +import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS @@ -23,6 +26,9 @@ from .termcolor import colored +colorama.init() + + class Yaspin(object): """Implements a context manager that spawns a thread to write spinner frames into a tty (stdout) during @@ -369,11 +375,14 @@ def _register_signal_handlers(self): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this # signal. - if signal.SIGKILL in self._sigmap.keys(): - raise ValueError( - "Trying to set handler for SIGKILL signal. " - "SIGKILL cannot be cought or ignored in POSIX systems." - ) + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass for sig, sig_handler in iteritems(self._sigmap): # A handler for a particular signal, once set, remains @@ -521,14 +530,12 @@ def _set_text(text): @staticmethod def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() + cursor.hide() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() + cursor.show() @staticmethod def _clear_line(): - sys.stdout.write("\033[K") + sys.stdout.write(chr(27) + "[K") diff --git a/pytest.ini b/pytest.ini index 48dfab025a..2bfca079ab 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] addopts = -ra -n auto -testpaths = tests/ +testpaths = tests ; Add vendor and patched in addition to the default list of ignored dirs norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched news tasks docs filterwarnings = diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 38ec032f6b..4d968a7d0b 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -20,6 +20,7 @@ 'requirements-parser': 'requirements', 'backports.shutil_get_terminal_size': 'backports/shutil_get_terminal_size', 'backports.weakref': 'backports/weakref', + 'backports.functools_lru_cache': 'backports/functools_lru_cache', 'shutil_backports': 'backports/shutil_get_terminal_size', 'python-dotenv': 'dotenv', 'pip-tools': 'piptools', @@ -49,7 +50,8 @@ 'master/LICENSE', 'requirementslib': 'https://github.com/techalchemy/requirementslib/raw/master/LICENSE', 'distlib': 'https://github.com/vsajip/distlib/raw/master/LICENSE.txt', - 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt' + 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt', + 'pyparsing': 'https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE', } FILE_WHITE_LIST = ( @@ -78,7 +80,7 @@ def drop_dir(path): if path.exists() and path.is_dir(): - shutil.rmtree(str(path)) + shutil.rmtree(str(path), ignore_errors=True) def remove_all(paths): @@ -112,8 +114,6 @@ def clean_vendor(ctx, vendor_dir): for item in vendor_dir.iterdir(): if item.is_dir(): shutil.rmtree(str(item)) - elif "LICENSE" in item.name or "COPYING" in item.name: - continue elif item.name not in FILE_WHITE_LIST: item.unlink() else: @@ -332,6 +332,7 @@ def post_install_cleanup(ctx, vendor_dir): # Cleanup setuptools unneeded parts drop_dir(vendor_dir / 'bin') drop_dir(vendor_dir / 'tests') + remove_all(vendor_dir.glob('toml.py')) def vendor(ctx, vendor_dir, package=None, rewrite=True): @@ -355,6 +356,10 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): log("Removing scandir library files...") remove_all(vendor_dir.glob('*.so')) + drop_dir(vendor_dir / 'setuptools') + drop_dir(vendor_dir / 'pkg_resources' / '_vendor') + drop_dir(vendor_dir / 'pkg_resources' / 'extern') + drop_dir(vendor_dir / 'bin') # Global import rewrites log('Renaming specified libs...') diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch index 654a3da9ef..fce6ae89e3 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch @@ -1,20 +1,8 @@ diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py -index c9804ab5..89c95c45 100644 +index f3b9b5b4..182c1c88 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py -@@ -10,10 +10,7 @@ import sysconfig - import warnings - from collections import OrderedDict - --try: -- import pipenv.patched.notpip._internal.utils.glibc --except ImportError: -- import pipenv.patched.notpip.utils.glibc -+import pipenv.patched.notpip._internal.utils.glibc - - logger = logging.getLogger(__name__) - -@@ -157,7 +154,7 @@ def is_manylinux1_compatible(): +@@ -159,7 +159,7 @@ def is_manylinux1_compatible(): pass # Check glibc version. CentOS 5 uses glibc 2.5. diff --git a/tasks/vendoring/patches/patched/pip18.patch b/tasks/vendoring/patches/patched/pip18.patch index 558c28e8f8..150ee32f55 100644 --- a/tasks/vendoring/patches/patched/pip18.patch +++ b/tasks/vendoring/patches/patched/pip18.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/patched/pip/_internal/download.py b/pipenv/patched/pip/_internal/download.py -index 96f3b65c..3fb4ebef 100644 +index 96f3b65c..cc5b3d15 100644 --- a/pipenv/patched/pip/_internal/download.py +++ b/pipenv/patched/pip/_internal/download.py @@ -19,6 +19,7 @@ from pip._vendor.lockfile import LockError @@ -19,43 +19,11 @@ index 96f3b65c..3fb4ebef 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), -@@ -322,7 +323,7 @@ class InsecureHTTPAdapter(HTTPAdapter): - conn.ca_certs = None - - --class PipSession(requests.Session): -+class PipSession(Session): - - timeout = None - -@@ -752,7 +753,7 @@ def _copy_dist_from_dir(link_path, location): - - # build an sdist - setup_py = 'setup.py' -- sdist_args = [sys.executable] -+ sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] - sdist_args.append('-c') - sdist_args.append(SETUPTOOLS_SHIM % setup_py) - sdist_args.append('sdist') diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 8c0ec82c..ad00ba04 100644 +index 8c2f24f1..cdd48874 100644 --- a/pipenv/patched/pip/_internal/index.py +++ b/pipenv/patched/pip/_internal/index.py -@@ -58,11 +58,12 @@ logger = logging.getLogger(__name__) - - class InstallationCandidate(object): - -- def __init__(self, project, version, location): -+ def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) -+ self.requires_python = requires_python - - def __repr__(self): - return "".format( -@@ -168,6 +169,9 @@ class PackageFinder(object): +@@ -246,6 +246,9 @@ class PackageFinder(object): # The Session we'll use to make requests self.session = session @@ -65,7 +33,7 @@ index 8c0ec82c..ad00ba04 100644 # The valid tags to check potential found wheel candidates against self.valid_tags = get_supported( versions=versions, -@@ -220,6 +224,24 @@ class PackageFinder(object): +@@ -298,6 +301,25 @@ class PackageFinder(object): ) self.dependency_links.extend(links) @@ -86,11 +54,12 @@ index 8c0ec82c..ad00ba04 100644 + current_list.append(link) + + return extras ++ + @staticmethod def _sort_locations(locations, expand_dir=False): """ -@@ -272,7 +294,7 @@ class PackageFinder(object): +@@ -350,7 +372,7 @@ class PackageFinder(object): return files, urls @@ -99,7 +68,7 @@ index 8c0ec82c..ad00ba04 100644 """ Function used to generate link sort key for link tuples. The greater the return value, the more preferred it is. -@@ -292,14 +314,19 @@ class PackageFinder(object): +@@ -370,14 +392,19 @@ class PackageFinder(object): if candidate.location.is_wheel: # can raise InvalidWheelFilename wheel = Wheel(candidate.location.filename) @@ -121,7 +90,19 @@ index 8c0ec82c..ad00ba04 100644 if wheel.build_tag is not None: match = re.match(r'^(\d+)(.*)$', wheel.build_tag) build_tag_groups = match.groups() -@@ -484,7 +511,7 @@ class PackageFinder(object): +@@ -528,7 +555,10 @@ class PackageFinder(object): + + page_versions = [] + for page in self._get_pages(url_locations, project_name): +- logger.debug('Analyzing links from page %s', page.url) ++ try: ++ logger.debug('Analyzing links from page %s', page.url) ++ except AttributeError: ++ continue + with indent_log(): + page_versions.extend( + self._package_versions(page.iter_links(), search) +@@ -562,7 +592,7 @@ class PackageFinder(object): dependency_versions ) @@ -130,19 +111,19 @@ index 8c0ec82c..ad00ba04 100644 """Try to find a Link matching req Expects req, an InstallRequirement and upgrade, a boolean -@@ -594,8 +621,9 @@ class PackageFinder(object): +@@ -672,7 +702,10 @@ class PackageFinder(object): continue seen.add(location) - page = self._get_page(location) -- if page is None: + try: + page = self._get_page(location) -+ except requests.HTTPError as e: ++ except requests.HTTPError: ++ continue + if page is None: continue - yield page -@@ -631,7 +659,7 @@ class PackageFinder(object): +@@ -709,7 +742,7 @@ class PackageFinder(object): logger.debug('Skipping link %s; %s', link, reason) self.logged_links.add(link) @@ -151,7 +132,7 @@ index 8c0ec82c..ad00ba04 100644 """Return an InstallationCandidate or None""" version = None if link.egg_fragment: -@@ -647,12 +675,12 @@ class PackageFinder(object): +@@ -725,12 +758,12 @@ class PackageFinder(object): link, 'unsupported archive format: %s' % ext, ) return @@ -166,7 +147,7 @@ index 8c0ec82c..ad00ba04 100644 self._log_skipped_link(link, 'macosx10 one') return if ext == wheel_ext: -@@ -666,7 +694,7 @@ class PackageFinder(object): +@@ -744,7 +777,7 @@ class PackageFinder(object): link, 'wrong project name (not %s)' % search.supplied) return @@ -175,7 +156,7 @@ index 8c0ec82c..ad00ba04 100644 self._log_skipped_link( link, 'it is not compatible with this Python') return -@@ -702,14 +730,14 @@ class PackageFinder(object): +@@ -780,14 +813,14 @@ class PackageFinder(object): link.filename, link.requires_python) support_this_python = True @@ -191,13 +172,30 @@ index 8c0ec82c..ad00ba04 100644 + return InstallationCandidate(search.supplied, version, link, link.requires_python) def _get_page(self, link): - return HTMLPage.get_page(link, session=self.session) + return _get_html_page(link, session=self.session) +diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py +index c736de6c..a78566c1 100644 +--- a/pipenv/patched/pip/_internal/models/candidate.py ++++ b/pipenv/patched/pip/_internal/models/candidate.py +@@ -7,10 +7,11 @@ class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + +- def __init__(self, project, version, location): ++ def __init__(self, project, version, location, requires_python=None): + self.project = project + self.version = parse_version(version) + self.location = location ++ self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py -index 7740c284..b6e946d8 100644 +index 104bea33..ecf78b9a 100644 --- a/pipenv/patched/pip/_internal/operations/prepare.py +++ b/pipenv/patched/pip/_internal/operations/prepare.py @@ -17,7 +17,7 @@ from pip._internal.exceptions import ( - ) + from pip._internal.utils.compat import expanduser from pip._internal.utils.hashes import MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, normalize_path @@ -206,8 +204,8 @@ index 7740c284..b6e946d8 100644 logger = logging.getLogger(__name__) @@ -123,7 +123,11 @@ class IsSDist(DistAbstraction): - "Installing build dependencies" - ) + " and ".join(map(repr, sorted(missing))) + ) - self.req.run_egg_info() + try: @@ -237,10 +235,10 @@ index 7740c284..b6e946d8 100644 # We can't hit this spot and have populate_link return None. diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py -index 0b5c7832..bea31585 100644 +index ab1a0298..763c0a24 100644 --- a/pipenv/patched/pip/_internal/pep425tags.py +++ b/pipenv/patched/pip/_internal/pep425tags.py -@@ -10,7 +10,10 @@ import sysconfig +@@ -10,7 +10,11 @@ import sysconfig import warnings from collections import OrderedDict @@ -249,81 +247,80 @@ index 0b5c7832..bea31585 100644 + import pip._internal.utils.glibc +except ImportError: + import pip.utils.glibc ++ + from pip._internal.utils.compat import get_extension_suffixes logger = logging.getLogger(__name__) - diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py -index 462c80aa..d039adc8 100644 +index c2624fee..ee75acd6 100644 --- a/pipenv/patched/pip/_internal/req/req_install.py +++ b/pipenv/patched/pip/_internal/req/req_install.py -@@ -615,7 +615,7 @@ class InstallRequirement(object): +@@ -452,7 +452,8 @@ class InstallRequirement(object): with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py - base_cmd = [sys.executable, '-c', script] -+ base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ base_cmd = [sys_executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] -@@ -797,7 +797,7 @@ class InstallRequirement(object): +@@ -613,10 +614,11 @@ class InstallRequirement(object): + + with indent_log(): + # FIXME: should we do --install-headers here too? ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: call_subprocess( [ - sys.executable, -+ os.environ.get('PIP_PYTHON_PATH', sys.executable), ++ sys_executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + -@@ -1015,7 +1015,7 @@ class InstallRequirement(object): +@@ -834,7 +836,8 @@ class InstallRequirement(object): def get_install_args(self, global_options, record_filename, root, prefix, pycompile): - install_args = [sys.executable, "-u"] -+ install_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable), "-u"] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ install_args = [sys_executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py -index 2bc6b745..e552afc1 100644 +index b1983171..0bab231d 100644 --- a/pipenv/patched/pip/_internal/req/req_set.py +++ b/pipenv/patched/pip/_internal/req/req_set.py -@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) +@@ -12,13 +12,16 @@ logger = logging.getLogger(__name__) class RequirementSet(object): -- def __init__(self, require_hashes=False): -+ def __init__(self, require_hashes=False, ignore_compatibility=True): +- def __init__(self, require_hashes=False, check_supported_wheels=True): ++ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): """Create a RequirementSet. """ -@@ -24,6 +24,7 @@ class RequirementSet(object): - self.unnamed_requirements = [] - self.successfully_downloaded = [] - self.reqs_to_cleanup = [] -+ self.ignore_compatibility = ignore_compatibility + self.requirements = OrderedDict() + self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels ++ if ignore_compatibility: ++ self.check_supported_wheels = False ++ self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False - def __str__(self): - reqs = [req for req in self.requirements.values() -@@ -65,7 +66,7 @@ class RequirementSet(object): - # environment markers. - if install_req.link and install_req.link.is_wheel: - wheel = Wheel(install_req.link.filename) -- if not wheel.supported(): -+ if not wheel.supported() and not self.ignore_compatibility: - raise InstallationError( - "%s is not a supported wheel on this platform." % - wheel.filename -@@ -151,7 +152,7 @@ class RequirementSet(object): + # Mapping of alias: real_name + self.requirement_aliases = {} +@@ -171,7 +174,7 @@ class RequirementSet(object): return self.requirements[name] if name in self.requirement_aliases: return self.requirements[self.requirement_aliases[name]] - raise KeyError("No project with the name %r" % project_name) -+ # raise KeyError("No project with the name %r" % project_name) ++ pass def cleanup_files(self): """Clean up files, remove builds.""" diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py -index 8480e48c..ffc4aa7d 100644 +index 2d9f1c56..bedc2582 100644 --- a/pipenv/patched/pip/_internal/resolve.py +++ b/pipenv/patched/pip/_internal/resolve.py @@ -35,7 +35,7 @@ class Resolver(object): @@ -356,22 +353,12 @@ index 8480e48c..ffc4aa7d 100644 """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. -@@ -245,6 +249,9 @@ class Resolver(object): - # Tell user what we are doing for this requirement: - # obtain (editable), skipping, processing (local url), collecting - # (remote url or package name) -+ if ignore_requires_python or self.ignore_requires_python: -+ self.ignore_compatibility = True -+ - if req_to_install.constraint or req_to_install.prepared: - return [] - -@@ -260,11 +267,17 @@ class Resolver(object): +@@ -260,11 +264,17 @@ class Resolver(object): try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_requires_python: -+ if self.ignore_compatibility: ++ if self.ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -385,14 +372,13 @@ index 8480e48c..ffc4aa7d 100644 more_reqs = [] def add_req(subreq, extras_requested): -@@ -290,10 +303,14 @@ class Resolver(object): - # We add req_to_install before its dependencies, so that we +@@ -291,9 +301,13 @@ class Resolver(object): # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here + available_requested = sorted( + set(dist.extras) & set(req_to_install.extras) + ) - # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -400,7 +386,7 @@ index 8480e48c..ffc4aa7d 100644 ) if not self.ignore_dependencies: -@@ -317,6 +334,19 @@ class Resolver(object): +@@ -317,6 +331,19 @@ class Resolver(object): for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) @@ -408,7 +394,7 @@ index 8480e48c..ffc4aa7d 100644 + for available in available_requested: + if hasattr(dist, '_DistInfoDistribution__dep_map'): + for req in dist._DistInfoDistribution__dep_map[available]: -+ req = InstallRequirement.from_req( ++ req = install_req_from_req( + str(req), + req_to_install, + isolated=self.isolated, @@ -420,24 +406,11 @@ index 8480e48c..ffc4aa7d 100644 if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took -diff --git a/pipenv/patched/pip/_internal/utils/misc.py b/pipenv/patched/pip/_internal/utils/misc.py -index 3236af63..439a831d 100644 ---- a/pipenv/patched/pip/_internal/utils/misc.py -+++ b/pipenv/patched/pip/_internal/utils/misc.py -@@ -96,7 +96,7 @@ def get_prog(): - try: - prog = os.path.basename(sys.argv[0]) - if prog in ('__main__.py', '-c'): -- return "%s -m pip" % sys.executable -+ return "%s -m pip" % os.environ.get('PIP_PYTHON_PATH', sys.executable) - else: - return prog - except (AttributeError, TypeError, IndexError): diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py -index 5f9bb93d..276a9ccc 100644 +index c43142f0..f241cce0 100644 --- a/pipenv/patched/pip/_internal/utils/packaging.py +++ b/pipenv/patched/pip/_internal/utils/packaging.py -@@ -28,7 +28,7 @@ def check_requires_python(requires_python): +@@ -29,7 +29,7 @@ def check_requires_python(requires_python): requires_python_specifier = specifiers.SpecifierSet(requires_python) # We only use major.minor.micro @@ -446,33 +419,21 @@ index 5f9bb93d..276a9ccc 100644 return python_version in requires_python_specifier -@@ -40,20 +40,17 @@ def get_metadata(dist): - return dist.get_metadata('PKG-INFO') +@@ -48,9 +48,11 @@ def get_metadata(dist): + return feed_parser.close() -def check_dist_requires_python(dist): +def check_dist_requires_python(dist, absorb=True): - metadata = get_metadata(dist) - feed_parser = FeedParser() - feed_parser.feed(metadata) - pkg_info_dict = feed_parser.close() + pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') -+ if not absorb: ++ if absorb: + return requires_python try: if not check_requires_python(requires_python): -- raise exceptions.UnsupportedPythonVersion( -- "%s requires Python '%s' but the running Python is %s" % ( -- dist.project_name, -- requires_python, -- '.'.join(map(str, sys.version_info[:3])),) -- ) -+ return requires_python - except specifiers.InvalidSpecifier as e: - logger.warning( - "Package %s has an invalid Requires-Python entry %s - %s", + raise exceptions.UnsupportedPythonVersion( diff --git a/pipenv/patched/pip/_internal/wheel.py b/pipenv/patched/pip/_internal/wheel.py -index fcf9d3d3..d8aff848 100644 +index 5ce890eb..46c0181c 100644 --- a/pipenv/patched/pip/_internal/wheel.py +++ b/pipenv/patched/pip/_internal/wheel.py @@ -83,7 +83,7 @@ def fix_script(path): @@ -484,28 +445,24 @@ index fcf9d3d3..d8aff848 100644 firstline = b'#!' + exename + os.linesep.encode("ascii") rest = script.read() with open(path, 'wb') as script: -@@ -665,7 +665,7 @@ class WheelBuilder(object): +@@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): + ] + # If an executable sits with sys.executable, we don't warn for it. + # This covers the case of venv invocations without activating the venv. +- not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) ++ executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) ++ not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) + warn_for = { + parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() + if os.path.normcase(parent_dir) not in not_warn_dirs +@@ -667,8 +668,9 @@ class WheelBuilder(object): + # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. ++ executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) return [ - sys.executable, '-u', '-c', -+ os.environ.get('PIP_PYTHON_PATH', sys.executable), '-u', '-c', ++ executable_loc, '-u', '-c', SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 793dd1cb..426880e9 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py -@@ -477,7 +477,10 @@ class PackageFinder(object): - - page_versions = [] - for page in self._get_pages(url_locations, project_name): -- logger.debug('Analyzing links from page %s', page.url) -+ try: -+ logger.debug('Analyzing links from page %s', page.url) -+ except AttributeError: -+ continue - with indent_log(): - page_versions.extend( - self._package_versions(page.links, search) diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch index ae0bf088b0..7e42237f4f 100644 --- a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -1,8 +1,8 @@ -diff --git a/delegator.py b/delegator.py -index 25d21f0..582f4fe 100644 ---- a/delegator.py -+++ b/delegator.py -@@ -7,15 +7,18 @@ import locale +diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py +index d15aeb97..cf6f91c8 100644 +--- a/pipenv/vendor/delegator.py ++++ b/pipenv/vendor/delegator.py +@@ -7,6 +7,8 @@ import locale import errno from pexpect.popen_spawn import PopenSpawn @@ -11,96 +11,13 @@ index 25d21f0..582f4fe 100644 # Include `unicode` in STR_TYPES for Python 2.X try: - STR_TYPES = (str, unicode) - except NameError: -- STR_TYPES = (str, ) -+ STR_TYPES = (str,) - - TIMEOUT = 30 - -+ - def pid_exists(pid): - """Check whether pid exists in the current process table.""" - if pid == 0: -@@ -43,8 +46,8 @@ def pid_exists(pid): - else: - return True - --class Command(object): - -+class Command(object): - def __init__(self, cmd, timeout=TIMEOUT): - super(Command, self).__init__() - self.cmd = cmd -@@ -56,7 +59,7 @@ class Command(object): - self.__err = None - - def __repr__(self): -- return ''.format(self.cmd) -+ return "".format(self.cmd) - - @property - def _popen_args(self): -@@ -65,27 +68,23 @@ class Command(object): - @property - def _default_popen_kwargs(self): - return { -- 'env': os.environ.copy(), -- 'stdin': subprocess.PIPE, -- 'stdout': subprocess.PIPE, -- 'stderr': subprocess.PIPE, -- 'shell': True, -- 'universal_newlines': True, -- 'bufsize': 0 -+ "env": os.environ.copy(), -+ "stdin": subprocess.PIPE, -+ "stdout": subprocess.PIPE, -+ "stderr": subprocess.PIPE, -+ "shell": True, -+ "universal_newlines": True, -+ "bufsize": 0, - } - - @property - def _default_pexpect_kwargs(self): -- encoding = 'utf-8' -- if sys.platform == 'win32': -+ encoding = "utf-8" -+ if sys.platform == "win32": - default_encoding = locale.getdefaultlocale()[1] - if default_encoding is not None: - encoding = default_encoding -- return { -- 'env': os.environ.copy(), -- 'encoding': encoding, -- 'timeout': self.timeout -- } -+ return {"env": os.environ.copy(), "encoding": encoding, "timeout": self.timeout} - - @property - def _uses_subprocess(self): -@@ -99,18 +98,25 @@ class Command(object): - def std_out(self): - return self.subprocess.stdout - -+ @property -+ def ok(self): -+ return self.return_code == 0 -+ - @property - def _pexpect_out(self): - if self.subprocess.encoding: -- result = '' -+ result = "" - else: -- result = b'' -+ result = b"" - +@@ -110,8 +112,11 @@ class Command(object): if self.subprocess.before: result += self.subprocess.before - if self.subprocess.after: +- if self.subprocess.after: - result += self.subprocess.after ++ if self.subprocess.after and self.subprocess.after is not pexpect.EOF: + try: + result += self.subprocess.after + except (pexpect.EOF, pexpect.TIMEOUT): @@ -108,53 +25,17 @@ index 25d21f0..582f4fe 100644 result += self.subprocess.read() return result -@@ -148,7 +154,7 @@ class Command(object): - def pid(self): - """The process' PID.""" - # Support for pexpect's functionality. -- if hasattr(self.subprocess, 'proc'): -+ if hasattr(self.subprocess, "proc"): - return self.subprocess.proc.pid - # Standard subprocess method. - return self.subprocess.pid -@@ -177,23 +183,24 @@ class Command(object): +@@ -178,6 +183,7 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() -- popen_kwargs['universal_newlines'] = not binary + del popen_kwargs["stdin"] -+ popen_kwargs["universal_newlines"] = not binary + popen_kwargs["universal_newlines"] = not binary if cwd: -- popen_kwargs['cwd'] = cwd -+ popen_kwargs["cwd"] = cwd - if env: -- popen_kwargs['env'].update(env) -+ popen_kwargs["env"].update(env) - s = subprocess.Popen(self._popen_args, **popen_kwargs) - # Otherwise, use pexpect. - else: - pexpect_kwargs = self._default_pexpect_kwargs.copy() - if binary: -- pexpect_kwargs['encoding'] = None -+ pexpect_kwargs["encoding"] = None - if cwd: -- pexpect_kwargs['cwd'] = cwd -+ pexpect_kwargs["cwd"] = cwd - if env: -- pexpect_kwargs['env'].update(env) -+ pexpect_kwargs["env"].update(env) - # Enable Python subprocesses to work with expect functionality. -- pexpect_kwargs['env']['PYTHONUNBUFFERED'] = '1' -+ pexpect_kwargs["env"]["PYTHONUNBUFFERED"] = "1" - s = PopenSpawn(self._popen_args, **pexpect_kwargs) - self.subprocess = s - self.was_run = True -@@ -202,15 +209,18 @@ class Command(object): - """Waits on the given pattern to appear in std_out""" - + popen_kwargs["cwd"] = cwd +@@ -205,7 +211,10 @@ class Command(object): if self.blocking: -- raise RuntimeError('expect can only be used on non-blocking commands.') -+ raise RuntimeError("expect can only be used on non-blocking commands.") + raise RuntimeError("expect can only be used on non-blocking commands.") - self.subprocess.expect(pattern=pattern, timeout=timeout) + try: @@ -164,14 +45,7 @@ index 25d21f0..582f4fe 100644 def send(self, s, end=os.linesep, signal=False): """Sends the given string or signal to std_in.""" - - if self.blocking: -- raise RuntimeError('send can only be used on non-blocking commands.') -+ raise RuntimeError("send can only be used on non-blocking commands.") - - if not signal: - if self._uses_subprocess: -@@ -233,14 +243,25 @@ class Command(object): +@@ -234,14 +243,25 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: # consume stdout and stderr @@ -204,7 +78,7 @@ index 25d21f0..582f4fe 100644 def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next -@@ -262,7 +283,6 @@ class Command(object): +@@ -263,7 +283,6 @@ class Command(object): c.run(block=False, cwd=cwd) if data: c.send(data) @@ -212,24 +86,3 @@ index 25d21f0..582f4fe 100644 c.block() return c -@@ -273,12 +293,12 @@ def _expand_args(command): - # Prepare arguments. - if isinstance(command, STR_TYPES): - if sys.version_info[0] == 2: -- splitter = shlex.shlex(command.encode('utf-8')) -+ splitter = shlex.shlex(command.encode("utf-8")) - elif sys.version_info[0] == 3: - splitter = shlex.shlex(command) - else: -- splitter = shlex.shlex(command.encode('utf-8')) -- splitter.whitespace = '|' -+ splitter = shlex.shlex(command.encode("utf-8")) -+ splitter.whitespace = "|" - splitter.whitespace_split = True - command = [] - -@@ -319,4 +339,3 @@ def run(command, block=True, binary=False, timeout=TIMEOUT, cwd=None, env=None): - c.block() - - return c -- diff --git a/tasks/vendoring/patches/vendor/passa-close-session.patch b/tasks/vendoring/patches/vendor/passa-close-session.patch new file mode 100644 index 0000000000..38846befb5 --- /dev/null +++ b/tasks/vendoring/patches/vendor/passa-close-session.patch @@ -0,0 +1,12 @@ +diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py +index 53b19b17..358cc33b 100644 +--- a/pipenv/vendor/passa/internals/dependencies.py ++++ b/pipenv/vendor/passa/internals/dependencies.py +@@ -154,6 +154,7 @@ def _get_dependencies_from_json(ireq, sources): + return dependencies + except Exception as e: + print("unable to read dependencies via {0} ({1})".format(url, e)) ++ session.close() + return + + diff --git a/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch b/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch new file mode 100644 index 0000000000..69e4cac735 --- /dev/null +++ b/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch @@ -0,0 +1,28 @@ +diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py +index 2a848922..57a90277 100644 +--- a/pipenv/vendor/vistir/spin.py ++++ b/pipenv/vendor/vistir/spin.py +@@ -5,6 +5,7 @@ import os + import signal + import sys + ++import colorama + import cursor + import six + +@@ -31,6 +32,7 @@ CLEAR_LINE = chr(27) + "[K" + + class DummySpinner(object): + def __init__(self, text="", **kwargs): ++ colorama.init() + self.text = to_native_string(text) + self.stdout = kwargs.get("stdout", sys.stdout) + self.stderr = kwargs.get("stderr", sys.stderr) +@@ -112,7 +114,6 @@ class VistirSpinner(base_obj): + """ + + self.handler = handler +- import colorama + colorama.init() + sigmap = {} + if handler: From 8a56a75cd544410fa39965aedcecf21539d812f0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 27 Oct 2018 21:02:22 -0400 Subject: [PATCH 10/13] Fix virtualenv creation and error logging Signed-off-by: Dan Ryan --- pipenv/_compat.py | 15 ++++++++++-- pipenv/project.py | 25 ++++++++------------ pipenv/utils.py | 46 ++++++++++++++++++++++++++++++++++--- tasks/vendoring/__init__.py | 1 + 4 files changed, 66 insertions(+), 21 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 087025a4a2..6cc4c73216 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -11,6 +11,7 @@ import six import sys import warnings +import vistir from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp from .utils import logging, rmtree @@ -367,12 +368,22 @@ def force_encoding(): OUT_ENCODING, ERR_ENCODING = force_encoding() +UNICODE_TO_ASCII_TRANSLATION_MAP = { + 8230: u"...", + 8211: u"-" +} + + def decode_output(output): if not isinstance(output, six.string_types): return output try: output = output.encode(DEFAULT_ENCODING) - except AttributeError: - pass + except (AttributeError, UnicodeDecodeError): + if six.PY2: + output = unicode.translate(vistir.misc.to_text(output), + UNICODE_TO_ASCII_TRANSLATION_MAP) + else: + output = output.translate(UNICODE_TO_ASCII_TRANSLATION_MAP) output = output.decode(DEFAULT_ENCODING) return output diff --git a/pipenv/project.py b/pipenv/project.py index dcf9b41794..f7e57245ad 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -15,6 +15,7 @@ import pipfile.api import six import vistir +import virtualenv as _virtualenv import toml from .cmdparse import Script @@ -985,25 +986,17 @@ def _pyversion(self): @property def env_paths(self): - import sysconfig location = self.virtualenv_location if self.virtualenv_location else sys.prefix - prefix = vistir.compat.Path(location).as_posix() - scheme = sysconfig._get_default_scheme() - if not scheme: - scheme = "posix_prefix" if not sys.platform == "win32" else "nt" - config = { - "base": prefix, - "installed_base": prefix, - "platbase": prefix, - "installed_platbase": prefix - } - config.update(self._pyversion) + prefix = vistir.compat.Path(location) + home, lib, inc, bin_ = _virtualenv.path_locations(prefix) paths = { - k: v.format(**config) - for k, v in sysconfig._INSTALL_SCHEMES[scheme].items() + "lib": lib, + "include": inc, + "scripts": bin_, + "purelib": lib, + "prefix": home, + "base": home } - if "prefix" not in paths: - paths["prefix"] = prefix return paths @cached_property diff --git a/pipenv/utils.py b/pipenv/utils.py index 893ed88f08..9fa6e571f7 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -331,7 +331,7 @@ def venv_resolve_deps( pypi_mirror=None, ): from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, to_native_string + from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir from .cmdparse import Script from .core import spinner @@ -370,7 +370,7 @@ def venv_resolve_deps( with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, nospin=environments.PIPENV_NOSPIN) as sp: c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) - _out = to_native_string("") + _out = decode_output("") result = None while True: try: @@ -404,7 +404,7 @@ def venv_resolve_deps( try: return json.loads(c.out.split("RESULTS:")[1].strip()) - except IndexError: + except (IndexError, JSONDecodeError): click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) raise RuntimeError("There was a problem with locking.") @@ -1305,3 +1305,43 @@ def parse_indexes(line): indexes = index + extra_indexes trusted_hosts = args.trusted_host if args.trusted_host else [] return indexes, trusted_hosts, remainder + + +def fix_venv_site(venv_lib_dir): + # From https://github.com/pypa/pip/blob/master/tests/lib/venv.py#L84 + # Prevent accidental inclusions of site packages during virtualenv operations + from .vendor.vistir.compat import Path + import compileall + site_py = Path(venv_lib_dir).joinpath('site.py').as_posix() + with open(site_py) as fp: + site_contents = fp.read() + for pattern, replace in ( + ( + # Ensure enabling user site does not result in adding + # the real site-packages' directory to `sys.path`. + ( + '\ndef virtual_addsitepackages(known_paths):\n' + ), + ( + '\ndef virtual_addsitepackages(known_paths):\n' + ' return known_paths\n' + ), + ), + ( + # Fix sites ordering: user site must be added before system. + ( + '\n paths_in_sys = addsitepackages(paths_in_sys)' + '\n paths_in_sys = addusersitepackages(paths_in_sys)\n' + ), + ( + '\n paths_in_sys = addusersitepackages(paths_in_sys)' + '\n paths_in_sys = addsitepackages(paths_in_sys)\n' + ), + ), + ): + if pattern in site_contents and replace not in site_contents: + site_contents = site_contents.replace(pattern, replace) + with open(site_py, 'w') as fp: + fp.write(site_contents) + # Make sure bytecode is up-to-date too. + assert compileall.compile_file(str(site_py), quiet=1, force=True) diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 4d968a7d0b..ea0d038c98 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -52,6 +52,7 @@ 'distlib': 'https://github.com/vsajip/distlib/raw/master/LICENSE.txt', 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt', 'pyparsing': 'https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE', + 'resolvelib': 'https://raw.githubusercontent.com/sarugaku/resolvelib/master/LICENSE' } FILE_WHITE_LIST = ( From d069dff844c04babe4827eb33e443c180e83c5bb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 27 Oct 2018 21:46:11 -0400 Subject: [PATCH 11/13] Update project and spinner outputs Signed-off-by: Dan Ryan Try again Signed-off-by: Dan Ryan Fix test config to skip failed removals Signed-off-by: Dan Ryan Update piptools to handle some errors Signed-off-by: Dan Ryan Fix test config to skip failed removals Signed-off-by: Dan Ryan Update tempfile.py Use vistirs temporary directory implementation Update temp_dir.py Force pip to use weakrefs in tempdirs Fix pip implementation to set name of tempdir typo fix Signed-off-by: Dan Ryan fix pip tempdir implementation Signed-off-by: Dan Ryan Update tempfiles to use weakrefs Signed-off-by: Dan Ryan fix patch paths Signed-off-by: Dan Ryan Fix pip tempdir implementation Signed-off-by: Dan Ryan Syntax error fix Signed-off-by: Dan Ryan Unconstrain windows tests Signed-off-by: Dan Ryan Update dependencies, add news Signed-off-by: Dan Ryan Fix pythonfinder path search nesting bug - Fixes #3121 Signed-off-by: Dan Ryan Update requirementslib - Fix subdirectory issue Signed-off-by: Dan Ryan Fix logic error Signed-off-by: Dan Ryan conditional builds Signed-off-by: Dan Ryan --- .vsts-ci/linux.yml | 19 +++ .vsts-ci/steps/run-tests.yml | 2 +- .vsts-ci/windows.yml | 19 +++ news/3090.bugfix.rst | 1 + news/3094.bugfix.rst | 1 + news/3102.bugfix.rst | 1 + news/3109.bugfix.rst | 1 + news/3113.bugfix.rst | 1 + news/3114.bugfix.rst | 1 + news/3117.bugfix.rst | 1 + news/3121.bugfix.rst | 1 + news/3121.vendor.rst | 1 + pipenv/core.py | 3 +- .../notpip/_internal/utils/temp_dir.py | 38 ++++- pipenv/patched/piptools/repositories/pypi.py | 19 ++- pipenv/project.py | 33 +++- pipenv/resolver.py | 5 +- pipenv/utils.py | 26 ++- pipenv/vendor/pythonfinder/utils.py | 24 ++- pipenv/vendor/requirementslib/__init__.py | 2 +- .../requirementslib/models/requirements.py | 64 +++++--- pipenv/vendor/requirementslib/models/vcs.py | 26 ++- pipenv/vendor/resolvelib/LICENSE | 13 ++ pipenv/vendor/vendor.txt | 2 +- pipenv/vendor/vistir/__init__.py | 30 +++- pipenv/vendor/vistir/path.py | 65 +++++--- pipenv/vendor/vistir/spin.py | 2 +- tasks/vendoring/patches/patched/pip18.patch | 67 ++++++++ .../vendoring/patches/patched/piptools.patch | 152 +++++++++++------- tests/integration/conftest.py | 22 +-- 30 files changed, 478 insertions(+), 164 deletions(-) create mode 100644 news/3090.bugfix.rst create mode 100644 news/3094.bugfix.rst create mode 100644 news/3102.bugfix.rst create mode 100644 news/3109.bugfix.rst create mode 100644 news/3113.bugfix.rst create mode 100644 news/3114.bugfix.rst create mode 100644 news/3117.bugfix.rst create mode 100644 news/3121.bugfix.rst create mode 100644 news/3121.vendor.rst create mode 100644 pipenv/vendor/resolvelib/LICENSE diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 7968801c5e..95c62dc51f 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -1,3 +1,22 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + phases: - template: phases/test.yml parameters: diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index 4c2640ab37..a7b99a1317 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -12,7 +12,7 @@ steps: $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" $env:TEMP='T:\' - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor -k 'test_get_vcs_refs or test_install_editable_git_tag' --junitxml=test-results.xml tests + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index a397a23c67..e423c3abba 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -1,3 +1,22 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + phases: - template: phases/test.yml parameters: diff --git a/news/3090.bugfix.rst b/news/3090.bugfix.rst new file mode 100644 index 0000000000..af772c3d63 --- /dev/null +++ b/news/3090.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug in ``requirementslib`` which prevented successful installation from mercurial repositories. diff --git a/news/3094.bugfix.rst b/news/3094.bugfix.rst new file mode 100644 index 0000000000..e17b2e3273 --- /dev/null +++ b/news/3094.bugfix.rst @@ -0,0 +1 @@ +Fixed random resource warnings when using pyenv or any other subprocess calls. diff --git a/news/3102.bugfix.rst b/news/3102.bugfix.rst new file mode 100644 index 0000000000..2224ac0bd9 --- /dev/null +++ b/news/3102.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3109.bugfix.rst b/news/3109.bugfix.rst new file mode 100644 index 0000000000..af5718b6ae --- /dev/null +++ b/news/3109.bugfix.rst @@ -0,0 +1 @@ +Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures. diff --git a/news/3113.bugfix.rst b/news/3113.bugfix.rst new file mode 100644 index 0000000000..af43b87df8 --- /dev/null +++ b/news/3113.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue resolving virtualenv paths for users without ``platlib`` values on their systems. diff --git a/news/3114.bugfix.rst b/news/3114.bugfix.rst new file mode 100644 index 0000000000..2224ac0bd9 --- /dev/null +++ b/news/3114.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3117.bugfix.rst b/news/3117.bugfix.rst new file mode 100644 index 0000000000..2224ac0bd9 --- /dev/null +++ b/news/3117.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3121.bugfix.rst b/news/3121.bugfix.rst new file mode 100644 index 0000000000..fb815c428c --- /dev/null +++ b/news/3121.bugfix.rst @@ -0,0 +1 @@ +Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/news/3121.vendor.rst b/news/3121.vendor.rst new file mode 100644 index 0000000000..fb815c428c --- /dev/null +++ b/news/3121.vendor.rst @@ -0,0 +1 @@ +Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/pipenv/core.py b/pipenv/core.py index ee1658bf6e..59b9a29c4c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1355,7 +1355,8 @@ def pip_install( write_to_tmpfile = False if requirement: needs_hashes = not requirement.editable and not ignore_hashes and r is None - write_to_tmpfile = needs_hashes + has_subdir = requirement.is_vcs and requirement.req.subdirectory + write_to_tmpfile = needs_hashes or has_subdir if not trusted_hosts: trusted_hosts = [] diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py index ba472b0d79..893dc975e6 100644 --- a/pipenv/patched/notpip/_internal/utils/temp_dir.py +++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -3,8 +3,10 @@ import logging import os.path import tempfile +import warnings from pipenv.patched.notpip._internal.utils.misc import rmtree +from pipenv.vendor.vistir.compat import finalize, ResourceWarning logger = logging.getLogger(__name__) @@ -45,6 +47,20 @@ def __init__(self, path=None, delete=None, kind="temp"): self.path = path self.delete = delete self.kind = kind + self._finalizer = None + if path: + self._register_finalizer() + + def _register_finalizer(self): + if self.delete and self.path: + self._finalizer = finalize( + self, + self._cleanup, + self.path, + warn_message=None + ) + else: + self._finalizer = None def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.path) @@ -72,11 +88,27 @@ def create(self): self.path = os.path.realpath( tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) ) + self._register_finalizer() logger.debug("Created temporary directory: {}".format(self.path)) + @classmethod + def _cleanup(cls, name, warn_message=None): + try: + rmtree(name) + except OSError: + pass + else: + if warn_message: + warnings.warn(warn_message, ResourceWarning) + def cleanup(self): """Remove the temporary directory created and reset state """ - if self.path is not None and os.path.exists(self.path): - rmtree(self.path) - self.path = None + if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): + if os.path.exists(self.path): + try: + rmtree(self.path) + except OSError: + pass + else: + self.path = None diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index c18716706b..2a0743a338 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -19,11 +19,10 @@ InstallRequirement, SafeFileCache ) -os.environ["PIP_SHIMS_BASE_MODULE"] = str("notpip") +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") from pip_shims.shims import do_import, VcsSupport, WheelCache from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet, Specifier -from packaging.markers import Op, Value, Variable, Marker InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver @@ -31,7 +30,7 @@ from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR from ..exceptions import NoCandidateFound from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, - make_install_requirement, clean_requires_python) + make_install_requirement, clean_requires_python) from .base import BaseRepository try: @@ -243,6 +242,7 @@ def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist= dist = None ireq.isolated = False ireq._wheel_cache = wheel_cache + try: from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer except ImportError: @@ -295,7 +295,18 @@ def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist= resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() + + cleanup_fn = getattr(reqset, "cleanup_files", None) + if cleanup_fn is not None: + try: + cleanup_fn() + except OSError: + pass + + if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): + if ireq.editable: + self._source_dir = TemporaryDirectory(fs_str("source")) + ireq.ensure_has_source_dir(self.source_dir) if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # Collect setup_requires info from local eggs. diff --git a/pipenv/project.py b/pipenv/project.py index f7e57245ad..d25dba9a56 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -15,7 +15,6 @@ import pipfile.api import six import vistir -import virtualenv as _virtualenv import toml from .cmdparse import Script @@ -84,6 +83,8 @@ def default(self, obj): if isinstance(obj, (ContainerElement, TokenElement)): return obj.primitive_value + elif isinstance(obj, vistir.compat.Path): + obj = obj.as_posix() return super(_LockFileEncoder, self).default(obj) def encode(self, obj): @@ -988,7 +989,35 @@ def _pyversion(self): def env_paths(self): location = self.virtualenv_location if self.virtualenv_location else sys.prefix prefix = vistir.compat.Path(location) - home, lib, inc, bin_ = _virtualenv.path_locations(prefix) + import importlib + try: + _virtualenv = importlib.import_module("virtualenv") + except ImportError: + with vistir.contextmanagers.temp_path(): + from string import Formatter + formatter = Formatter() + import sysconfig + if getattr(sys, "real_prefix", None): + scheme = sysconfig._get_default_scheme() + sysconfig._INSTALL_SCHEMES["posix_prefix"]["purelib"] + if not scheme: + scheme = "posix_prefix" if not sys.platform == "win32" else "nt" + is_purelib = "purelib" in sysconfig._INSTALL_SCHEMES[scheme] + lib_key = "purelib" if is_purelib else "platlib" + lib = sysconfig._INSTALL_SCHEMES[scheme][lib_key] + fields = [field for _, field, _, _ in formatter.parse() if field] + config = { + "py_version_short": self._pyversion, + } + for field in fields: + if field not in config: + config[field] = prefix + sys.path = [ + os.path.join(sysconfig._INSTALL_SCHEMES[scheme][lib_key], "site-packages"), + ] + sys.path + six.reload_module(importlib) + _virtualenv = importlib.import_module("virtualenv") + home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) paths = { "lib": lib, "include": inc, diff --git a/pipenv/resolver.py b/pipenv/resolver.py index b5ea41d9ed..9ef46878c9 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -70,7 +70,6 @@ def resolve(packages, pre, project, sources, clear, system, requirements_dir=Non ) from pipenv.core import project - sources = ( replace_pypi_sources(project.pipfile_sources, pypi_mirror_source) if pypi_mirror_source @@ -111,8 +110,8 @@ def main(): parsed, remaining = parser.parse_known_args() # sys.argv = remaining parsed = handle_parsed_args(parsed) - _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir, - parsed.packages) + _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, + parsed.requirements_dir, parsed.packages) if __name__ == "__main__": diff --git a/pipenv/utils.py b/pipenv/utils.py index 9fa6e571f7..c0979863c9 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -345,7 +345,6 @@ def venv_resolve_deps( return [] req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -364,7 +363,6 @@ def venv_resolve_deps( os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - out = to_native_string("") EOF.__module__ = "pexpect.exceptions" with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, @@ -430,6 +428,8 @@ def resolve_deps( index_lookup = {} markers_lookup = {} python_path = which("python", allow_global=allow_global) + if not os.environ.get("PIP_SRC"): + os.environ["PIP_SRC"] = project.virtualenv_src_location backup_python_path = sys.executable results = [] if not deps: @@ -1116,23 +1116,19 @@ def get_vcs_deps( packages = getattr(project, section) except AttributeError: return [], [] - if os.environ.get("PIP_SRC"): - src_dir = Path( - os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) - ) - src_dir.mkdir(mode=0o775, exist_ok=True) - else: - src_dir = create_tracked_tempdir(prefix="pipenv-lock-dir") for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name commit_hash = None if requirement.is_vcs: - with locked_repository(requirement) as repo: - commit_hash = repo.get_commit_hash() - lockfile[name] = requirement.pipfile_entry[1] - lockfile[name]['ref'] = commit_hash - reqs.append(requirement) + try: + with locked_repository(requirement) as repo: + commit_hash = repo.get_commit_hash() + lockfile[name] = requirement.pipfile_entry[1] + lockfile[name]['ref'] = commit_hash + reqs.append(requirement) + except OSError: + continue return reqs, lockfile @@ -1257,11 +1253,11 @@ def is_virtual_environment(path): @contextmanager def locked_repository(requirement): from .vendor.vistir.path import create_tracked_tempdir - src_dir = create_tracked_tempdir(prefix="pipenv-src") if not requirement.is_vcs: return original_base = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip") + src_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-src") try: with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo: yield repo diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index fd5ac99d6f..ca07b42f76 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function +import itertools import locale import os import subprocess @@ -21,6 +22,9 @@ except ImportError: from backports.functools_lru_cache import lru_cache +six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) +from six.moves import Iterable + PYTHON_IMPLEMENTATIONS = ( "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", @@ -123,7 +127,21 @@ def filter_pythons(path): return filter(lambda x: path_is_python(x), path.iterdir()) +# def unnest(item): +# if isinstance(next((i for i in item), None), (list, tuple)): +# return chain(*filter(None, item)) +# return chain(filter(None, item)) + + def unnest(item): - if isinstance(next((i for i in item), None), (list, tuple)): - return chain(*filter(None, item)) - return chain(filter(None, item)) + if isinstance(item, Iterable) and not isinstance(item, six.string_types): + item, target = itertools.tee(item, 2) + else: + target = item + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = itertools.tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index ba40e5f51f..8ceccd792d 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.1' +__version__ = '1.2.2' import logging diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 3a029cbcda..b3bc013239 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -16,7 +16,7 @@ from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path +from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path, Link from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path @@ -41,11 +41,11 @@ ) -@attr.s +@attr.s(slots=True) class NamedRequirement(BaseRequirement): name = attr.ib() version = attr.ib(validator=attr.validators.optional(validate_specifiers)) - req = attr.ib() + req = attr.ib(type=PkgResourcesRequirement) extras = attr.ib(default=attr.Factory(list)) editable = attr.ib(default=False) @@ -109,21 +109,28 @@ def pipfile_part(self): ) -@attr.s +@attr.s(slots=True) class FileRequirement(BaseRequirement): """File requirements for tar.gz installable files or wheels or setup.py containing directories.""" + #: Path to the relevant `setup.py` location setup_path = attr.ib(default=None) + #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) - # : path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - editable = attr.ib(default=False, type=bool) - extras = attr.ib(default=attr.Factory(list), type=list) - uri = attr.ib(type=six.string_types) + #: Whether the package is editable + editable = attr.ib(default=False) + #: Extras if applicable + extras = attr.ib(default=attr.Factory(list)) + #: URI of the package + uri = attr.ib() + #: Link object representing the package to clone link = attr.ib() - name = attr.ib(type=six.string_types) - req = attr.ib(type=PkgResourcesRequirement) - _has_hashed_name = False + _has_hashed_name = attr.ib(default=False) + #: Package name + name = attr.ib() + #: A :class:`~pkg_resources.Requirement` isntance + req = attr.ib() _uri_scheme = attr.ib(default=None) @classmethod @@ -470,14 +477,19 @@ def pipfile_part(self): return {name: pipfile_dict} -@attr.s +@attr.s(slots=True) class VCSRequirement(FileRequirement): + #: Whether the repository is editable editable = attr.ib(default=None) + #: URI for the repository uri = attr.ib(default=None) + #: path to the repository, if it's local path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + #: vcs type, i.e. git/hg/svn vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) - # : vcs reference name (branch / commit / tag) + #: vcs reference name (branch / commit / tag) ref = attr.ib(default=None) + #: Subdirectory to use for installation if applicable subdirectory = attr.ib(default=None) _repo = attr.ib(default=None) _base_line = attr.ib(default=None) @@ -585,16 +597,28 @@ def get_checkout_dir(self, src_dir=None): def get_vcs_repo(self, src_dir=None): from .vcs import VCSRepository checkout_dir = self.get_checkout_dir(src_dir=src_dir) - url = "{0}#egg={1}".format(self.vcs_uri, self.name) + url = build_vcs_link( + self.vcs, + self.uri, + name=self.name, + ref=self.ref, + subdirectory=self.subdirectory, + extras=self.extras + ) vcsrepo = VCSRepository( url=url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, - vcs_type=self.vcs + vcs_type=self.vcs, + subdirectory=self.subdirectory ) if not self.is_local: vcsrepo.obtain() + if self.subdirectory: + self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") + else: + self.setup_path = os.path.join(checkout_dir, "setup.py") return vcsrepo def get_commit_hash(self): @@ -612,15 +636,15 @@ def update_repo(self, src_dir=None, ref=None): if not self.is_local and ref is not None: self.repo.checkout_ref(ref) repo_hash = self.repo.get_commit_hash() + self.req.revision = repo_hash return repo_hash @contextmanager def locked_vcs_repo(self, src_dir=None): + if not src_dir: + src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) - if self.ref and not self.is_local: - vcsrepo.checkout_ref(self.ref) - self.ref = self.get_commit_hash() - self.req.revision = self.ref + self.req.revision = vcsrepo.get_commit_hash() # Remove potential ref in the end of uri after ref is parsed if "@" in self.link.show_url and "@" in self.uri: @@ -1070,7 +1094,7 @@ def as_ireq(self): if self.editable or self.req.editable: if ireq_line.startswith("-e "): ireq_line = ireq_line[len("-e "):] - with ensure_setup_py(self.req.path): + with ensure_setup_py(self.req.setup_path): ireq = ireq_from_editable(ireq_line) else: ireq = ireq_from_line(ireq_line) diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index fb2e6bc30f..4efb9bd319 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,7 +1,6 @@ # -*- coding=utf-8 -*- import attr from pip_shims import VcsSupport, parse_version, pip_version -import vistir import os @@ -14,6 +13,7 @@ class VCSRepository(object): name = attr.ib() checkout_directory = attr.ib() vcs_type = attr.ib() + subdirectory = attr.ib(default=None) commit_sha = attr.ib(default=None) ref = attr.ib(default=None) repo_instance = attr.ib() @@ -31,35 +31,31 @@ def is_local(self): return url.startswith("file") def obtain(self): - if not os.path.exists(self.checkout_directory): + if (os.path.exists(self.checkout_directory) and not + self.repo_instance.is_repository_directory(self.checkout_directory)): + self.repo_instance.unpack(self.checkout_directory) + elif not os.path.exists(self.checkout_directory): self.repo_instance.obtain(self.checkout_directory) - if self.ref: - self.checkout_ref(self.ref) - self.commit_sha = self.get_commit_hash(self.ref) else: - if not self.commit_sha: - self.commit_sha = self.get_commit_hash() + if self.ref: + self.checkout_ref(self.ref) + if not self.commit_sha: + self.commit_sha = self.get_commit_hash() def checkout_ref(self, ref): if not self.repo_instance.is_commit_id_equal( - self.checkout_directory, self.get_commit_hash(ref) + self.checkout_directory, self.get_commit_hash() ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref): if not self.is_local: self.update(ref) def update(self, ref): target_ref = self.repo_instance.make_rev_options(ref) - sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) - target_rev = target_ref.make_new(sha) if parse_version(pip_version) > parse_version("18.0"): self.repo_instance.update(self.checkout_directory, self.url, target_ref) else: self.repo_instance.update(self.checkout_directory, target_ref) - self.commit_hash = self.get_commit_hash(ref) + self.commit_sha = self.get_commit_hash() def get_commit_hash(self, ref=None): - if ref: - target_ref = self.repo_instance.make_rev_options(ref) - return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) - # return self.repo_instance.get_revision(self.checkout_directory) return self.repo_instance.get_revision(self.checkout_directory) diff --git a/pipenv/vendor/resolvelib/LICENSE b/pipenv/vendor/resolvelib/LICENSE new file mode 100644 index 0000000000..b9077766e9 --- /dev/null +++ b/pipenv/vendor/resolvelib/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2018, Tzu-ping Chung + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 35a32eb6ce..18352c2ebe 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.20.0 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.1 +requirementslib==1.2.2 attrs==18.2.0 distlib==0.2.8 packaging==18.0 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index 912ab0a436..c8a995faa0 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -1,7 +1,12 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals -from .compat import NamedTemporaryFile, TemporaryDirectory, partialmethod, to_native_string +from .compat import ( + NamedTemporaryFile, + TemporaryDirectory, + partialmethod, + to_native_string, +) from .contextmanagers import ( atomic_open_for_write, cd, @@ -10,12 +15,23 @@ temp_path, spinner, ) -from .misc import load_path, partialclass, run, shell_escape +from .misc import ( + load_path, + partialclass, + run, + shell_escape, + decode_for_output, + to_text, + to_bytes, + take, + chunked, + divide, +) from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile from .spin import VistirSpinner, create_spinner -__version__ = '0.2.2' +__version__ = "0.2.3" __all__ = [ @@ -38,5 +54,11 @@ "create_spinner", "create_tracked_tempdir", "create_tracked_tempfile", - "to_native_string" + "to_native_string", + "decode_for_output", + "to_text", + "to_bytes", + "take", + "chunked", + "divide", ] diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 68e6d464df..ba00815996 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -8,6 +8,7 @@ import posixpath import shutil import stat +import sys import warnings import six @@ -166,11 +167,12 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ - from .misc import to_bytes + from .compat import to_native_string - fn = to_bytes(fn, encoding="utf-8") + fn = to_native_string(fn) if os.path.exists(fn): - return bool(os.stat(fn).st_mode & stat.S_IREAD) and not os.access(fn, os.W_OK) + file_stat = os.stat(fn).st_mode + return not bool(file_stat & stat.S_IWRITE) or not os.access(fn, os.W_OK) return False @@ -182,9 +184,10 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - from .misc import to_bytes, to_text + from .misc import to_text + from .compat import to_native_string - newdir = to_bytes(newdir, "utf-8") + newdir = to_native_string(newdir) if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( @@ -193,9 +196,9 @@ def mkdir_p(newdir, mode=0o777): ) ) else: - head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) + head, tail = os.path.split(newdir) # Make sure the tail doesn't point to the asame place as the head - curdir = to_bytes(".", encoding="utf-8") + curdir = to_native_string(".") tail_and_head_match = ( os.path.relpath(tail, start=os.path.basename(head)) == curdir ) @@ -242,7 +245,7 @@ def create_tracked_tempdir(*args, **kwargs): tempdir = TemporaryDirectory(*args, **kwargs) TRACKED_TEMPORARY_DIRECTORIES.append(tempdir) atexit.register(tempdir.cleanup) - warnings.simplefilter("default", ResourceWarning) + warnings.simplefilter("ignore", ResourceWarning) return tempdir.name @@ -266,12 +269,20 @@ def set_write_bit(fn): :param str fn: The target filename or path """ - from .misc import to_bytes, locale_encoding + from .compat import to_native_string - fn = to_bytes(fn, encoding=locale_encoding) + fn = to_native_string(fn) if not os.path.exists(fn): return - os.chmod(fn, stat.S_IWRITE | stat.S_IWUSR | stat.S_IRUSR) + file_stat = os.stat(fn).st_mode + os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + if not os.path.isdir(fn): + return + for root, dirs, files in os.walk(fn, topdown=False): + for dir_ in [os.path.join(root,d) for d in dirs]: + set_write_bit(dir_) + for file_ in [os.path.join(root, f) for f in files]: + set_write_bit(file_) def rmtree(directory, ignore_errors=False): @@ -288,14 +299,14 @@ def rmtree(directory, ignore_errors=False): Setting `ignore_errors=True` may cause this to silently fail to delete the path """ - from .misc import locale_encoding, to_bytes + from .compat import to_native_string - directory = to_bytes(directory, encoding=locale_encoding) + directory = to_native_string(directory) try: shutil.rmtree( directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly ) - except (IOError, OSError) as exc: + except (IOError, OSError, FileNotFoundError) as exc: # Ignore removal failures where the file doesn't exist if exc.errno == errno.ENOENT: pass @@ -316,23 +327,24 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ResourceWarning - from .misc import to_bytes + from .compat import ResourceWarning, FileNotFoundError, to_native_string - PERM_ERRORS = (errno.EACCES, errno.EPERM) + PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) default_warning_message = ( "Unable to remove file due to permissions restriction: {!r}" ) # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc - path = to_bytes(path, encoding="utf-8") + path = to_native_string(path) if is_readonly_path(path): # Apply write permission and call original function set_write_bit(path) try: func(path) - except (OSError, IOError) as e: - if e.errno in PERM_ERRORS: + except (OSError, IOError, FileNotFoundError) as e: + if e.errno == errno.ENOENT: + return + elif e.errno in PERM_ERRORS: warnings.warn(default_warning_message.format(path), ResourceWarning) return @@ -340,17 +352,20 @@ def handle_remove_readonly(func, path, exc): set_write_bit(path) try: func(path) - except (OSError, IOError) as e: + except (OSError, IOError, FileNotFoundError) as e: if e.errno in PERM_ERRORS: warnings.warn(default_warning_message.format(path), ResourceWarning) + pass elif e.errno == errno.ENOENT: # File already gone - return + pass else: raise - return else: - raise - raise exc + return + elif exc_exception.errno == errno.ENOENT: + pass + else: + raise exc_exception def walk_up(bottom): diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 6b6e498f3d..20587d9db7 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -48,7 +48,7 @@ def __exit__(self, exc_type, exc_val, traceback): if exc_type: import traceback from .misc import decode_for_output - self.write_err(decode_for_output(traceback.format_exception(traceback))) + self.write_err(decode_for_output(traceback.format_exception(*sys.exc_info()))) self._close_output_buffer() return False diff --git a/tasks/vendoring/patches/patched/pip18.patch b/tasks/vendoring/patches/patched/pip18.patch index 150ee32f55..f4e607c158 100644 --- a/tasks/vendoring/patches/patched/pip18.patch +++ b/tasks/vendoring/patches/patched/pip18.patch @@ -19,6 +19,73 @@ index 96f3b65c..cc5b3d15 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), +diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py +index edc506bf..84d57dac 100644 +--- a/pipenv/patched/pip/_internal/utils/temp_dir.py ++++ b/pipenv/patched/pip/_internal/utils/temp_dir.py +@@ -3,8 +3,10 @@ from __future__ import absolute_import + import logging + import os.path + import tempfile ++import warnings + + from pip._internal.utils.misc import rmtree ++from pipenv.vendor.vistir.compat import finalize, ResourceWarning + + logger = logging.getLogger(__name__) + +@@ -45,6 +47,20 @@ class TempDirectory(object): + self.path = path + self.delete = delete + self.kind = kind ++ self._finalizer = None ++ if path: ++ self._register_finalizer() ++ ++ def _register_finalizer(self): ++ if self.delete and self.path: ++ self._finalizer = finalize( ++ self, ++ self._cleanup, ++ self.path, ++ warn_message=None ++ ) ++ else: ++ self._finalizer = None + + def __repr__(self): + return "<{} {!r}>".format(self.__class__.__name__, self.path) +@@ -72,11 +88,27 @@ class TempDirectory(object): + self.path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) + ) ++ self._register_finalizer() + logger.debug("Created temporary directory: {}".format(self.path)) + ++ @classmethod ++ def _cleanup(cls, name, warn_message=None): ++ try: ++ rmtree(name) ++ except OSError: ++ pass ++ else: ++ if warn_message: ++ warnings.warn(warn_message, ResourceWarning) ++ + def cleanup(self): + """Remove the temporary directory created and reset state + """ +- if self.path is not None and os.path.exists(self.path): +- rmtree(self.path) +- self.path = None ++ if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): ++ if os.path.exists(self.path): ++ try: ++ rmtree(self.path) ++ except OSError: ++ pass ++ else: ++ self.path = None diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py index 8c2f24f1..cdd48874 100644 --- a/pipenv/patched/pip/_internal/index.py diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 1db5ef44bd..3799ccf49a 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -12,30 +12,56 @@ index 1fa3805..c0ecec8 100644 install_req_from_editable, ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py -index 28da51f..de9b435 100644 +index 28da51f..c466ef0 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py -@@ -1,12 +1,13 @@ +@@ -1,45 +1,55 @@ # -*- coding=utf-8 -*- - import importlib +-import importlib -import pip -+from pip_shims import pip_version - import pkg_resources +-import pkg_resources ++__all__ = [ ++ "InstallRequirement", ++ "parse_requirements", ++ "RequirementSet", ++ "user_cache_dir", ++ "FAVORITE_HASH", ++ "is_file_url", ++ "url_to_path", ++ "PackageFinder", ++ "FormatControl", ++ "Wheel", ++ "Command", ++ "cmdoptions", ++ "get_installed_distributions", ++ "PyPI", ++ "SafeFileCache", ++ "InstallationError", ++ "parse_version", ++ "pip_version", ++ "install_req_from_editable", ++ "install_req_from_line", ++ "user_cache_dir" ++] -def do_import(module_path, subimport=None, old_path=None): -+def do_import(module_path, subimport=None, old_path=None, vendored_name=None): - old_path = old_path or module_path +- old_path = old_path or module_path - prefixes = ["pip._internal", "pip"] -+ prefix = vendored_name if vendored_name else "pip" -+ prefixes = ["{0}._internal".format(prefix), "{0}".format(prefix)] - paths = [module_path, old_path] - search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] - package = subimport if subimport else None -@@ -21,25 +22,28 @@ def do_import(module_path, subimport=None, old_path=None): - return getattr(imported, package) - - +- paths = [module_path, old_path] +- search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] +- package = subimport if subimport else None +- for to_import in search_order: +- if not subimport: +- to_import, _, package = to_import.rpartition(".") +- try: +- imported = importlib.import_module(to_import) +- except ImportError: +- continue +- else: +- return getattr(imported, package) +- +- -InstallRequirement = do_import('req.req_install', 'InstallRequirement') -parse_requirements = do_import('req.req_file', 'parse_requirements') -RequirementSet = do_import('req.req_set', 'RequirementSet') @@ -50,34 +76,38 @@ index 28da51f..de9b435 100644 -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions') -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') -PyPI = do_import('models.index', 'PyPI') -+InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name="notpip") -+parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name="notpip") -+RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name="notpip") -+user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name="notpip") -+FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name="notpip") -+is_file_url = do_import('download', 'is_file_url', vendored_name="notpip") -+url_to_path = do_import('download', 'url_to_path', vendored_name="notpip") -+PackageFinder = do_import('index', 'PackageFinder', vendored_name="notpip") -+FormatControl = do_import('index', 'FormatControl', vendored_name="notpip") -+Wheel = do_import('wheel', 'Wheel', vendored_name="notpip") -+Command = do_import('cli.base_command', 'Command', old_path='basecommand', vendored_name="notpip") -+cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions', vendored_name="notpip") -+get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name="notpip") -+PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') -+SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') -+InstallationError = do_import('exceptions', 'InstallationError', vendored_name='notpip') ++from pipenv.vendor.appdirs import user_cache_dir ++from pip_shims.shims import ( ++ InstallRequirement, ++ parse_requirements, ++ RequirementSet, ++ FAVORITE_HASH, ++ is_file_url, ++ url_to_path, ++ PackageFinder, ++ FormatControl, ++ Wheel, ++ Command, ++ cmdoptions, ++ get_installed_distributions, ++ PyPI, ++ SafeFileCache, ++ InstallationError, ++ parse_version, ++ pip_version, ++) # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): -+if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('18.1'): ++if parse_version(pip_version) < parse_version('18.1'): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - install_req_from_line = do_import('req.constructors', 'install_req_from_line') - install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') -+ install_req_from_line = do_import('req.constructors', 'install_req_from_line', vendored_name="notpip") -+ install_req_from_editable = do_import('req.constructors', 'install_req_from_editable', vendored_name="notpip") -+ ++ from pip_shims.shims import ( ++ install_req_from_editable, install_req_from_line ++ ) diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 08dabe1..480ad1e 100644 --- a/pipenv/patched/piptools/repositories/local.py @@ -92,7 +122,7 @@ index 08dabe1..480ad1e 100644 else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index bf69803..a1a3906 100644 +index bf69803..31b85b9 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,7 +1,7 @@ @@ -104,7 +134,7 @@ index bf69803..a1a3906 100644 import hashlib import os from contextlib import contextmanager -@@ -15,13 +15,23 @@ from .._compat import ( +@@ -15,13 +15,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -113,11 +143,10 @@ index bf69803..a1a3906 100644 + InstallRequirement, + SafeFileCache ) -+os.environ["PIP_SHIMS_BASE_MODULE"] = str("notpip") ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") +from pip_shims.shims import do_import, VcsSupport, WheelCache +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet, Specifier -+from packaging.markers import Op, Value, Variable, Marker +InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) +from pip._internal.resolve import Resolver as PipResolver + @@ -128,11 +157,11 @@ index bf69803..a1a3906 100644 -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, -+ make_install_requirement, clean_requires_python) ++ make_install_requirement, clean_requires_python) from .base import BaseRepository try: -@@ -31,10 +41,44 @@ except ImportError: +@@ -31,10 +40,44 @@ except ImportError: def RequirementTracker(): yield @@ -181,7 +210,7 @@ index bf69803..a1a3906 100644 class PyPIRepository(BaseRepository): -@@ -46,8 +90,9 @@ class PyPIRepository(BaseRepository): +@@ -46,8 +89,9 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -192,7 +221,7 @@ index bf69803..a1a3906 100644 self.pip_options = pip_options index_urls = [pip_options.index_url] + pip_options.extra_index_urls -@@ -73,6 +118,10 @@ class PyPIRepository(BaseRepository): +@@ -73,6 +117,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -203,7 +232,7 @@ index bf69803..a1a3906 100644 # Setup file paths self.freshen_build_caches() -@@ -113,10 +162,13 @@ class PyPIRepository(BaseRepository): +@@ -113,10 +161,13 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -219,7 +248,7 @@ index bf69803..a1a3906 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,25 +178,86 @@ class PyPIRepository(BaseRepository): +@@ -126,25 +177,87 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( @@ -236,8 +265,7 @@ index bf69803..a1a3906 100644 + def gen(ireq): + if self.DEFAULT_INDEX_URL not in self.finder.index_urls: + return - -- def resolve_reqs(self, download_dir, ireq, wheel_cache): ++ + url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) + releases = self.session.get(url).json()['releases'] + @@ -266,7 +294,8 @@ index bf69803..a1a3906 100644 + try: + if ireq not in self._json_dep_cache: + self._json_dep_cache[ireq] = [g for g in gen(ireq)] -+ + +- def resolve_reqs(self, download_dir, ireq, wheel_cache): + return set(self._json_dep_cache[ireq]) + except Exception: + return set() @@ -291,6 +320,7 @@ index bf69803..a1a3906 100644 + dist = None + ireq.isolated = False + ireq._wheel_cache = wheel_cache ++ try: from pip._internal.operations.prepare import RequirementPreparer - from pip._internal.resolve import Resolver as PipResolver @@ -330,7 +360,7 @@ index bf69803..a1a3906 100644 } resolver = None preparer = None -@@ -177,15 +291,98 @@ class PyPIRepository(BaseRepository): +@@ -177,15 +291,109 @@ class PyPIRepository(BaseRepository): resolver_kwargs['preparer'] = preparer reqset = RequirementSet() ireq.is_direct = True @@ -339,9 +369,21 @@ index bf69803..a1a3906 100644 resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() +- reqset.cleanup_files() - return set(results) ++ cleanup_fn = getattr(reqset, "cleanup_files", None) ++ if cleanup_fn is not None: ++ try: ++ cleanup_fn() ++ except OSError: ++ pass ++ ++ if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): ++ if ireq.editable: ++ self._source_dir = TemporaryDirectory(fs_str("source")) ++ ireq.ensure_has_source_dir(self.source_dir) ++ + if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): + # Collect setup_requires info from local eggs. + # Do this after we call the preparer on these reqs to make sure their @@ -432,7 +474,7 @@ index bf69803..a1a3906 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -200,6 +397,7 @@ class PyPIRepository(BaseRepository): +@@ -200,6 +408,7 @@ class PyPIRepository(BaseRepository): # If a download_dir is passed, pip will unnecessarely # archive the entire source directory download_dir = None @@ -440,7 +482,7 @@ index bf69803..a1a3906 100644 elif ireq.link and not ireq.link.is_artifact: # No download_dir for VCS sources. This also works around pip # using git-checkout-index, which gets rid of the .git dir. -@@ -214,7 +412,8 @@ class PyPIRepository(BaseRepository): +@@ -214,7 +423,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: @@ -450,7 +492,7 @@ index bf69803..a1a3906 100644 finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: -@@ -236,6 +435,10 @@ class PyPIRepository(BaseRepository): +@@ -236,6 +446,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -461,7 +503,7 @@ index bf69803..a1a3906 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -243,24 +446,22 @@ class PyPIRepository(BaseRepository): +@@ -243,24 +457,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 5f5c193019..0ab0ab22ed 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -56,9 +56,6 @@ def check_github_ssh(): return res -WE_HAVE_INTERNET = check_internet() -WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() - TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') prepare_pypi_packages(PYPI_VENDOR_DIR) @@ -71,10 +68,13 @@ def pytest_runtest_setup(item): pytest.skip('requires github ssh') -@pytest.yield_fixture +@pytest.fixture def pathlib_tmpdir(request, tmpdir): yield Path(str(tmpdir)) - tmpdir.remove(ignore_errors=True) + try: + tmpdir.remove(ignore_errors=True) + except Exception: + pass # Borrowed from pip's test runner filesystem isolation @@ -102,6 +102,10 @@ def isolate(pathlib_tmpdir): os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) +WE_HAVE_INTERNET = check_internet() +WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() + + class _PipenvInstance(object): """An instance of a Pipenv Project...""" def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): @@ -207,14 +211,12 @@ def PipenvInstance(): yield _PipenvInstance -@pytest.fixture(scope='module') -def pip_src_dir(request): +@pytest.fixture(autouse=True) +def pip_src_dir(request, pathlib_tmpdir): old_src_dir = os.environ.get('PIP_SRC', '') - new_src_dir = TemporaryDirectory(prefix='pipenv-', suffix='-testsrc') - os.environ['PIP_SRC'] = fs_str(new_src_dir.name) + os.environ['PIP_SRC'] = pathlib_tmpdir.as_posix() def finalize(): - new_src_dir.cleanup() os.environ['PIP_SRC'] = fs_str(old_src_dir) request.addfinalizer(finalize) From 8459cd8d45de720e383faebb80de2f998bde0722 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 01:11:54 -0400 Subject: [PATCH 12/13] Fix requirementslib update Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index b3bc013239..4f39269676 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -597,7 +597,7 @@ def get_checkout_dir(self, src_dir=None): def get_vcs_repo(self, src_dir=None): from .vcs import VCSRepository checkout_dir = self.get_checkout_dir(src_dir=src_dir) - url = build_vcs_link( + link = build_vcs_link( self.vcs, self.uri, name=self.name, @@ -606,7 +606,7 @@ def get_vcs_repo(self, src_dir=None): extras=self.extras ) vcsrepo = VCSRepository( - url=url, + url=link.url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, From 2cbf0e050c008127d19e3e06ff5e6ffa499a2f66 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 01:58:15 -0400 Subject: [PATCH 13/13] Exclude setup_path from pipfile Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 4f39269676..08eb56183c 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -753,7 +753,7 @@ def _choose_vcs_source(pipfile): @property def pipfile_part(self): - excludes = ["_repo", "_base_line"] + excludes = ["_repo", "_base_line", "setup_path"] filter_func = lambda k, v: bool(v) is True and k.name not in excludes pipfile_dict = attr.asdict(self, filter=filter_func).copy() if "vcs" in pipfile_dict: