Skip to content

Commit

Permalink
Ensure resolver doesn't compare editable specifiers
Browse files Browse the repository at this point in the history 
- Don't compare versions of editable dependencies when updating using
  `--keep-outdated` -- editable dependencies will now be updated to
  the latest version
- Ensure we don't drop markers from the lockfile when versions are not
  updated
- Fixes #3656
- Fixes #3659

Signed-off-by: Dan Ryan <[email protected]>
  • Loading branch information
@techalchemy
techalchemy committed Mar 31, 2019
1 parent 4db6d70 commit 72b9671
Show file tree
Hide file tree
Showing 8 changed files with 782 additions and 76 deletions.
2 changes: 2 additions & 0 deletions news/3656.bugfix.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The ``--keep-outdated`` argument to ``pipenv install`` and ``pipenv lock`` will now drop specifier constraints when encountering editable dependencies.
- In addition, ``--keep-outdated`` will retain specifiers that would otherwise be dropped from any entries that have not been updated.
128 changes: 106 additions & 22 deletions pipenv/resolver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,14 @@ def __init__(self, name, entry_dict, project, resolver, reverse_deps=None, dev=F
self.pipfile = project.parsed_pipfile.get(pipfile_section, {})
self.lockfile = project.lockfile_content.get(section, {})
self.pipfile_dict = self.pipfile.get(self.pipfile_name, {})
self.lockfile_dict = self.lockfile.get(name, entry_dict)
if self.dev and self.name in project.lockfile_content.get("default", {}):
self.lockfile_dict = project.lockfile_content["default"][name]
else:
self.lockfile_dict = self.lockfile.get(name, entry_dict)
self.resolver = resolver
self.reverse_deps = reverse_deps
self._original_markers = None
self._markers = None
self._entry = None
self._lockfile_entry = None
self._pipfile_entry = None
Expand All @@ -133,13 +138,79 @@ def clean_initial_dict(cls, entry_dict):
del entry_dict["name"]
return entry_dict

@classmethod
def get_markers_from_dict(cls, entry_dict):
from pipenv.vendor.packaging import markers as packaging_markers
from pipenv.vendor.requirementslib.models.markers import normalize_marker_str
marker_keys = list(packaging_markers.VARIABLE.exprs)
markers = set()
keys_in_dict = [k for k in marker_keys if k in entry_dict]
markers = {
normalize_marker_str("{k} {v}".format(k=k, v=entry_dict.pop(k)))
for k in keys_in_dict
}
if "markers" in entry_dict:
markers.add(normalize_marker_str(entry_dict["markers"]))
if markers:
entry_dict["markers"] = " and ".join(list(markers))
return markers, entry_dict

@property
def markers(self):
self._markers, self.entry_dict = self.get_markers_from_dict(self.entry_dict)
return self._markers

@markers.setter
def markers(self, markers):
if not markers:
pass
marker_str = self.marker_to_str(markers)
self._entry = self.entry.merge_markers(self.marker_to_str(markers))
self._markers = self.marker_to_str(self._entry.markers)
entry_dict = self.entry_dict.copy()
entry_dict["markers"] = self.marker_to_str(self._entry.markers)
self.entry_dict = entry_dict

@property
def original_markers(self):
original_markers, lockfile_dict = self.get_markers_from_dict(
self.lockfile_dict
)
self.lockfile_dict = lockfile_dict
self._original_markers = self.marker_to_str(original_markers)
return self._original_markers

@staticmethod
def marker_to_str(marker):
from pipenv.vendor.requirementslib.models.markers import normalize_marker_str
if not marker:
return None
from pipenv.vendor import six
from pipenv.vendor.vistir.compat import Mapping
marker_str = None
if isinstance(marker, Mapping):
marker_dict, _ = Entry.get_markers_from_dict(marker)
if marker_dict:
marker_str = "{0}".format(marker_dict.popitem()[1])
elif isinstance(marker, (list, set, tuple)):
marker_str = " and ".join([normalize_marker_str(m) for m in marker if m])
elif isinstance(marker, six.string_types):
marker_str = "{0}".format(normalize_marker_str(marker))
if isinstance(marker_str, six.string_types):
return marker_str
return None

def get_cleaned_dict(self):
if self.is_updated:
self.validate_constraints()
self.ensure_least_updates_possible()
if self.entry.extras != self.lockfile_entry.extras:
self._entry.req.extras.extend(self.lockfile_entry.req.extras)
self.entry_dict["extras"] = self.entry.extras
if self.original_markers and not self.markers:
original_markers = self.marker_to_str(self.original_markers)
self.markers = original_markers
self.entry_dict["markers"] = self.marker_to_str(original_markers)
entry_hashes = set(self.entry.hashes)
locked_hashes = set(self.lockfile_entry.hashes)
if entry_hashes != locked_hashes and not self.is_updated:
Expand All @@ -154,6 +225,10 @@ def lockfile_entry(self):
self._lockfile_entry = self.make_requirement(self.name, self.lockfile_dict)
return self._lockfile_entry

@lockfile_entry.setter
def lockfile_entry(self, entry):
self._lockfile_entry = entry

@property
def pipfile_entry(self):
if self._pipfile_entry is None:
Expand Down Expand Up @@ -265,6 +340,7 @@ def updated_version(self):

@property
def updated_specifier(self):
# type: () -> str
return self.entry.specifiers

@property
Expand All @@ -279,7 +355,7 @@ def original_version(self):
return None

def validate_specifiers(self):
if self.is_in_pipfile:
if self.is_in_pipfile and not self.pipfile_entry.editable:
return self.pipfile_entry.requirement.specifier.contains(self.updated_version)
return True

Expand Down Expand Up @@ -373,7 +449,7 @@ def get_constraints(self):
if c and c.name == self.entry.name
}
pipfile_constraint = self.get_pipfile_constraint()
if pipfile_constraint:
if pipfile_constraint and not self.pipfile_entry.editable:
constraints.add(pipfile_constraint)
return constraints

Expand Down Expand Up @@ -446,8 +522,11 @@ def validate_constraints(self):
constraint.check_if_exists(False)
except Exception:
from pipenv.exceptions import DependencyConflict
from pipenv.environments import is_verbose
if is_verbose():
print("Tried constraint: {0!r}".format(constraint), file=sys.stderr)
msg = (
"Cannot resolve conflicting version {0}{1} while {1}{2} is "
"Cannot resolve conflicting version {0}{1} while {2}{3} is "
"locked.".format(
self.name, self.updated_specifier, self.old_name, self.old_specifiers
)
Expand Down Expand Up @@ -502,6 +581,7 @@ def __getattribute__(self, key):

def clean_outdated(results, resolver, project, dev=False):
from pipenv.vendor.requirementslib.models.requirements import Requirement
from pipenv.environments import is_verbose
if not project.lockfile_exists:
return results
lockfile = project.lockfile_content
Expand All @@ -520,23 +600,28 @@ def clean_outdated(results, resolver, project, dev=False):
# TODO: Should this be the case for all locking?
if entry.was_editable and not entry.is_editable:
continue
# if the entry has not changed versions since the previous lock,
# don't introduce new markers since that is more restrictive
if entry.has_markers and not entry.had_markers and not entry.is_updated:
del entry.entry_dict["markers"]
entry._entry.req.req.marker = None
entry._entry.markers = ""
# do make sure we retain the original markers for entries that are not changed
elif entry.had_markers and not entry.has_markers and not entry.is_updated:
if entry._entry and entry._entry.req and entry._entry.req.req and (
entry.lockfile_entry and entry.lockfile_entry.req and
entry.lockfile_entry.req.req and entry.lockfile_entry.req.req.marker
):
entry._entry.req.req.marker = entry.lockfile_entry.req.req.marker
if entry.lockfile_entry and entry.lockfile_entry.markers:
entry._entry.markers = entry.lockfile_entry.markers
if entry.lockfile_dict and "markers" in entry.lockfile_dict:
entry.entry_dict["markers"] = entry.lockfile_dict["markers"]
lockfile_entry = lockfile[section].get(name, None)
if not lockfile_entry:
alternate_section = "develop" if not dev else "default"
if name in lockfile[alternate_section]:
lockfile_entry = lockfile[alternate_section][name]
if lockfile_entry and not entry.is_updated:
old_markers = next(iter(m for m in (
entry.lockfile_entry.markers, lockfile_entry.get("markers", None)
) if m is not None), None)
new_markers = entry_dict.get("markers", None)
if old_markers:
old_markers = Entry.marker_to_str(old_markers)
if old_markers and not new_markers:
entry.markers = old_markers
elif new_markers and not old_markers:
del entry.entry_dict["markers"]
entry._entry.req.req.marker = None
entry._entry.markers = ""
# if the entry has not changed versions since the previous lock,
# don't introduce new markers since that is more restrictive
# if entry.has_markers and not entry.had_markers and not entry.is_updated:
# do make sure we retain the original markers for entries that are not changed
entry_dict = entry.get_cleaned_dict()
new_results.append(entry_dict)
return new_results
Expand All @@ -557,7 +642,6 @@ def parse_packages(packages, pre, clear, system, requirements_dir=None):
sys.path.insert(0, req.req.setup_info.base_dir)
req.req._setup_info.get_info()
req.update_name_from_path(req.req.setup_info.base_dir)
print(os.listdir(req.req.setup_info.base_dir))
try:
name, entry = req.pipfile_entry
except Exception:
Expand Down
29 changes: 15 additions & 14 deletions pipenv/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -858,8 +858,8 @@ def resolve(cmd, sp):
_out = decode_output("{0}".format(_out))
out += _out
sp.text = to_native_string("{0}".format(_out[:100]))
if environments.is_verbose():
sp.hide_and_write(_out.rstrip())
if environments.is_verbose():
sp.hide_and_write(_out.rstrip())
if result is None:
break
c.block()
Expand Down Expand Up @@ -1609,27 +1609,28 @@ def translate_markers(pipfile_entry):
raise TypeError("Entry is not a pipfile formatted mapping.")
from .vendor.distlib.markers import DEFAULT_CONTEXT as marker_context
from .vendor.packaging.markers import Marker
from .vendor.requirementslib.models.markers import normalize_marker_str
from .vendor.vistir.misc import dedup

allowed_marker_keys = ["markers"] + [k for k in marker_context.keys()]
provided_keys = list(pipfile_entry.keys()) if hasattr(pipfile_entry, "keys") else []
pipfile_markers = [k for k in provided_keys if k in allowed_marker_keys]
new_pipfile = dict(pipfile_entry).copy()
marker_set = set()
marker_list = []
if "markers" in new_pipfile:
marker = str(Marker(new_pipfile.pop("markers")))
marker = new_pipfile.pop("markers")
if 'extra' not in marker:
marker_set.add(marker)
marker_list.append(normalize_marker_str(str(Marker(marker))))
for m in pipfile_markers:
entry = "{0}".format(pipfile_entry[m])
if m != "markers":
marker_set.add(str(Marker("{0}{1}".format(m, entry))))
new_pipfile.pop(m)
if marker_set:
new_pipfile["markers"] = str(Marker(" or ".join(
"{0}".format(s) if " and " in s else s
for s in sorted(dedup(marker_set))
))).replace('"', "'")
entry = "{0}".format(pipfile_entry.pop(m, None))
if m != "markers" and entry:
marker_list.append(normalize_marker_str(str(Marker(
"{0} {1}".format(m, entry)
))))
markers_to_add = " and ".join(dedup([m for m in marker_list if m]))
if markers_to_add:
markers_to_add = normalize_marker_str(str(Marker(markers_to_add)))
new_pipfile["markers"] = str(Marker(markers_to_add)).replace('"', "'")
return new_pipfile


Expand Down
Loading

0 comments on commit 72b9671

Please sign in to comment.