⚡ Improve performance based on profiling (#107)

* ⚡ Improve performance based on profiling

* 👷 Fix unordered dict CI artifact

src/pipgrip/cli.py

@@ -324,7 +324,7 @@ def render_lock(packages, include_dot=True, sort=False):
     "--threads",
     type=click.INT,
     envvar="PIPGRIP_THREADS",
-    default=min(8, cpu_count()),
+    default=max(8, cpu_count() * 2),
     help="Maximum amount of threads to use for running concurrent pip subprocesses.",
 )
 @click.option(

src/pipgrip/libs/mixology/partial_solution.py

@@ -126,7 +126,7 @@ def backtrack(self, decision_level):  # type: (int) -> None
         while self._assignments[-1].decision_level > decision_level:
             removed = self._assignments.pop(-1)
             packages.add(removed.package)
-            if removed.is_decision():
+            if removed.is_decision() and removed.package in self._decisions:
                 del self._decisions[removed.package]
 
         # Re-compute _positive and _negative for the packages that were removed.
@@ -195,6 +195,8 @@ def satisfier(self, term):  # type: (Term) -> Assignment
                 assigned_term = assignment
             else:
                 assigned_term = assigned_term.intersect(assignment)
+                if assigned_term is None:
+                    continue
 
             # As soon as we have enough assignments to satisfy term, return them.
             if assigned_term.satisfies(term):

src/pipgrip/libs/mixology/range.py

@@ -305,7 +305,7 @@ def is_single_version(self):  # type: () -> bool
         )
 
     def is_vcs_version(self):  # type: () -> bool
-        return self.is_single_version() and self._min.is_vcs()
+        return bool(self.min) and self.min.is_vcs()
 
     def __eq__(self, other):
         if not isinstance(other, Range):

src/pipgrip/libs/mixology/version_solver.py

@@ -4,6 +4,7 @@
 from multiprocessing.pool import ThreadPool
 from typing import Dict, Hashable, List, Optional, Union
 
+from pipgrip.libs.mixology._compat import OrderedDict
 from pipgrip.libs.mixology.constraint import Constraint
 from pipgrip.libs.mixology.failure import SolverFailure
 from pipgrip.libs.mixology.incompatibility import Incompatibility
@@ -126,7 +127,9 @@ def _propagate(self, package):  # type: (Hashable) -> None
                     # decision level, so we clear [changed] and refill it with the
                     # newly-propagated assignment.
                     changed.clear()
-                    changed.add(str(self._propagate_incompatibility(root_cause)))
+                    prop = self._propagate_incompatibility(root_cause)
+                    if prop is not None:
+                        changed.add(str(prop))
                     break
                 elif result is not None:
                     changed.add(result)
@@ -338,8 +341,10 @@ def _get_min(term):
         if len(unsatisfied) == 1:
             term = unsatisfied[0]
         else:
-            self._threadpool.map(_get_min, unsatisfied)  # populate self._source
-            term = min(*unsatisfied, key=_get_min)
+            terms = OrderedDict(
+                zip(unsatisfied, self._threadpool.map(_get_min, unsatisfied))
+            )
+            term = min(terms, key=terms.get)
 
         return term
 

src/pipgrip/package_source.py

@@ -17,7 +17,7 @@
 
 
 def is_vcs_version(version):  # type: (str) -> bool
-    return str(Version.parse(version).major) not in version
+    return Version.parse(version).is_vcs()
 
 
 def render_pin(package, version):  # type: (str, str) -> str
@@ -119,17 +119,19 @@ def add(
         if extras not in self._packages[name]:
             self._packages[name][extras] = {}
 
-        # if already added without deps, assume now called with discovered deps and overwrite
-        if version in self._packages[name][extras] and not (
-            deps is None or self._packages[name][extras][version] is None
-        ):
-            raise ValueError("{} ({}) already exists".format(name, version))
+        if version in self._packages[name][extras]:
+            if self._packages[name][extras][version] is not None:
+                if deps is not None:
+                    raise ValueError("{} ({}) already exists".format(name, version))
+                # already discovered, now called with deps is None from discovering a different version
+                return
 
         # not existing and deps undiscovered
         if deps is None:
             self._packages[name][extras][version] = None
             return
 
+        # not existing and deps now discovered
         dependencies = []
         for dep in deps:
             req = parse_req(dep)

src/pipgrip/pipper.py

@@ -28,21 +28,28 @@ def read_requirements(path):
         raise RuntimeError("{} is broken".format(path))
 
 
+_parse_req_cache = {}
+
+
 def parse_req(requirement, extras=None):
+    extras = extras or set()
+    cache_key = (requirement, frozenset(extras))
+    if cache_key in _parse_req_cache:
+        return _parse_req_cache[cache_key]
     if requirement == "_root_" or requirement == "." or requirement.startswith(".["):
         req = pkg_resources.Requirement.parse(
             requirement.replace(".", "rubbish", 1)
             if requirement.startswith(".[")
             else "rubbish"
         )
-        if extras is not None:
+        if extras:
             req.extras = extras
         req.key = "." if requirement.startswith(".[") else requirement
         full_str = req.__str__().replace(req.name, req.key)
         req.name = req.key
     else:
         req = pkg_resources.Requirement.parse(requirement)
-        if extras is not None:
+        if extras:
             req.extras = extras
         req.key = canonicalize_name(req.key)
         req.name = req.key
@@ -56,6 +63,7 @@ def __str__():
         req.name + "[" + ",".join(sorted(req.extras)) + "]" if req.extras else req.name
     )
     req.extras = frozenset(req.extras)
+    _parse_req_cache[cache_key] = req
     return req