Switch Pex installed wheels to --prefix scheme.

Previously Pex used Pip's `--target` scheme which had both known bugs
(pypa/pip#7658) and unknown quirks that Pex
was failing to fully be able to work around. Switch to the `--prefix`
scheme which exactly mirrors the scheme venvs use so that venvs can be
created with content of all sorts placed where it belongs.

This removes fragile parsing and interpretation of the RECORD; now Pex
only creates a RECORD, which is much more straight forward, when
building a venv.

Fixes #1656

pex/common.py

@@ -47,6 +47,16 @@
 DETERMINISTIC_DATETIME_TIMESTAMP = (DETERMINISTIC_DATETIME - _UNIX_EPOCH).total_seconds()
 
 
+def find_site_packages(prefix_dir):
+    # type: (str) -> Optional[str]
+    """Return the absolute path to the site-packages directory of the given Python installation."""
+    for root, dirs, _ in os.walk(prefix_dir):
+        for d in dirs:
+            if "site-packages" == d:
+                return os.path.join(root, d)
+    return None
+
+
 def filter_pyc_dirs(dirs):
     # type: (Iterable[str]) -> Iterator[str]
     """Return an iterator over the input `dirs` filtering out Python bytecode cache directories."""

pex/finders.py

@@ -7,6 +7,7 @@
 import os
 
 from pex.common import is_python_script
+from pex.pep_376 import InstalledWheel
 from pex.third_party.pkg_resources import Distribution
 from pex.typing import TYPE_CHECKING, cast
 
@@ -27,7 +28,7 @@ def find(
         name,  # type: str
     ):
         # type: (...) -> Optional[DistributionScript]
-        script_path = os.path.join(dist.location, "bin", name)
+        script_path = InstalledWheel.load(dist.location).stashed_path("bin", name)
         return cls(dist=dist, path=script_path) if os.path.isfile(script_path) else None
 
     dist = attr.ib()  # type: Distribution

pex/interpreter.py

@@ -773,7 +773,7 @@ def create_interpreter(
             process = Executor.open_process(
                 cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd
             )
-            job = Job(command=cmd, process=process, finalizer=lambda: safe_rmtree(cwd))
+            job = Job(command=cmd, process=process, finalizer=lambda _: safe_rmtree(cwd))
             return SpawnedJob.file(job, output_file=cache_file, result_func=create_interpreter)
 
     @classmethod

pex/jobs.py

@@ -50,14 +50,15 @@ def __init__(
         self,
         command,  # type: Iterable[str]
         process,  # type: subprocess.Popen
-        finalizer=None,  # type: Optional[Callable[[], None]]
+        finalizer=None,  # type: Optional[Callable[[int], None]]
     ):
         # type: (...) -> None
         """
         :param command: The command used to spawn the job process.
         :param process: The spawned process handle.
-        :param finalizer: An optional cleanup function to call exactly once when the underlying
-                          process terminates in the course of calling this job's public methods.
+        :param finalizer: An optional cleanup function to call exactly once with the process return
+                          code when the underlying process terminates in the course of calling this
+                          job's public methods.
         """
         self._command = tuple(command)
         self._process = process
@@ -131,7 +132,7 @@ def create_error(
 
     def _finalize_job(self):
         if self._finalizer is not None:
-            self._finalizer()
+            self._finalizer(self._process.returncode)
             self._finalizer = None
 
     def _check_returncode(self, stderr=None):