Fix setting of PEX_PATH in ./pants run (v2 backend) (#4969)

Problem

App servers that need to re-execute (apps watching for file systems changes, jupyter notebooks, etc.) throw an ImportError when being ran with ./pants run because the pants runner pex is losing context of where to find the requirements pex, the sources pex, and any other pexes to merge into the runtime environment. This context is encapsulated in a search path called PEX_PATH, which is being loaded into the PEX_PATH environment variable and is consequently scrubbed out of the environment upon re-exec by pex internal logic.

Solution

Completely remove any setting of PEX_PATH in the environment upon construction of the pants runner pex and instead pass this extra pex path information to the pex_info metadata that is used by the PEXBuilder object to construct the pants runner pex.

Result

Applications that need to re-execute will now be able to locate the sources module and entry point because the search path to resolve them (PEX_PATH) will be persisted in the pants runner pex's PEX-INFO metadata.

3rdparty/python/requirements.txt

@@ -13,7 +13,7 @@ packaging==16.8
 pathspec==0.5.0
 parameterized==0.6.1
 pep8==1.6.2
-pex==1.2.11
+pex==1.2.13
 psutil==4.3.0
 pyflakes==1.1.0
 Pygments==1.4

contrib/python/src/python/pants/contrib/python/checks/tasks2/python_eval.py

@@ -171,7 +171,7 @@ def _compile_target(self, vt):
 
       exec_pex = PEX(exec_pex_path, interpreter)
       extra_pex_paths = [pex.path() for pex in filter(None, [reqs_pex, srcs_pex])]
-      pex = WrappedPEX(exec_pex, extra_pex_paths, interpreter)
+      pex = WrappedPEX(exec_pex, interpreter, extra_pex_paths)
 
       with self.context.new_workunit(name='eval',
                                      labels=[WorkUnitLabel.COMPILER, WorkUnitLabel.RUN,

src/python/pants/backend/python/tasks2/python_execution_task_base.py

@@ -32,15 +32,15 @@ class WrappedPEX(object):
 
   _PEX_PATH_ENV_VAR_NAME = 'PEX_PATH'
 
-  def __init__(self, pex, extra_pex_paths, interpreter):
+  def __init__(self, pex, interpreter, extra_pex_paths=None):
     """
     :param pex: The main pex we wrap.
-    :param extra_pex_paths: Other pexes, to "merge" in via the PEX_PATH mechanism.
     :param interpreter: The interpreter the main pex will run on.
+    :param extra_pex_paths: Other pexes, to "merge" in via the PEX_PATH mechanism.
     """
     self._pex = pex
-    self._extra_pex_paths = extra_pex_paths
     self._interpreter = interpreter
+    self._extra_pex_paths = extra_pex_paths
 
   @property
   def interpreter(self):
@@ -60,14 +60,21 @@ def cmdline(self, args=()):
       return cmdline
 
   def run(self, *args, **kwargs):
-    kwargs_copy = copy(kwargs)
-    env = copy(kwargs_copy.get('env')) if 'env' in kwargs_copy else {}
-    env[self._PEX_PATH_ENV_VAR_NAME] = self._pex_path()
-    kwargs_copy['env'] = env
-    return self._pex.run(*args, **kwargs_copy)
+    pex_path = self._pex_path()
+    if pex_path:
+      kwargs_copy = copy(kwargs)
+      env = copy(kwargs_copy.get('env')) if 'env' in kwargs_copy else {}
+      env[self._PEX_PATH_ENV_VAR_NAME] = self._pex_path()
+      kwargs_copy['env'] = env
+      return self._pex.run(*args, **kwargs_copy)
+    else:
+      return self._pex.run(*args, **kwargs)
 
   def _pex_path(self):
-    return ':'.join(self._extra_pex_paths)
+    if self._extra_pex_paths:
+      return ':'.join(self._extra_pex_paths)
+    else:
+      return None
 
 
 class PythonExecutionTaskBase(ResolveRequirementsTaskBase):
@@ -108,7 +115,6 @@ def create_pex(self, pex_info=None):
 
       interpreter = self.context.products.get_data(PythonInterpreter)
       path = os.path.join(self.workdir, str(interpreter.identity), target_set_id)
-      extra_pex_paths_file_path = path + '.extra_pex_paths'
       extra_pex_paths = None
 
       # Note that we check for the existence of the directory, instead of for invalid_vts,
@@ -130,16 +136,11 @@ def create_pex(self, pex_info=None):
 
         extra_pex_paths = [pex.path() for pex in pexes if pex]
 
+        if extra_pex_paths:
+          pex_info.merge_pex_path(':'.join(extra_pex_paths))
+
         with safe_concurrent_creation(path) as safe_path:
           builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
           builder.freeze()
 
-        with open(extra_pex_paths_file_path, 'w') as outfile:
-          for epp in extra_pex_paths:
-            outfile.write(epp)
-            outfile.write(b'\n')
-
-    if extra_pex_paths is None:
-      with open(extra_pex_paths_file_path, 'r') as infile:
-        extra_pex_paths = [p.strip() for p in infile.readlines()]
-    return WrappedPEX(PEX(os.path.realpath(path), interpreter), extra_pex_paths, interpreter)
+    return WrappedPEX(PEX(os.path.realpath(path), interpreter), interpreter)