From 2180a3d4364e48e9bc526e7a1264c5ffc2bf296b Mon Sep 17 00:00:00 2001 From: Christopher Livingston Date: Wed, 18 Oct 2017 12:33:49 -0700 Subject: [PATCH] Fix setting of PEX_PATH in ./pants run (v2 backend) (#4969) Problem App servers that need to re-execute (apps watching for file systems changes, jupyter notebooks, etc.) throw an ImportError when being ran with ./pants run because the pants runner pex is losing context of where to find the requirements pex, the sources pex, and any other pexes to merge into the runtime environment. This context is encapsulated in a search path called PEX_PATH, which is being loaded into the PEX_PATH environment variable and is consequently scrubbed out of the environment upon re-exec by pex internal logic. Solution Completely remove any setting of PEX_PATH in the environment upon construction of the pants runner pex and instead pass this extra pex path information to the pex_info metadata that is used by the PEXBuilder object to construct the pants runner pex. Result Applications that need to re-execute will now be able to locate the sources module and entry point because the search path to resolve them (PEX_PATH) will be persisted in the pants runner pex's PEX-INFO metadata. --- 3rdparty/python/requirements.txt | 2 +- .../python/checks/tasks2/python_eval.py | 2 +- .../tasks2/python_execution_task_base.py | 39 ++++++++++--------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 89f14dcc661..7621b4f6821 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -13,7 +13,7 @@ packaging==16.8 pathspec==0.5.0 parameterized==0.6.1 pep8==1.6.2 -pex==1.2.11 +pex==1.2.13 psutil==4.3.0 pyflakes==1.1.0 Pygments==1.4 diff --git a/contrib/python/src/python/pants/contrib/python/checks/tasks2/python_eval.py b/contrib/python/src/python/pants/contrib/python/checks/tasks2/python_eval.py index e13521590d7..250bab343df 100644 --- a/contrib/python/src/python/pants/contrib/python/checks/tasks2/python_eval.py +++ b/contrib/python/src/python/pants/contrib/python/checks/tasks2/python_eval.py @@ -171,7 +171,7 @@ def _compile_target(self, vt): exec_pex = PEX(exec_pex_path, interpreter) extra_pex_paths = [pex.path() for pex in filter(None, [reqs_pex, srcs_pex])] - pex = WrappedPEX(exec_pex, extra_pex_paths, interpreter) + pex = WrappedPEX(exec_pex, interpreter, extra_pex_paths) with self.context.new_workunit(name='eval', labels=[WorkUnitLabel.COMPILER, WorkUnitLabel.RUN, diff --git a/src/python/pants/backend/python/tasks2/python_execution_task_base.py b/src/python/pants/backend/python/tasks2/python_execution_task_base.py index 36200ff5749..3b289a1b979 100644 --- a/src/python/pants/backend/python/tasks2/python_execution_task_base.py +++ b/src/python/pants/backend/python/tasks2/python_execution_task_base.py @@ -32,15 +32,15 @@ class WrappedPEX(object): _PEX_PATH_ENV_VAR_NAME = 'PEX_PATH' - def __init__(self, pex, extra_pex_paths, interpreter): + def __init__(self, pex, interpreter, extra_pex_paths=None): """ :param pex: The main pex we wrap. - :param extra_pex_paths: Other pexes, to "merge" in via the PEX_PATH mechanism. :param interpreter: The interpreter the main pex will run on. + :param extra_pex_paths: Other pexes, to "merge" in via the PEX_PATH mechanism. """ self._pex = pex - self._extra_pex_paths = extra_pex_paths self._interpreter = interpreter + self._extra_pex_paths = extra_pex_paths @property def interpreter(self): @@ -60,14 +60,21 @@ def cmdline(self, args=()): return cmdline def run(self, *args, **kwargs): - kwargs_copy = copy(kwargs) - env = copy(kwargs_copy.get('env')) if 'env' in kwargs_copy else {} - env[self._PEX_PATH_ENV_VAR_NAME] = self._pex_path() - kwargs_copy['env'] = env - return self._pex.run(*args, **kwargs_copy) + pex_path = self._pex_path() + if pex_path: + kwargs_copy = copy(kwargs) + env = copy(kwargs_copy.get('env')) if 'env' in kwargs_copy else {} + env[self._PEX_PATH_ENV_VAR_NAME] = self._pex_path() + kwargs_copy['env'] = env + return self._pex.run(*args, **kwargs_copy) + else: + return self._pex.run(*args, **kwargs) def _pex_path(self): - return ':'.join(self._extra_pex_paths) + if self._extra_pex_paths: + return ':'.join(self._extra_pex_paths) + else: + return None class PythonExecutionTaskBase(ResolveRequirementsTaskBase): @@ -108,7 +115,6 @@ def create_pex(self, pex_info=None): interpreter = self.context.products.get_data(PythonInterpreter) path = os.path.join(self.workdir, str(interpreter.identity), target_set_id) - extra_pex_paths_file_path = path + '.extra_pex_paths' extra_pex_paths = None # Note that we check for the existence of the directory, instead of for invalid_vts, @@ -130,16 +136,11 @@ def create_pex(self, pex_info=None): extra_pex_paths = [pex.path() for pex in pexes if pex] + if extra_pex_paths: + pex_info.merge_pex_path(':'.join(extra_pex_paths)) + with safe_concurrent_creation(path) as safe_path: builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info) builder.freeze() - with open(extra_pex_paths_file_path, 'w') as outfile: - for epp in extra_pex_paths: - outfile.write(epp) - outfile.write(b'\n') - - if extra_pex_paths is None: - with open(extra_pex_paths_file_path, 'r') as infile: - extra_pex_paths = [p.strip() for p in infile.readlines()] - return WrappedPEX(PEX(os.path.realpath(path), interpreter), extra_pex_paths, interpreter) + return WrappedPEX(PEX(os.path.realpath(path), interpreter), interpreter)