From 8ae2ef83743e16f2e57d83faf8e6bdbeb6f6747d Mon Sep 17 00:00:00 2001 From: Vincent Fazio Date: Wed, 28 Feb 2024 13:55:04 -0600 Subject: [PATCH] gh-115382: Fix cross compiles when host and target use same SOABI Previously, when a build was configured to use a host interpreter via --with-build-python, the PYTHON_FOR_BUILD config value included a path in PYTHONPATH that pointed to the target's built external modules. For "normal" foreign architecture cross compiles, when loading compiled external libraries, the target libraries were processed first due to their precedence in sys.path. These libraries were then ruled out due to a mismatch in the SOABI so the import mechanism continued searching until it found the host's native modules. However, if the host interpreter and the target python were on the same version + SOABI combination, the host interpreter would attempt to load the target's external modules due to their precedence in sys.path. Despite the "match", the target build may have been linked against a different libc or may include unsupported instructions so loading or executing the target's external modules can lead to crashes. Now, the path to the target's external modules is no longer defined in PYTHONPATH to prevent accidentally loading these foreign modules. One caveat is that during certain build stages, the target's sysconfig module requires higher precedence than the host's version in order to accurately query the target build's configuration. This worked previously due to the target's sysconfig data module having precedence over the host's (see above). In order to keep this desired behavior, a new environment variable, _PYTHON_SYSCONFIGDATA_PATH, has been defined so sysconfig can search this directory for the target's sysconfig data. Signed-off-by: Vincent Fazio --- Lib/sysconfig/__init__.py | 13 ++++++++++++- Lib/test/libregrtest/main.py | 1 + Lib/test/pythoninfo.py | 1 + configure | 2 +- configure.ac | 2 +- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 07ab27c7fb0c35b..3875c74fd4b90b6 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -337,7 +337,18 @@ def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" # _sysconfigdata is generated at build time, see _generate_posix_vars() name = _get_sysconfigdata_name() - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) + + # For cross builds, this value will be specified so sysconfigdata for the + # target can be imported. It cannot be in PYTHONPATH, as foreign modules in + # sys.path can cause crashes when loaded by the host interpreter. + if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')) is not None: + from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES + from importlib.util import module_from_spec + spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) + _temp = module_from_spec(spec) + spec.loader.exec_module(_temp) + else: + _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) build_time_vars = _temp.build_time_vars vars.update(build_time_vars) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 6fbe3d00981ddd2..dbbd961f6429a4d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -584,6 +584,7 @@ def _add_cross_compile_opts(self, regrtest_opts): '_PYTHON_PROJECT_BASE', '_PYTHON_HOST_PLATFORM', '_PYTHON_SYSCONFIGDATA_NAME', + "_PYTHON_SYSCONFIGDATA_PATH", 'PYTHONPATH' } old_environ = os.environ diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 814358746d6d8a4..5f98f495cb1ab77 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -326,6 +326,7 @@ def format_groups(groups): "_PYTHON_HOST_PLATFORM", "_PYTHON_PROJECT_BASE", "_PYTHON_SYSCONFIGDATA_NAME", + "_PYTHON_SYSCONFIGDATA_PATH", "__PYVENV_LAUNCHER__", # Sanitizer options diff --git a/configure b/configure index c0614138ead4b7a..f8f24af99f19e31 100755 --- a/configure +++ b/configure @@ -3689,7 +3689,7 @@ fi fi ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_build_python" >&5 printf "%s\n" "$with_build_python" >&6; } diff --git a/configure.ac b/configure.ac index e40b8e0d0de9fc9..3e25c0e37760b2d 100644 --- a/configure.ac +++ b/configure.ac @@ -164,7 +164,7 @@ AC_ARG_WITH([build-python], dnl Build Python interpreter is used for regeneration and freezing. ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python AC_MSG_RESULT([$with_build_python]) ], [ AS_VAR_IF([cross_compiling], [yes],