pythonGH-127178: improve compatibility in _sysconfig_vars_(...).json

This patch improves environment and platform compatibility
- Data now matches `sysconfig.get_config_vars` after install
- `userbase` now correctly reflects the target platform when cross-compiling
- `test_sysconfigdata_json` now takes into account the following situations:
  - Running with a non-default sys.executable path
  - Running under virtual environments
  - Running under relocatable installs

To simplify the detection of relocatable installs, which needs to look
at `_sysconfigdata_(...)`, this module is now saved in `sys.modules`.
As part of this change, the code to import the module from a specific
directory was refactored to use `PathFinder`, simplifying the
implementation.

Signed-off-by: Filipe Laíns <[email protected]

Lib/sysconfig/__init__.py

@@ -116,8 +116,10 @@ def _getuserbase():
     if env_base:
         return env_base
 
-    # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories
-    if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}:
+    # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories.
+    # Use sysconfig.get_platform() to get the correct platform when cross-compiling.
+    system_name = get_platform().split('-')[0]
+    if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}:
         return None
 
     def joinuser(*args):
@@ -335,34 +337,53 @@ def get_makefile_filename():
     return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile')
 
 
+def _import_from_directory(path, name):
+    if name not in sys.modules:
+        import importlib.machinery
+        import importlib.util
+
+        spec = importlib.machinery.PathFinder.find_spec(name, [path])
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        sys.modules[name] = module
+    return sys.modules[name]
+
+
 def _get_sysconfigdata_name():
     multiarch = getattr(sys.implementation, '_multiarch', '')
     return os.environ.get(
         '_PYTHON_SYSCONFIGDATA_NAME',
         f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}',
     )
 
-def _init_posix(vars):
-    """Initialize the module as appropriate for POSIX systems."""
-    # _sysconfigdata is generated at build time, see _generate_posix_vars()
+
+def _get_sysconfigdata():
+    import importlib
+
     name = _get_sysconfigdata_name()
+    path = os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')
+    module = _import_from_directory(path, name) if path else importlib.import_module(name)
 
-    # For cross builds, the path to the target's sysconfigdata must be specified
-    # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in
-    # sys.path can cause crashes when loaded by the host interpreter.
-    # Rely on truthiness as a valueless env variable is still an empty string.
-    # See OS X note in _generate_posix_vars re _sysconfigdata.
-    if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')):
-        from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES
-        from importlib.util import module_from_spec
-        spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name)
-        _temp = module_from_spec(spec)
-        spec.loader.exec_module(_temp)
-    else:
-        _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0)
-    build_time_vars = _temp.build_time_vars
+    return module.build_time_vars
+
+
+def _installation_is_relocated():
+    """Is the Python installation running from a different prefix than what was targetted when building?"""
+    if os.name != 'posix':
+        raise NotImplementedError('sysconfig._installation_is_relocated() is currently only supported on POSIX')
+
+    data = _get_sysconfigdata()
+    return (
+        data['prefix'] != getattr(sys, 'base_prefix', '')
+        or data['exec_prefix'] != getattr(sys, 'base_exec_prefix', '')
+    )
+
+
+def _init_posix(vars):
+    """Initialize the module as appropriate for POSIX systems."""
     # GH-126920: Make sure we don't overwrite any of the keys already set
-    vars.update(build_time_vars | vars)
+    vars.update(_get_sysconfigdata() | vars)
+
 
 def _init_non_posix(vars):
     """Initialize the module as appropriate for NT"""

Lib/sysconfig/__main__.py

@@ -232,10 +232,14 @@ def _generate_posix_vars():
 
     print(f'Written {destfile}')
 
+    install_vars = get_config_vars()
+    # Fix config vars to match the values after install (of the default environment)
+    install_vars['projectbase'] = install_vars['BINDIR']
+    install_vars['srcdir'] = install_vars['LIBPL']
     # Write a JSON file with the output of sysconfig.get_config_vars
     jsonfile = os.path.join(pybuilddir, _get_json_data_name())
     with open(jsonfile, 'w') as f:
-        json.dump(get_config_vars(), f, indent=2)
+        json.dump(install_vars, f, indent=2)
 
     print(f'Written {jsonfile}')
 

Lib/test/test_sysconfig.py

@@ -650,8 +650,18 @@ def test_sysconfigdata_json(self):
 
         system_config_vars = get_config_vars()
 
-        # Ignore keys in the check
-        for key in ('projectbase', 'srcdir'):
+        ignore_keys = {}
+        # Keys dependent on the executable location
+        if os.path.dirname(sys.executable) != system_config_vars['BINDIR']:
+            ignore_keys += {'projectbase'}
+        # Keys dependent on the environment (different inside virtual environments)
+        if sys.prefix != sys.base_prefix:
+            ignore_keys += {'prefix', 'exec_prefix', 'base', 'platbase'}
+        # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs)
+        if sysconfig._installation_is_relocated():
+            ignore_keys += {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'}
+
+        for key in ignore_keys:
             json_config_vars.pop(key)
             system_config_vars.pop(key)