From 80dd6f7c2b3b16e638ab5d836758d5b60c8a82d5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 6 Aug 2022 12:07:38 -0400 Subject: [PATCH 01/14] Add test capturing failed expectation. --- distutils/tests/test_ccompiler.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 distutils/tests/test_ccompiler.py diff --git a/distutils/tests/test_ccompiler.py b/distutils/tests/test_ccompiler.py new file mode 100644 index 0000000000..9dfa918b37 --- /dev/null +++ b/distutils/tests/test_ccompiler.py @@ -0,0 +1,14 @@ + +from distutils import ccompiler + + +def test_set_include_dirs(tmp_path): + """ + Extensions should build even if set_include_dirs is invoked. + In particular, compiler-specific paths should not be overridden. + """ + c_file = tmp_path / 'foo.c' + c_file.write_text('void PyInit_foo(void) {}\n') + compiler = ccompiler.new_compiler() + compiler.set_include_dirs([]) + compiler.compile([c_file]) From dc1130766d356e1e9a613ba924e4af942631428c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 6 Aug 2022 12:17:13 -0400 Subject: [PATCH 02/14] Add compatibility for Python 3.7 --- distutils/tests/test_ccompiler.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/distutils/tests/test_ccompiler.py b/distutils/tests/test_ccompiler.py index 9dfa918b37..3dff273a21 100644 --- a/distutils/tests/test_ccompiler.py +++ b/distutils/tests/test_ccompiler.py @@ -1,7 +1,18 @@ +import os +import sys from distutils import ccompiler +def _make_strs(paths): + """ + Convert paths to strings for legacy compatibility. + """ + if sys.version_info > (3, 8): + return paths + return list(map(os.fspath, paths)) + + def test_set_include_dirs(tmp_path): """ Extensions should build even if set_include_dirs is invoked. @@ -11,4 +22,4 @@ def test_set_include_dirs(tmp_path): c_file.write_text('void PyInit_foo(void) {}\n') compiler = ccompiler.new_compiler() compiler.set_include_dirs([]) - compiler.compile([c_file]) + compiler.compile(_make_strs([c_file])) From 47c61e1097396a65ff74081930edd35eac90c10b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 6 Aug 2022 12:21:08 -0400 Subject: [PATCH 03/14] Windows is sensitive even on Python 3.10 --- distutils/tests/test_ccompiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distutils/tests/test_ccompiler.py b/distutils/tests/test_ccompiler.py index 3dff273a21..a7fc632432 100644 --- a/distutils/tests/test_ccompiler.py +++ b/distutils/tests/test_ccompiler.py @@ -1,5 +1,6 @@ import os import sys +import platform from distutils import ccompiler @@ -8,7 +9,7 @@ def _make_strs(paths): """ Convert paths to strings for legacy compatibility. """ - if sys.version_info > (3, 8): + if sys.version_info > (3, 8) and platform.system() != "Windows": return paths return list(map(os.fspath, paths)) From 561b70519ccc19a47de1e53f65e74287901083fc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 6 Aug 2022 12:21:50 -0400 Subject: [PATCH 04/14] Also test library dirs --- distutils/tests/test_ccompiler.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/distutils/tests/test_ccompiler.py b/distutils/tests/test_ccompiler.py index a7fc632432..0d3692b23e 100644 --- a/distutils/tests/test_ccompiler.py +++ b/distutils/tests/test_ccompiler.py @@ -24,3 +24,15 @@ def test_set_include_dirs(tmp_path): compiler = ccompiler.new_compiler() compiler.set_include_dirs([]) compiler.compile(_make_strs([c_file])) + + +def test_set_library_dirs(tmp_path): + """ + Extensions should build even if set_library_dirs is invoked. + In particular, compiler-specific paths should not be overridden. + """ + c_file = tmp_path / 'foo.c' + c_file.write_text('void PyInit_foo(void) {}\n') + compiler = ccompiler.new_compiler() + compiler.set_library_dirs([]) + compiler.compile(_make_strs([c_file])) From 1afdbe320d99ee4c3001ba7dcc834b101b7f9bef Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 6 Aug 2022 12:27:48 -0400 Subject: [PATCH 05/14] Extract fixture for c_file --- distutils/tests/test_ccompiler.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/distutils/tests/test_ccompiler.py b/distutils/tests/test_ccompiler.py index 0d3692b23e..8b60abf070 100644 --- a/distutils/tests/test_ccompiler.py +++ b/distutils/tests/test_ccompiler.py @@ -2,6 +2,8 @@ import sys import platform +import pytest + from distutils import ccompiler @@ -14,25 +16,28 @@ def _make_strs(paths): return list(map(os.fspath, paths)) -def test_set_include_dirs(tmp_path): +@pytest.fixture +def c_file(tmp_path): + c_file = tmp_path / 'foo.c' + c_file.write_text('void PyInit_foo(void) {}\n') + return c_file + + +def test_set_include_dirs(c_file): """ Extensions should build even if set_include_dirs is invoked. In particular, compiler-specific paths should not be overridden. """ - c_file = tmp_path / 'foo.c' - c_file.write_text('void PyInit_foo(void) {}\n') compiler = ccompiler.new_compiler() compiler.set_include_dirs([]) compiler.compile(_make_strs([c_file])) -def test_set_library_dirs(tmp_path): +def test_set_library_dirs(c_file): """ Extensions should build even if set_library_dirs is invoked. In particular, compiler-specific paths should not be overridden. """ - c_file = tmp_path / 'foo.c' - c_file.write_text('void PyInit_foo(void) {}\n') compiler = ccompiler.new_compiler() compiler.set_library_dirs([]) compiler.compile(_make_strs([c_file])) From 7849c89b8926e52157bc64a41ee3793804764e7f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 8 Aug 2022 16:03:04 -0400 Subject: [PATCH 06/14] Generate a C file that imports Python.h and something platform specific. --- distutils/tests/test_ccompiler.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/distutils/tests/test_ccompiler.py b/distutils/tests/test_ccompiler.py index 8b60abf070..c395f14d56 100644 --- a/distutils/tests/test_ccompiler.py +++ b/distutils/tests/test_ccompiler.py @@ -1,6 +1,7 @@ import os import sys import platform +import textwrap import pytest @@ -19,7 +20,17 @@ def _make_strs(paths): @pytest.fixture def c_file(tmp_path): c_file = tmp_path / 'foo.c' - c_file.write_text('void PyInit_foo(void) {}\n') + gen_headers = ('Python.h',) + is_windows = platform.system() == "Windows" + plat_headers = ('windows.h',) * is_windows + all_headers = gen_headers + plat_headers + headers = '\n'.join(f'#import <{header}>\n' for header in all_headers) + payload = textwrap.dedent( + """ + #headers + void PyInit_foo(void) {} + """).lstrip().replace('#headers', headers) + c_file.write_text(payload) return c_file From 530b119f4eda4b3b73eb17c4b58a0ffa8a5d6f8c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 8 Aug 2022 16:12:31 -0400 Subject: [PATCH 07/14] Ensure Python include directory is configured. --- distutils/tests/test_ccompiler.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/distutils/tests/test_ccompiler.py b/distutils/tests/test_ccompiler.py index c395f14d56..42a62c8661 100644 --- a/distutils/tests/test_ccompiler.py +++ b/distutils/tests/test_ccompiler.py @@ -2,6 +2,7 @@ import sys import platform import textwrap +import sysconfig import pytest @@ -24,12 +25,17 @@ def c_file(tmp_path): is_windows = platform.system() == "Windows" plat_headers = ('windows.h',) * is_windows all_headers = gen_headers + plat_headers - headers = '\n'.join(f'#import <{header}>\n' for header in all_headers) - payload = textwrap.dedent( - """ + headers = '\n'.join(f'#include <{header}>\n' for header in all_headers) + payload = ( + textwrap.dedent( + """ #headers void PyInit_foo(void) {} - """).lstrip().replace('#headers', headers) + """ + ) + .lstrip() + .replace('#headers', headers) + ) c_file.write_text(payload) return c_file @@ -40,15 +46,6 @@ def test_set_include_dirs(c_file): In particular, compiler-specific paths should not be overridden. """ compiler = ccompiler.new_compiler() - compiler.set_include_dirs([]) - compiler.compile(_make_strs([c_file])) - - -def test_set_library_dirs(c_file): - """ - Extensions should build even if set_library_dirs is invoked. - In particular, compiler-specific paths should not be overridden. - """ - compiler = ccompiler.new_compiler() - compiler.set_library_dirs([]) + python = sysconfig.get_paths()['include'] + compiler.set_include_dirs([python]) compiler.compile(_make_strs([c_file])) From 71cffcb8a8ec7e36dc389a5aa6dc2cc9769a9e97 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 9 Aug 2022 10:33:39 -0400 Subject: [PATCH 08/14] Extend the test to compile a second time after setting include dirs again. --- distutils/tests/test_ccompiler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/distutils/tests/test_ccompiler.py b/distutils/tests/test_ccompiler.py index 42a62c8661..da1879f237 100644 --- a/distutils/tests/test_ccompiler.py +++ b/distutils/tests/test_ccompiler.py @@ -49,3 +49,7 @@ def test_set_include_dirs(c_file): python = sysconfig.get_paths()['include'] compiler.set_include_dirs([python]) compiler.compile(_make_strs([c_file])) + + # do it again, setting include dirs after any initialization + compiler.set_include_dirs([python]) + compiler.compile(_make_strs([c_file])) From c84d3e59161496e19e1dab47abe5b9f8d4b6210b Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 17 Jun 2022 15:38:44 +0100 Subject: [PATCH 09/14] Ensure Windows SDK directories are not cleared when caller specifies include/library dirs --- distutils/_msvccompiler.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/distutils/_msvccompiler.py b/distutils/_msvccompiler.py index 516639aa1b..846e42f853 100644 --- a/distutils/_msvccompiler.py +++ b/distutils/_msvccompiler.py @@ -253,13 +253,17 @@ def initialize(self, plat_name=None): self.mc = _find_exe("mc.exe", paths) # message compiler self.mt = _find_exe("mt.exe", paths) # message compiler - for dir in vc_env.get('include', '').split(os.pathsep): - if dir: - self.add_include_dir(dir.rstrip(os.sep)) + self.__include_dirs = [ + dir.rstrip(os.sep) + for dir in vc_env.get('include', '').split(os.pathsep) + if dir + ] - for dir in vc_env.get('lib', '').split(os.pathsep): - if dir: - self.add_library_dir(dir.rstrip(os.sep)) + self.__library_dirs = [ + dir.rstrip(os.sep) + for dir in vc_env.get('lib', '').split(os.pathsep): + if dir + ] self.preprocess_options = None # bpo-38597: Always compile with dynamic linking @@ -557,6 +561,24 @@ def _fallback_spawn(self, cmd, env): with mock.patch.dict('os.environ', env): bag.value = super().spawn(cmd) + def _fix_compile_args(self, output_dir, macros, include_dirs): + """Corrects arguments to the compile() method and add compiler-specific dirs""" + fixed_args = super()._fix_lib_args(output_dir, macros, include_dirs) + return ( + fixed_args[0], # output_dir + fixed_args[1], # macros + fixed_args[2] + self.__include_dirs, + ) + + def _fix_lib_args(self, libraries, library_dirs, runtime_library_dirs): + """Corrects arguments to the link_*() methods and add linker-specific dirs""" + fixed_args = super()._fix_lib_args(libraries, library_dirs, runtime_library_dirs) + return ( + fixed_args[0], # libraries + fixed_args[1] + self.__library_dirs, + fixed_args[2], # runtime_library_dirs + ) + # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in # ccompiler.py. From d7b020b32349c3d93bb95502fa4f5c566fab2269 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 17 Jun 2022 16:06:39 +0100 Subject: [PATCH 10/14] Remove stray colon --- distutils/_msvccompiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distutils/_msvccompiler.py b/distutils/_msvccompiler.py index 846e42f853..c9f0bff07f 100644 --- a/distutils/_msvccompiler.py +++ b/distutils/_msvccompiler.py @@ -261,7 +261,7 @@ def initialize(self, plat_name=None): self.__library_dirs = [ dir.rstrip(os.sep) - for dir in vc_env.get('lib', '').split(os.pathsep): + for dir in vc_env.get('lib', '').split(os.pathsep) if dir ] From a223350c9af7f1aba69993020b126f6d0646d4f5 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 17 Jun 2022 16:11:24 +0100 Subject: [PATCH 11/14] Fixup bad super() call --- distutils/_msvccompiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distutils/_msvccompiler.py b/distutils/_msvccompiler.py index c9f0bff07f..0a19109fd7 100644 --- a/distutils/_msvccompiler.py +++ b/distutils/_msvccompiler.py @@ -563,7 +563,7 @@ def _fallback_spawn(self, cmd, env): def _fix_compile_args(self, output_dir, macros, include_dirs): """Corrects arguments to the compile() method and add compiler-specific dirs""" - fixed_args = super()._fix_lib_args(output_dir, macros, include_dirs) + fixed_args = super()._fix_compile_args(output_dir, macros, include_dirs) return ( fixed_args[0], # output_dir fixed_args[1], # macros From d1e3b46b380e77fa3cb70a42818a29578069ab40 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 2 Aug 2022 00:21:57 +0100 Subject: [PATCH 12/14] Use CCompiler._fix_compile_args to fix args to compile() --- distutils/ccompiler.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/distutils/ccompiler.py b/distutils/ccompiler.py index b4d3d0fbe0..47cd5ad40d 100644 --- a/distutils/ccompiler.py +++ b/distutils/ccompiler.py @@ -324,24 +324,7 @@ def set_link_objects(self, objects): def _setup_compile(self, outdir, macros, incdirs, sources, depends, extra): """Process arguments and decide which source files to compile.""" - if outdir is None: - outdir = self.output_dir - elif not isinstance(outdir, str): - raise TypeError("'output_dir' must be a string or None") - - if macros is None: - macros = self.macros - elif isinstance(macros, list): - macros = macros + (self.macros or []) - else: - raise TypeError("'macros' (if supplied) must be a list of tuples") - - if incdirs is None: - incdirs = self.include_dirs - elif isinstance(incdirs, (list, tuple)): - incdirs = list(incdirs) + (self.include_dirs or []) - else: - raise TypeError("'include_dirs' (if supplied) must be a list of strings") + outdir, macros, incdirs = self._fix_compile_args(outdir, macros, incdirs) if extra is None: extra = [] From 7d9c9d46565181770919ad77c133ab16a8721c59 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 9 Aug 2022 19:34:48 -0400 Subject: [PATCH 13/14] =?UTF-8?q?=E2=9A=AB=20Fade=20to=20black.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- distutils/_msvccompiler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/distutils/_msvccompiler.py b/distutils/_msvccompiler.py index 0a19109fd7..7abd24e371 100644 --- a/distutils/_msvccompiler.py +++ b/distutils/_msvccompiler.py @@ -260,9 +260,7 @@ def initialize(self, plat_name=None): ] self.__library_dirs = [ - dir.rstrip(os.sep) - for dir in vc_env.get('lib', '').split(os.pathsep) - if dir + dir.rstrip(os.sep) for dir in vc_env.get('lib', '').split(os.pathsep) if dir ] self.preprocess_options = None @@ -572,7 +570,9 @@ def _fix_compile_args(self, output_dir, macros, include_dirs): def _fix_lib_args(self, libraries, library_dirs, runtime_library_dirs): """Corrects arguments to the link_*() methods and add linker-specific dirs""" - fixed_args = super()._fix_lib_args(libraries, library_dirs, runtime_library_dirs) + fixed_args = super()._fix_lib_args( + libraries, library_dirs, runtime_library_dirs + ) return ( fixed_args[0], # libraries fixed_args[1] + self.__library_dirs, From 9f9a3e57643cb49796c1b08b5b5afb2826ecd7f6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 9 Aug 2022 21:02:16 -0400 Subject: [PATCH 14/14] Allow compiler classes to supply include and library dirs at the class level. --- distutils/_msvccompiler.py | 43 ++++++++++++-------------------------- distutils/ccompiler.py | 16 ++++++++++++++ 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/distutils/_msvccompiler.py b/distutils/_msvccompiler.py index 7abd24e371..ade80056e9 100644 --- a/distutils/_msvccompiler.py +++ b/distutils/_msvccompiler.py @@ -224,6 +224,18 @@ def __init__(self, verbose=0, dry_run=0, force=0): self.plat_name = None self.initialized = False + @classmethod + def _configure(cls, vc_env): + """ + Set class-level include/lib dirs. + """ + cls.include_dirs = cls._parse_path(vc_env.get('include', '')) + cls.library_dirs = cls._parse_path(vc_env.get('lib', '')) + + @staticmethod + def _parse_path(val): + return [dir.rstrip(os.sep) for dir in val.split(os.pathsep) if dir] + def initialize(self, plat_name=None): # multi-init means we would need to check platform same each time... assert not self.initialized, "don't init multiple times" @@ -243,6 +255,7 @@ def initialize(self, plat_name=None): raise DistutilsPlatformError( "Unable to find a compatible " "Visual Studio installation." ) + self._configure(vc_env) self._paths = vc_env.get('path', '') paths = self._paths.split(os.pathsep) @@ -253,16 +266,6 @@ def initialize(self, plat_name=None): self.mc = _find_exe("mc.exe", paths) # message compiler self.mt = _find_exe("mt.exe", paths) # message compiler - self.__include_dirs = [ - dir.rstrip(os.sep) - for dir in vc_env.get('include', '').split(os.pathsep) - if dir - ] - - self.__library_dirs = [ - dir.rstrip(os.sep) for dir in vc_env.get('lib', '').split(os.pathsep) if dir - ] - self.preprocess_options = None # bpo-38597: Always compile with dynamic linking # Future releases of Python 3.x will include all past @@ -559,26 +562,6 @@ def _fallback_spawn(self, cmd, env): with mock.patch.dict('os.environ', env): bag.value = super().spawn(cmd) - def _fix_compile_args(self, output_dir, macros, include_dirs): - """Corrects arguments to the compile() method and add compiler-specific dirs""" - fixed_args = super()._fix_compile_args(output_dir, macros, include_dirs) - return ( - fixed_args[0], # output_dir - fixed_args[1], # macros - fixed_args[2] + self.__include_dirs, - ) - - def _fix_lib_args(self, libraries, library_dirs, runtime_library_dirs): - """Corrects arguments to the link_*() methods and add linker-specific dirs""" - fixed_args = super()._fix_lib_args( - libraries, library_dirs, runtime_library_dirs - ) - return ( - fixed_args[0], # libraries - fixed_args[1] + self.__library_dirs, - fixed_args[2], # runtime_library_dirs - ) - # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in # ccompiler.py. diff --git a/distutils/ccompiler.py b/distutils/ccompiler.py index 47cd5ad40d..3cf5761cf2 100644 --- a/distutils/ccompiler.py +++ b/distutils/ccompiler.py @@ -91,6 +91,16 @@ class CCompiler: } language_order = ["c++", "objc", "c"] + include_dirs = [] + """ + include dirs specific to this compiler class + """ + + library_dirs = [] + """ + library dirs specific to this compiler class + """ + def __init__(self, verbose=0, dry_run=0, force=0): self.dry_run = dry_run self.force = force @@ -383,6 +393,9 @@ def _fix_compile_args(self, output_dir, macros, include_dirs): else: raise TypeError("'include_dirs' (if supplied) must be a list of strings") + # add include dirs for class + include_dirs += self.__class__.include_dirs + return output_dir, macros, include_dirs def _prep_compile(self, sources, output_dir, depends=None): @@ -439,6 +452,9 @@ def _fix_lib_args(self, libraries, library_dirs, runtime_library_dirs): else: raise TypeError("'library_dirs' (if supplied) must be a list of strings") + # add library dirs for class + library_dirs += self.__class__.library_dirs + if runtime_library_dirs is None: runtime_library_dirs = self.runtime_library_dirs elif isinstance(runtime_library_dirs, (list, tuple)):