From 0a2fdd1c122f7c196ce367d84b1e46af22ae5839 Mon Sep 17 00:00:00 2001 From: Cyrille Rossant Date: Sat, 5 Feb 2022 16:09:35 +0100 Subject: [PATCH 1/4] Add --exclude option This allows excluding libraries from the resulting wheels, like OpenGL or Vulkan, which are provided by the OS. --- src/auditwheel/main_repair.py | 9 +++++++++ src/auditwheel/repair.py | 6 ++++++ tests/integration/test_bundled_wheels.py | 1 + 3 files changed, 16 insertions(+) diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index 1e0ae584..09d3bfaa 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -88,6 +88,14 @@ def configure_parser(sub_parsers): help="Strip symbols in the resulting wheel", default=False, ) + p.add_argument( + "--exclude", + dest="EXCLUDE", + help="Exclude SONAME from grafting into the resulting wheel " + "(can be specified multiple times)", + action="append", + default=[], + ) p.add_argument( "--only-plat", dest="ONLY_PLAT", @@ -169,6 +177,7 @@ def execute(args, p): out_dir=args.WHEEL_DIR, update_tags=args.UPDATE_TAGS, patcher=patcher, + exclude=args.EXCLUDE, strip=args.STRIP, ) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 4edb6250..0d873f16 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -37,6 +37,7 @@ def repair_wheel( out_dir: str, update_tags: bool, patcher: ElfPatcher, + exclude: List[str], strip: bool = False, ) -> Optional[str]: @@ -70,6 +71,11 @@ def repair_wheel( ext_libs = v[abis[0]]["libs"] # type: Dict[str, str] replacements = [] # type: List[Tuple[str, str]] for soname, src_path in ext_libs.items(): + + if soname in exclude: + logger.info(f"Excluding {soname}") + continue + if src_path is None: raise ValueError( ( diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index f3f5bc80..ab057c10 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -67,6 +67,7 @@ def test_wheel_source_date_epoch(tmp_path, monkeypatch): UPDATE_TAGS=True, WHEEL_DIR=str(wheel_output_path), WHEEL_FILE=[str(wheel_path)], + EXCLUDE=[], cmd="repair", func=Mock(), prog="auditwheel", From cb87609b7571abe99b144f00784e8804bfbabc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Kiss=20Koll=C3=A1r?= Date: Mon, 18 Jul 2022 14:31:19 +0100 Subject: [PATCH 2/4] test: Extract function to build numpy We have fairly complex code in a test to build the numpy wheels. We can extract this and reuse it in other tests in the future which need a cleanly build wheel before testing `auditwheel repair` functionality. --- tests/integration/test_manylinux.py | 76 ++++++++++++++++------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 2eb372d6..a91dfadf 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -200,6 +200,45 @@ def assert_show_output(manylinux_ctr, wheel, expected_tag, strict): assert actual_glibc <= expected_glibc +def build_numpy(container, policy, output_dir): + """Helper to build numpy from source using the specified container, into + output_dir.""" + + if policy.startswith("musllinux_"): + docker_exec(container, "apk add openblas-dev") + elif policy.startswith("manylinux_2_24_"): + docker_exec(container, "apt-get update") + docker_exec( + container, "apt-get install -y --no-install-recommends libatlas-dev" + ) + else: + docker_exec(container, "yum install -y atlas atlas-devel") + + if op.exists(op.join(WHEEL_CACHE_FOLDER, policy, ORIGINAL_NUMPY_WHEEL)): + # If numpy has already been built and put in cache, let's reuse this. + shutil.copy2( + op.join(WHEEL_CACHE_FOLDER, policy, ORIGINAL_NUMPY_WHEEL), + op.join(output_dir, ORIGINAL_NUMPY_WHEEL), + ) + else: + # otherwise build the original linux_x86_64 numpy wheel from source + # and put the result in the cache folder to speed-up future build. + # This part of the build is independent of the auditwheel code-base + # so it's safe to put it in cache. + + docker_exec( + container, + f"pip wheel -w /io --no-binary=:all: numpy=={NUMPY_VERSION}", + ) + os.makedirs(op.join(WHEEL_CACHE_FOLDER, policy), exist_ok=True) + shutil.copy2( + op.join(output_dir, ORIGINAL_NUMPY_WHEEL), + op.join(WHEEL_CACHE_FOLDER, policy, ORIGINAL_NUMPY_WHEEL), + ) + orig_wheel, *_ = os.listdir(output_dir) + return orig_wheel + + class Anylinux: @pytest.fixture() def io_folder(self, tmp_path): @@ -230,43 +269,12 @@ def test_build_repair_numpy( self, any_manylinux_container, docker_python, io_folder ): # Integration test: repair numpy built from scratch + policy, tag, manylinux_ctr = any_manylinux_container # First build numpy from source as a naive linux wheel that is tied # to system libraries (atlas, libgfortran...) - policy, tag, manylinux_ctr = any_manylinux_container - if policy.startswith("musllinux_"): - docker_exec(manylinux_ctr, "apk add openblas-dev") - elif policy.startswith("manylinux_2_24_"): - docker_exec(manylinux_ctr, "apt-get update") - docker_exec( - manylinux_ctr, "apt-get install -y --no-install-recommends libatlas-dev" - ) - else: - docker_exec(manylinux_ctr, "yum install -y atlas atlas-devel") - - if op.exists(op.join(WHEEL_CACHE_FOLDER, policy, ORIGINAL_NUMPY_WHEEL)): - # If numpy has already been built and put in cache, let's reuse this. - shutil.copy2( - op.join(WHEEL_CACHE_FOLDER, policy, ORIGINAL_NUMPY_WHEEL), - op.join(io_folder, ORIGINAL_NUMPY_WHEEL), - ) - else: - # otherwise build the original linux_x86_64 numpy wheel from source - # and put the result in the cache folder to speed-up future build. - # This part of the build is independent of the auditwheel code-base - # so it's safe to put it in cache. - docker_exec( - manylinux_ctr, - f"pip wheel -w /io --no-binary=:all: numpy=={NUMPY_VERSION}", - ) - os.makedirs(op.join(WHEEL_CACHE_FOLDER, policy), exist_ok=True) - shutil.copy2( - op.join(io_folder, ORIGINAL_NUMPY_WHEEL), - op.join(WHEEL_CACHE_FOLDER, policy, ORIGINAL_NUMPY_WHEEL), - ) - filenames = os.listdir(io_folder) - assert filenames == [ORIGINAL_NUMPY_WHEEL] - orig_wheel = filenames[0] + orig_wheel = build_numpy(manylinux_ctr, policy, io_folder) + assert orig_wheel == ORIGINAL_NUMPY_WHEEL assert "manylinux" not in orig_wheel # Repair the wheel using the manylinux container From b5f3907a463228d741b80f4ea1be5436e027bd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Kiss=20Koll=C3=A1r?= Date: Mon, 18 Jul 2022 15:01:47 +0100 Subject: [PATCH 3/4] Add test for `repair --exclude` functionality Build numpy from source and when repairing, exclude the gfortran library. This is only tested in a single manylinux image to avoid increasing the test time too much, as the functionality is not dependent on the platform. --- tests/integration/test_manylinux.py | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index a91dfadf..92ac5723 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -312,6 +312,65 @@ def test_build_repair_numpy( # at once in the same Python program: docker_exec(docker_python, ["python", "-c", "'import numpy; import foo'"]) + @pytest.mark.skipif( + PLATFORM != "x86_64", reason="Only needs checking on one platform" + ) + def test_repair_exclude(self, io_folder): + """Test the --exclude argument to avoid grafting certain libraries.""" + + policy = "manylinux_2_12" + policy_plat = f"{policy}_{PLATFORM}" + base = MANYLINUX_IMAGES[policy] + env = {"PATH": PATH[policy]} + + with tmp_docker_image( + base, + [ + 'git config --global --add safe.directory "/auditwheel_src"', + "pip install -U pip setuptools pytest-cov", + "pip install -U -e /auditwheel_src", + ], + env, + ) as tmp_img: + with docker_container_ctx(tmp_img, io_folder, env) as container: + platform_tag = ".".join( + [ + f"{p}_{PLATFORM}" + for p in [policy] + POLICY_ALIASES.get(policy, []) + ] + ) + + orig_wheel = build_numpy(container, policy_plat, io_folder) + assert orig_wheel == ORIGINAL_NUMPY_WHEEL + assert "manylinux" not in orig_wheel + + # Exclude libgfortran from grafting into the wheel + output = docker_exec( + container, + [ + "auditwheel", + "repair", + "--plat", + policy_plat, + "--exclude", + "libgfortran.so.5", + "--only-plat", + "-w", + "/io", + f"/io/{orig_wheel}", + ], + ) + + assert "Excluding libgfortran" in output + filenames = os.listdir(io_folder) + assert len(filenames) == 2 + repaired_wheel = f"numpy-{NUMPY_VERSION}-{PYTHON_ABI}-{platform_tag}.whl" + assert repaired_wheel in filenames + + # Make sure we don't have libatlas in the result + contents = zipfile.ZipFile(os.path.join(io_folder, repaired_wheel)).namelist() + assert not any(x for x in contents if x.startswith("libatlas")) + def test_build_wheel_with_binary_executable( self, any_manylinux_container, docker_python, io_folder ): From 83ae4123fa111651146c8602f630008810ad9a83 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 15 Oct 2022 14:47:47 +0200 Subject: [PATCH 4/4] fix: test coverage --- tests/integration/test_manylinux.py | 78 ++++++++++++----------------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 92ac5723..bf4645b7 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -315,61 +315,49 @@ def test_build_repair_numpy( @pytest.mark.skipif( PLATFORM != "x86_64", reason="Only needs checking on one platform" ) - def test_repair_exclude(self, io_folder): + def test_repair_exclude(self, any_manylinux_container, io_folder): """Test the --exclude argument to avoid grafting certain libraries.""" - policy = "manylinux_2_12" - policy_plat = f"{policy}_{PLATFORM}" - base = MANYLINUX_IMAGES[policy] - env = {"PATH": PATH[policy]} - - with tmp_docker_image( - base, - [ - 'git config --global --add safe.directory "/auditwheel_src"', - "pip install -U pip setuptools pytest-cov", - "pip install -U -e /auditwheel_src", - ], - env, - ) as tmp_img: - with docker_container_ctx(tmp_img, io_folder, env) as container: - platform_tag = ".".join( - [ - f"{p}_{PLATFORM}" - for p in [policy] + POLICY_ALIASES.get(policy, []) - ] - ) + policy, tag, manylinux_ctr = any_manylinux_container - orig_wheel = build_numpy(container, policy_plat, io_folder) - assert orig_wheel == ORIGINAL_NUMPY_WHEEL - assert "manylinux" not in orig_wheel + orig_wheel = build_numpy(manylinux_ctr, policy, io_folder) + assert orig_wheel == ORIGINAL_NUMPY_WHEEL + assert "manylinux" not in orig_wheel - # Exclude libgfortran from grafting into the wheel - output = docker_exec( - container, - [ - "auditwheel", - "repair", - "--plat", - policy_plat, - "--exclude", - "libgfortran.so.5", - "--only-plat", - "-w", - "/io", - f"/io/{orig_wheel}", - ], - ) + # Exclude libgfortran from grafting into the wheel + excludes = { + "manylinux_2_5_x86_64": ["libgfortran.so.1", "libgfortran.so.3"], + "manylinux_2_12_x86_64": ["libgfortran.so.3", "libgfortran.so.5"], + "manylinux_2_17_x86_64": ["libgfortran.so.3", "libgfortran.so.5"], + "manylinux_2_24_x86_64": ["libgfortran.so.3"], + "manylinux_2_28_x86_64": ["libgfortran.so.5"], + "musllinux_1_1_x86_64": ["libgfortran.so.5"], + }[policy] + + repair_command = [ + "auditwheel", + "repair", + "--plat", + policy, + "--only-plat", + "-w", + "/io", + ] + for exclude in excludes: + repair_command.extend(["--exclude", exclude]) + repair_command.append(f"/io/{orig_wheel}") + output = docker_exec(manylinux_ctr, repair_command) - assert "Excluding libgfortran" in output + for exclude in excludes: + assert f"Excluding {exclude}" in output filenames = os.listdir(io_folder) assert len(filenames) == 2 - repaired_wheel = f"numpy-{NUMPY_VERSION}-{PYTHON_ABI}-{platform_tag}.whl" + repaired_wheel = f"numpy-{NUMPY_VERSION}-{PYTHON_ABI}-{tag}.whl" assert repaired_wheel in filenames - # Make sure we don't have libatlas in the result + # Make sure we don't have libgfortran in the result contents = zipfile.ZipFile(os.path.join(io_folder, repaired_wheel)).namelist() - assert not any(x for x in contents if x.startswith("libatlas")) + assert not any(x for x in contents if "/libgfortran" in x) def test_build_wheel_with_binary_executable( self, any_manylinux_container, docker_python, io_folder