From 45fc85ecafc3032f2001ac639744262a9797479d Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 18 Jul 2021 18:29:11 +0200 Subject: [PATCH] feat: add musllinux support (PEP 656) Add musllinux support (PEP 656) allowing to build wheels for musl libc based distros (e.g. Alpine) --- bin/update_docker.py | 13 ++++-- cibuildwheel/__main__.py | 31 ++++++++++++- cibuildwheel/linux.py | 6 +++ cibuildwheel/logger.py | 5 +++ cibuildwheel/resources/build-platforms.toml | 25 +++++++++++ cibuildwheel/resources/defaults.toml | 6 +++ .../resources/pinned_docker_images.cfg | 45 ++++++++++--------- cibuildwheel/util.py | 1 + test/test_0_basic.py | 4 +- test/test_docker_images.py | 4 +- test/test_manylinuxXXXX_only.py | 6 ++- test/utils.py | 14 ++++++ 12 files changed, 131 insertions(+), 29 deletions(-) diff --git a/bin/update_docker.py b/bin/update_docker.py index 2e97a8e6f..73da5a1ba 100755 --- a/bin/update_docker.py +++ b/bin/update_docker.py @@ -19,14 +19,15 @@ class Image(NamedTuple): images = [ + # manylinux1 images Image("manylinux1", "x86_64", "quay.io/pypa/manylinux1_x86_64", None), Image("manylinux1", "i686", "quay.io/pypa/manylinux1_i686", None), - # 2010 images + # manylinux2010 images Image("manylinux2010", "x86_64", "quay.io/pypa/manylinux2010_x86_64", None), Image("manylinux2010", "i686", "quay.io/pypa/manylinux2010_i686", None), Image("manylinux2010", "pypy_x86_64", "quay.io/pypa/manylinux2010_x86_64", None), Image("manylinux2010", "pypy_i686", "quay.io/pypa/manylinux2010_i686", None), - # 2014 images + # manylinux2014 images Image("manylinux2014", "x86_64", "quay.io/pypa/manylinux2014_x86_64", None), Image("manylinux2014", "i686", "quay.io/pypa/manylinux2014_i686", None), Image("manylinux2014", "aarch64", "quay.io/pypa/manylinux2014_aarch64", None), @@ -35,7 +36,7 @@ class Image(NamedTuple): Image("manylinux2014", "pypy_x86_64", "quay.io/pypa/manylinux2014_x86_64", None), Image("manylinux2014", "pypy_i686", "quay.io/pypa/manylinux2014_i686", None), Image("manylinux2014", "pypy_aarch64", "quay.io/pypa/manylinux2014_aarch64", None), - # 2_24 images + # manylinux_2_24 images Image("manylinux_2_24", "x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None), Image("manylinux_2_24", "i686", "quay.io/pypa/manylinux_2_24_i686", None), Image("manylinux_2_24", "aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None), @@ -44,6 +45,12 @@ class Image(NamedTuple): Image("manylinux_2_24", "pypy_x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None), Image("manylinux_2_24", "pypy_i686", "quay.io/pypa/manylinux_2_24_i686", None), Image("manylinux_2_24", "pypy_aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None), + # musllinux_1_1 images + Image("musllinux_1_1", "x86_64", "quay.io/pypa/musllinux_1_1_x86_64", None), + Image("musllinux_1_1", "i686", "quay.io/pypa/musllinux_1_1_i686", None), + Image("musllinux_1_1", "aarch64", "quay.io/pypa/musllinux_1_1_aarch64", None), + Image("musllinux_1_1", "ppc64le", "quay.io/pypa/musllinux_1_1_ppc64le", None), + Image("musllinux_1_1", "s390x", "quay.io/pypa/musllinux_1_1_s390x", None), ] config = configparser.ConfigParser() diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 9d1421ed9..22d3db5ab 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -41,6 +41,14 @@ "pypy_i686", ) +MUSLLINUX_ARCHS = ( + "x86_64", + "i686", + "aarch64", + "ppc64le", + "s390x", +) + def main() -> None: platform: PlatformName @@ -167,10 +175,13 @@ def main() -> None: manylinux_identifiers = { f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS } + musllinux_identifiers = { + f"musllinux-{build_platform}-image" for build_platform in MUSLLINUX_ARCHS + } disallow = { "linux": {"dependency-versions"}, - "macos": manylinux_identifiers, - "windows": manylinux_identifiers, + "macos": manylinux_identifiers | musllinux_identifiers, + "windows": manylinux_identifiers | musllinux_identifiers, } options = ConfigOptions(package_dir, args.config_file, platform=platform, disallow=disallow) output_dir = Path( @@ -278,6 +289,7 @@ def main() -> None: sys.exit(0) manylinux_images: Dict[str, str] = {} + musllinux_images: Dict[str, str] = {} if platform == "linux": pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg" all_pinned_docker_images = ConfigParser() @@ -303,6 +315,20 @@ def main() -> None: manylinux_images[build_platform] = image + for build_platform in MUSLLINUX_ARCHS: + pinned_images = all_pinned_docker_images[build_platform] + + config_value = options(f"musllinux-{build_platform}-image") + + if config_value is None: + image = pinned_images.get("musllinux_1_1") + elif config_value in pinned_images: + image = pinned_images[config_value] + else: + image = config_value + + musllinux_images[build_platform] = image + build_options = BuildOptions( architectures=archs, package_dir=package_dir, @@ -320,6 +346,7 @@ def main() -> None: environment=environment, dependency_constraints=dependency_constraints, manylinux_images=manylinux_images or None, + musllinux_images=musllinux_images or None, build_frontend=build_frontend, ) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 24008cdf6..321101e96 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -61,6 +61,7 @@ def build(options: BuildOptions) -> None: sys.exit(2) assert options.manylinux_images is not None + assert options.musllinux_images is not None python_configurations = get_python_configurations(options.build_selector, options.architectures) platforms = [ ("cp", "manylinux_x86_64", options.manylinux_images["x86_64"]), @@ -71,6 +72,11 @@ def build(options: BuildOptions) -> None: ("pp", "manylinux_x86_64", options.manylinux_images["pypy_x86_64"]), ("pp", "manylinux_aarch64", options.manylinux_images["pypy_aarch64"]), ("pp", "manylinux_i686", options.manylinux_images["pypy_i686"]), + ("cp", "musllinux_x86_64", options.musllinux_images["x86_64"]), + ("cp", "musllinux_i686", options.musllinux_images["i686"]), + ("cp", "musllinux_aarch64", options.musllinux_images["aarch64"]), + ("cp", "musllinux_ppc64le", options.musllinux_images["ppc64le"]), + ("cp", "musllinux_s390x", options.musllinux_images["s390x"]), ] cwd = Path.cwd() diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 0ade9751a..12c4462a3 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -20,6 +20,11 @@ "manylinux_aarch64": "manylinux aarch64", "manylinux_ppc64le": "manylinux ppc64le", "manylinux_s390x": "manylinux s390x", + "musllinux_x86_64": "musllinux x86_64", + "musllinux_i686": "musllinux i686", + "musllinux_aarch64": "musllinux aarch64", + "musllinux_ppc64le": "musllinux ppc64le", + "musllinux_s390x": "manylinux s390x", "win32": "Windows 32bit", "win_amd64": "Windows 64bit", "macosx_x86_64": "macOS x86_64", diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 740dc6c8e..5f1b24257 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -28,6 +28,31 @@ python_configurations = [ { identifier = "cp310-manylinux_s390x", version = "3.10", path_str = "/opt/python/cp310-cp310" }, { identifier = "pp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, { identifier = "pp37-manylinux_i686", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, + { identifier = "cp36-musllinux_x86_64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-musllinux_x86_64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-musllinux_x86_64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-musllinux_x86_64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-musllinux_x86_64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, + { identifier = "cp36-musllinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-musllinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-musllinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-musllinux_i686", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-musllinux_i686", version = "3.10", path_str = "/opt/python/cp310-cp310" }, + { identifier = "cp36-musllinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-musllinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-musllinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-musllinux_aarch64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-musllinux_aarch64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, + { identifier = "cp36-musllinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-musllinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-musllinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-musllinux_ppc64le", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-musllinux_ppc64le", version = "3.10", path_str = "/opt/python/cp310-cp310" }, + { identifier = "cp36-musllinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, + { identifier = "cp37-musllinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp38-musllinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-musllinux_s390x", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-musllinux_s390x", version = "3.10", path_str = "/opt/python/cp310-cp310" }, ] [macos] diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 5bf179c89..3c7d3583a 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -27,6 +27,12 @@ manylinux-pypy_x86_64-image = "manylinux2010" manylinux-pypy_i686-image = "manylinux2010" manylinux-pypy_aarch64-image = "manylinux2014" +musllinux-x86_64-image = "musllinux_1_1" +musllinux-i686-image = "musllinux_1_1" +musllinux-aarch64-image = "musllinux_1_1" +musllinux-ppc64le-image = "musllinux_1_1" +musllinux-s390x-image = "musllinux_1_1" + [tool.cibuildwheel.linux] repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index 20b564f92..5d3ac3124 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,38 +1,43 @@ [x86_64] manylinux1 = quay.io/pypa/manylinux1_x86_64:2021-09-18-65abb78 -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2021-09-18-c1afc21 -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2021-09-18-c1afc21 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2021-09-18-c1afc21 +manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2021-09-18-f12faf3 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2021-09-18-f12faf3 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2021-09-18-f12faf3 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2021-09-18-f12faf3 [i686] manylinux1 = quay.io/pypa/manylinux1_i686:2021-09-18-65abb78 -manylinux2010 = quay.io/pypa/manylinux2010_i686:2021-09-18-c1afc21 -manylinux2014 = quay.io/pypa/manylinux2014_i686:2021-09-18-c1afc21 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2021-09-18-c1afc21 +manylinux2010 = quay.io/pypa/manylinux2010_i686:2021-09-18-f12faf3 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2021-09-18-f12faf3 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2021-09-18-f12faf3 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2021-09-18-f12faf3 [pypy_x86_64] -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2021-09-18-c1afc21 -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2021-09-18-c1afc21 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2021-09-18-c1afc21 +manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2021-09-18-f12faf3 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2021-09-18-f12faf3 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2021-09-18-f12faf3 [pypy_i686] -manylinux2010 = quay.io/pypa/manylinux2010_i686:2021-09-18-c1afc21 -manylinux2014 = quay.io/pypa/manylinux2014_i686:2021-09-18-c1afc21 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2021-09-18-c1afc21 +manylinux2010 = quay.io/pypa/manylinux2010_i686:2021-09-18-f12faf3 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2021-09-18-f12faf3 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2021-09-18-f12faf3 [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2021-09-18-c1afc21 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2021-09-18-c1afc21 +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2021-09-18-f12faf3 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2021-09-18-f12faf3 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2021-09-18-f12faf3 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2021-09-18-c1afc21 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2021-09-18-c1afc21 +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2021-09-18-f12faf3 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2021-09-18-f12faf3 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2021-09-18-f12faf3 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2021-09-18-c1afc21 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2021-09-18-c1afc21 +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2021-09-18-f12faf3 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2021-09-18-f12faf3 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2021-09-18-f12faf3 [pypy_aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2021-09-18-c1afc21 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2021-09-18-c1afc21 +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2021-09-18-f12faf3 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2021-09-18-f12faf3 diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 581799cf8..3d9eb1be6 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -215,6 +215,7 @@ class BuildOptions(NamedTuple): before_build: Optional[str] repair_command: str manylinux_images: Optional[Dict[str, str]] + musllinux_images: Optional[Dict[str, str]] dependency_constraints: Optional[DependencyConstraints] test_command: Optional[str] test_selector: TestSelector diff --git a/test/test_0_basic.py b/test/test_0_basic.py index 8f2cd4bfb..ec72483a3 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -56,4 +56,6 @@ def test_build_identifiers(tmp_path): build_identifiers = utils.cibuildwheel_get_build_identifiers( project_dir, prerelease_pythons=True ) - assert len(expected_wheels) == len(build_identifiers) + assert len(expected_wheels) == len( + build_identifiers + ), f"{expected_wheels} vs {build_identifiers}" diff --git a/test/test_docker_images.py b/test/test_docker_images.py index 260c74142..015f40696 100644 --- a/test/test_docker_images.py +++ b/test/test_docker_images.py @@ -37,14 +37,14 @@ def test(tmp_path): add_env={ "CIBW_MANYLINUX_X86_64_IMAGE": "dockcross/manylinux2010-x64", "CIBW_MANYLINUX_I686_IMAGE": "dockcross/manylinux2010-x86", - "CIBW_BUILD": "cp3{6,7,8,9}-*", + "CIBW_BUILD": "cp3{6,7,8,9}-manylinux*", }, ) # also check that we got the right wheels built expected_wheels = [ w - for w in utils.expected_wheels("spam", "0.1.0") + for w in utils.expected_wheels("spam", "0.1.0", musllinux_versions=[]) if "-cp36-" in w or "-cp37-" in w or "-cp38-" in w or "-cp39-" in w ] assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_manylinuxXXXX_only.py b/test/test_manylinuxXXXX_only.py index c4b88b7cb..f7c533d49 100644 --- a/test/test_manylinuxXXXX_only.py +++ b/test/test_manylinuxXXXX_only.py @@ -57,6 +57,7 @@ def test(manylinux_image, tmp_path): # CFLAGS environment variable is necessary to fail on 'malloc_info' (on manylinux1) during compilation/linking, # rather than when dynamically loading the Python add_env = { + "CIBW_BUILD": "*-manylinux*", "CIBW_ENVIRONMENT": 'CFLAGS="$CFLAGS -O0 -Werror=implicit-function-declaration"', "CIBW_MANYLINUX_X86_64_IMAGE": manylinux_image, "CIBW_MANYLINUX_I686_IMAGE": manylinux_image, @@ -79,7 +80,10 @@ def test(manylinux_image, tmp_path): "manylinux2014": ["manylinux_2_17", "manylinux2014"], } expected_wheels = utils.expected_wheels( - "spam", "0.1.0", manylinux_versions=platform_tag_map.get(manylinux_image, [manylinux_image]) + "spam", + "0.1.0", + manylinux_versions=platform_tag_map.get(manylinux_image, [manylinux_image]), + musllinux_versions=[], ) if manylinux_image in {"manylinux1"}: # remove PyPy & CPython 3.10 and above diff --git a/test/utils.py b/test/utils.py index c1baf4d93..524f77473 100644 --- a/test/utils.py +++ b/test/utils.py @@ -103,6 +103,7 @@ def expected_wheels( package_name, package_version, manylinux_versions=None, + musllinux_versions=None, macosx_deployment_target="10.9", machine_arch=None, ): @@ -123,6 +124,9 @@ def expected_wheels( else: manylinux_versions = ["manylinux_2_17", "manylinux2014"] + if musllinux_versions is None: + musllinux_versions = ["musllinux_1_1"] + python_abi_tags = ["cp36-cp36m", "cp37-cp37m", "cp38-cp38", "cp39-cp39", "cp310-cp310"] if machine_arch in ["x86_64", "AMD64", "x86", "aarch64"]: @@ -155,6 +159,16 @@ def expected_wheels( ) for architecture in architectures ] + if len(musllinux_versions) > 0 and not python_abi_tag.startswith("pp"): + platform_tags.extend( + [ + ".".join( + f"{musllinux_version}_{architecture}" + for musllinux_version in musllinux_versions + ) + for architecture in architectures + ] + ) elif platform == "windows": if python_abi_tag.startswith("cp"):