feat: add musllinux support (PEP 656)

Add musllinux support (PEP 656) allowing to build wheels for musl libc based distros (e.g. Alpine)
pypa · Sep 18, 2021 · 45fc85e · 45fc85e
1 parent 27d4b6a
commit 45fc85e
Show file tree

Hide file tree

Showing 12 changed files with 131 additions and 29 deletions.
diff --git a/bin/update_docker.py b/bin/update_docker.py
@@ -19,14 +19,15 @@ class Image(NamedTuple):
 
 
 images = [
+    # manylinux1 images
     Image("manylinux1", "x86_64", "quay.io/pypa/manylinux1_x86_64", None),
     Image("manylinux1", "i686", "quay.io/pypa/manylinux1_i686", None),
-    # 2010 images
+    # manylinux2010 images
     Image("manylinux2010", "x86_64", "quay.io/pypa/manylinux2010_x86_64", None),
     Image("manylinux2010", "i686", "quay.io/pypa/manylinux2010_i686", None),
     Image("manylinux2010", "pypy_x86_64", "quay.io/pypa/manylinux2010_x86_64", None),
     Image("manylinux2010", "pypy_i686", "quay.io/pypa/manylinux2010_i686", None),
-    # 2014 images
+    # manylinux2014 images
     Image("manylinux2014", "x86_64", "quay.io/pypa/manylinux2014_x86_64", None),
     Image("manylinux2014", "i686", "quay.io/pypa/manylinux2014_i686", None),
     Image("manylinux2014", "aarch64", "quay.io/pypa/manylinux2014_aarch64", None),
@@ -35,7 +36,7 @@ class Image(NamedTuple):
     Image("manylinux2014", "pypy_x86_64", "quay.io/pypa/manylinux2014_x86_64", None),
     Image("manylinux2014", "pypy_i686", "quay.io/pypa/manylinux2014_i686", None),
     Image("manylinux2014", "pypy_aarch64", "quay.io/pypa/manylinux2014_aarch64", None),
-    # 2_24 images
+    # manylinux_2_24 images
     Image("manylinux_2_24", "x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None),
     Image("manylinux_2_24", "i686", "quay.io/pypa/manylinux_2_24_i686", None),
     Image("manylinux_2_24", "aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None),
@@ -44,6 +45,12 @@ class Image(NamedTuple):
     Image("manylinux_2_24", "pypy_x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None),
     Image("manylinux_2_24", "pypy_i686", "quay.io/pypa/manylinux_2_24_i686", None),
     Image("manylinux_2_24", "pypy_aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None),
+    # musllinux_1_1 images
+    Image("musllinux_1_1", "x86_64", "quay.io/pypa/musllinux_1_1_x86_64", None),
+    Image("musllinux_1_1", "i686", "quay.io/pypa/musllinux_1_1_i686", None),
+    Image("musllinux_1_1", "aarch64", "quay.io/pypa/musllinux_1_1_aarch64", None),
+    Image("musllinux_1_1", "ppc64le", "quay.io/pypa/musllinux_1_1_ppc64le", None),
+    Image("musllinux_1_1", "s390x", "quay.io/pypa/musllinux_1_1_s390x", None),
 ]
 
 config = configparser.ConfigParser()

diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py
@@ -41,6 +41,14 @@
     "pypy_i686",
 )
 
+MUSLLINUX_ARCHS = (
+    "x86_64",
+    "i686",
+    "aarch64",
+    "ppc64le",
+    "s390x",
+)
+
 
 def main() -> None:
     platform: PlatformName
@@ -167,10 +175,13 @@ def main() -> None:
     manylinux_identifiers = {
         f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS
     }
+    musllinux_identifiers = {
+        f"musllinux-{build_platform}-image" for build_platform in MUSLLINUX_ARCHS
+    }
     disallow = {
         "linux": {"dependency-versions"},
-        "macos": manylinux_identifiers,
-        "windows": manylinux_identifiers,
+        "macos": manylinux_identifiers | musllinux_identifiers,
+        "windows": manylinux_identifiers | musllinux_identifiers,
     }
     options = ConfigOptions(package_dir, args.config_file, platform=platform, disallow=disallow)
     output_dir = Path(
@@ -278,6 +289,7 @@ def main() -> None:
         sys.exit(0)
 
     manylinux_images: Dict[str, str] = {}
+    musllinux_images: Dict[str, str] = {}
     if platform == "linux":
         pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg"
         all_pinned_docker_images = ConfigParser()
@@ -303,6 +315,20 @@ def main() -> None:
 
             manylinux_images[build_platform] = image
 
+        for build_platform in MUSLLINUX_ARCHS:
+            pinned_images = all_pinned_docker_images[build_platform]
+
+            config_value = options(f"musllinux-{build_platform}-image")
+
+            if config_value is None:
+                image = pinned_images.get("musllinux_1_1")
+            elif config_value in pinned_images:
+                image = pinned_images[config_value]
+            else:
+                image = config_value
+
+            musllinux_images[build_platform] = image
+
     build_options = BuildOptions(
         architectures=archs,
         package_dir=package_dir,
@@ -320,6 +346,7 @@ def main() -> None:
         environment=environment,
         dependency_constraints=dependency_constraints,
         manylinux_images=manylinux_images or None,
+        musllinux_images=musllinux_images or None,
         build_frontend=build_frontend,
     )
 

diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py
@@ -61,6 +61,7 @@ def build(options: BuildOptions) -> None:
         sys.exit(2)
 
     assert options.manylinux_images is not None
+    assert options.musllinux_images is not None
     python_configurations = get_python_configurations(options.build_selector, options.architectures)
     platforms = [
         ("cp", "manylinux_x86_64", options.manylinux_images["x86_64"]),
@@ -71,6 +72,11 @@ def build(options: BuildOptions) -> None:
         ("pp", "manylinux_x86_64", options.manylinux_images["pypy_x86_64"]),
         ("pp", "manylinux_aarch64", options.manylinux_images["pypy_aarch64"]),
         ("pp", "manylinux_i686", options.manylinux_images["pypy_i686"]),
+        ("cp", "musllinux_x86_64", options.musllinux_images["x86_64"]),
+        ("cp", "musllinux_i686", options.musllinux_images["i686"]),
+        ("cp", "musllinux_aarch64", options.musllinux_images["aarch64"]),
+        ("cp", "musllinux_ppc64le", options.musllinux_images["ppc64le"]),
+        ("cp", "musllinux_s390x", options.musllinux_images["s390x"]),
     ]
 
     cwd = Path.cwd()

diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py
@@ -20,6 +20,11 @@
     "manylinux_aarch64": "manylinux aarch64",
     "manylinux_ppc64le": "manylinux ppc64le",
     "manylinux_s390x": "manylinux s390x",
+    "musllinux_x86_64": "musllinux x86_64",
+    "musllinux_i686": "musllinux i686",
+    "musllinux_aarch64": "musllinux aarch64",
+    "musllinux_ppc64le": "musllinux ppc64le",
+    "musllinux_s390x": "manylinux s390x",
     "win32": "Windows 32bit",
     "win_amd64": "Windows 64bit",
     "macosx_x86_64": "macOS x86_64",

diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml
@@ -28,6 +28,31 @@ python_configurations = [
   { identifier = "cp310-manylinux_s390x", version = "3.10", path_str = "/opt/python/cp310-cp310" },
   { identifier = "pp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" },
   { identifier = "pp37-manylinux_i686", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" },
+  { identifier = "cp36-musllinux_x86_64", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
+  { identifier = "cp37-musllinux_x86_64", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
+  { identifier = "cp38-musllinux_x86_64", version = "3.8", path_str = "/opt/python/cp38-cp38" },
+  { identifier = "cp39-musllinux_x86_64", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-musllinux_x86_64", version = "3.10", path_str = "/opt/python/cp310-cp310" },
+  { identifier = "cp36-musllinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
+  { identifier = "cp37-musllinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
+  { identifier = "cp38-musllinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" },
+  { identifier = "cp39-musllinux_i686", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-musllinux_i686", version = "3.10", path_str = "/opt/python/cp310-cp310" },
+  { identifier = "cp36-musllinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
+  { identifier = "cp37-musllinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
+  { identifier = "cp38-musllinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" },
+  { identifier = "cp39-musllinux_aarch64", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-musllinux_aarch64", version = "3.10", path_str = "/opt/python/cp310-cp310" },
+  { identifier = "cp36-musllinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
+  { identifier = "cp37-musllinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
+  { identifier = "cp38-musllinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" },
+  { identifier = "cp39-musllinux_ppc64le", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-musllinux_ppc64le", version = "3.10", path_str = "/opt/python/cp310-cp310" },
+  { identifier = "cp36-musllinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
+  { identifier = "cp37-musllinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
+  { identifier = "cp38-musllinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" },
+  { identifier = "cp39-musllinux_s390x", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-musllinux_s390x", version = "3.10", path_str = "/opt/python/cp310-cp310" },
 ]
 
 [macos]

diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml
@@ -27,6 +27,12 @@ manylinux-pypy_x86_64-image = "manylinux2010"
 manylinux-pypy_i686-image = "manylinux2010"
 manylinux-pypy_aarch64-image = "manylinux2014"
 
+musllinux-x86_64-image = "musllinux_1_1"
+musllinux-i686-image = "musllinux_1_1"
+musllinux-aarch64-image = "musllinux_1_1"
+musllinux-ppc64le-image = "musllinux_1_1"
+musllinux-s390x-image = "musllinux_1_1"
+
 
 [tool.cibuildwheel.linux]
 repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"

diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg
@@ -1,38 +1,43 @@
 [x86_64]
 manylinux1 = quay.io/pypa/manylinux1_x86_64:2021-09-18-65abb78
-manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2021-09-18-c1afc21
-manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2021-09-18-c1afc21
-manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2021-09-18-c1afc21
+manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2021-09-18-f12faf3
+manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2021-09-18-f12faf3
+manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2021-09-18-f12faf3
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2021-09-18-f12faf3
 
 [i686]
 manylinux1 = quay.io/pypa/manylinux1_i686:2021-09-18-65abb78
-manylinux2010 = quay.io/pypa/manylinux2010_i686:2021-09-18-c1afc21
-manylinux2014 = quay.io/pypa/manylinux2014_i686:2021-09-18-c1afc21
-manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2021-09-18-c1afc21
+manylinux2010 = quay.io/pypa/manylinux2010_i686:2021-09-18-f12faf3
+manylinux2014 = quay.io/pypa/manylinux2014_i686:2021-09-18-f12faf3
+manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2021-09-18-f12faf3
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2021-09-18-f12faf3
 
 [pypy_x86_64]
-manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2021-09-18-c1afc21
-manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2021-09-18-c1afc21
-manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2021-09-18-c1afc21
+manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2021-09-18-f12faf3
+manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2021-09-18-f12faf3
+manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2021-09-18-f12faf3
 
 [pypy_i686]
-manylinux2010 = quay.io/pypa/manylinux2010_i686:2021-09-18-c1afc21
-manylinux2014 = quay.io/pypa/manylinux2014_i686:2021-09-18-c1afc21
-manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2021-09-18-c1afc21
+manylinux2010 = quay.io/pypa/manylinux2010_i686:2021-09-18-f12faf3
+manylinux2014 = quay.io/pypa/manylinux2014_i686:2021-09-18-f12faf3
+manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2021-09-18-f12faf3
 
 [aarch64]
-manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2021-09-18-c1afc21
-manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2021-09-18-c1afc21
+manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2021-09-18-f12faf3
+manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2021-09-18-f12faf3
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2021-09-18-f12faf3
 
 [ppc64le]
-manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2021-09-18-c1afc21
-manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2021-09-18-c1afc21
+manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2021-09-18-f12faf3
+manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2021-09-18-f12faf3
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2021-09-18-f12faf3
 
 [s390x]
-manylinux2014 = quay.io/pypa/manylinux2014_s390x:2021-09-18-c1afc21
-manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2021-09-18-c1afc21
+manylinux2014 = quay.io/pypa/manylinux2014_s390x:2021-09-18-f12faf3
+manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2021-09-18-f12faf3
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2021-09-18-f12faf3
 
 [pypy_aarch64]
-manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2021-09-18-c1afc21
-manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2021-09-18-c1afc21
+manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2021-09-18-f12faf3
+manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2021-09-18-f12faf3
 
diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py
@@ -215,6 +215,7 @@ class BuildOptions(NamedTuple):
     before_build: Optional[str]
     repair_command: str
     manylinux_images: Optional[Dict[str, str]]
+    musllinux_images: Optional[Dict[str, str]]
     dependency_constraints: Optional[DependencyConstraints]
     test_command: Optional[str]
     test_selector: TestSelector

diff --git a/test/test_0_basic.py b/test/test_0_basic.py
@@ -56,4 +56,6 @@ def test_build_identifiers(tmp_path):
     build_identifiers = utils.cibuildwheel_get_build_identifiers(
         project_dir, prerelease_pythons=True
     )
-    assert len(expected_wheels) == len(build_identifiers)
+    assert len(expected_wheels) == len(
+        build_identifiers
+    ), f"{expected_wheels} vs {build_identifiers}"
diff --git a/test/test_docker_images.py b/test/test_docker_images.py
@@ -37,14 +37,14 @@ def test(tmp_path):
         add_env={
             "CIBW_MANYLINUX_X86_64_IMAGE": "dockcross/manylinux2010-x64",
             "CIBW_MANYLINUX_I686_IMAGE": "dockcross/manylinux2010-x86",
-            "CIBW_BUILD": "cp3{6,7,8,9}-*",
+            "CIBW_BUILD": "cp3{6,7,8,9}-manylinux*",
         },
     )
 
     # also check that we got the right wheels built
     expected_wheels = [
         w
-        for w in utils.expected_wheels("spam", "0.1.0")
+        for w in utils.expected_wheels("spam", "0.1.0", musllinux_versions=[])
         if "-cp36-" in w or "-cp37-" in w or "-cp38-" in w or "-cp39-" in w
     ]
     assert set(actual_wheels) == set(expected_wheels)
diff --git a/test/test_manylinuxXXXX_only.py b/test/test_manylinuxXXXX_only.py
@@ -57,6 +57,7 @@ def test(manylinux_image, tmp_path):
     # CFLAGS environment variable is necessary to fail on 'malloc_info' (on manylinux1) during compilation/linking,
     # rather than when dynamically loading the Python
     add_env = {
+        "CIBW_BUILD": "*-manylinux*",
         "CIBW_ENVIRONMENT": 'CFLAGS="$CFLAGS -O0 -Werror=implicit-function-declaration"',
         "CIBW_MANYLINUX_X86_64_IMAGE": manylinux_image,
         "CIBW_MANYLINUX_I686_IMAGE": manylinux_image,
@@ -79,7 +80,10 @@ def test(manylinux_image, tmp_path):
         "manylinux2014": ["manylinux_2_17", "manylinux2014"],
     }
     expected_wheels = utils.expected_wheels(
-        "spam", "0.1.0", manylinux_versions=platform_tag_map.get(manylinux_image, [manylinux_image])
+        "spam",
+        "0.1.0",
+        manylinux_versions=platform_tag_map.get(manylinux_image, [manylinux_image]),
+        musllinux_versions=[],
     )
     if manylinux_image in {"manylinux1"}:
         # remove PyPy & CPython 3.10 and above

diff --git a/test/utils.py b/test/utils.py
@@ -103,6 +103,7 @@ def expected_wheels(
     package_name,
     package_version,
     manylinux_versions=None,
+    musllinux_versions=None,
     macosx_deployment_target="10.9",
     machine_arch=None,
 ):
@@ -123,6 +124,9 @@ def expected_wheels(
         else:
             manylinux_versions = ["manylinux_2_17", "manylinux2014"]
 
+    if musllinux_versions is None:
+        musllinux_versions = ["musllinux_1_1"]
+
     python_abi_tags = ["cp36-cp36m", "cp37-cp37m", "cp38-cp38", "cp39-cp39", "cp310-cp310"]
 
     if machine_arch in ["x86_64", "AMD64", "x86", "aarch64"]:
@@ -155,6 +159,16 @@ def expected_wheels(
                 )
                 for architecture in architectures
             ]
+            if len(musllinux_versions) > 0 and not python_abi_tag.startswith("pp"):
+                platform_tags.extend(
+                    [
+                        ".".join(
+                            f"{musllinux_version}_{architecture}"
+                            for musllinux_version in musllinux_versions
+                        )
+                        for architecture in architectures
+                    ]
+                )
 
         elif platform == "windows":
             if python_abi_tag.startswith("cp"):