[feature] implement lipo deployer (conan-io#52)

extensions/deployers/lipo/README.md

@@ -0,0 +1,57 @@
+## Lipo deployers
+
+These deployers create universal binaries on macOS or iOS.
+They are similar to Conan's full_deploy deployer but run `lipo` to
+produce universal binaries. An Xcode project can use the `full_deploy`
+folder to compile a universal application.
+
+
+#### [lipo](lipo.py)
+
+This deployer is identical to `full_deploy` except single architecture universal
+binaries will be created and a subfolder will not be created with the arch name.
+
+
+#### [lipo_add](lipo_add.py)
+
+This deployer doesn't remove the existing `full_deploy` folder but adds an architecture
+to the existing universal binaries. It is an error to run this more than once for a given
+architecture. Universal binaries can only contain one binary per architecture. See
+`lipo -create` for more information.
+
+```sh
+$ conan install . --deploy=lipo . -s arch=x86_64
+$ conan install . --deploy=lipo_add . -s arch=armv8
+```
+
+Profiles can be used to handle additional settings, for example when the architectures
+have a different minimum deployment OS.
+
+
+## Sample profiles
+
+#### x86_64
+```
+[settings]
+arch=x86_64
+build_type=Release
+compiler=apple-clang
+compiler.cppstd=gnu17
+compiler.libcxx=libc++
+compiler.version=14
+os=Macos
+os.version=10.13
+```
+
+#### armv8
+```
+[settings]
+arch=armv8
+build_type=Release
+compiler=apple-clang
+compiler.cppstd=gnu17
+compiler.libcxx=libc++
+compiler.version=14
+os=Macos
+os.version=11.0
+```

extensions/deployers/lipo/lipo.py

@@ -0,0 +1,31 @@
+import os
+import shutil
+
+from conan.tools.files import copy
+from conans.util.files import rmdir
+from conan.tools.apple import is_apple_os
+
+from tools_lipo import lipo_tree
+
+
+def deploy(graph, output_folder, **kwargs):
+    # Note the kwargs argument is mandatory to be robust against future changes.
+    conanfile = graph.root.conanfile
+    conanfile.output.info(f"lipo deployer to {output_folder}")
+    for name, dep in graph.root.conanfile.dependencies.items():
+        if dep.package_folder is None:
+            continue
+        folder_name = os.path.join("full_deploy", dep.context, dep.ref.name, str(dep.ref.version))
+        build_type = dep.info.settings.get_safe("build_type")
+        arch = dep.info.settings.get_safe("arch")
+        if build_type:
+            folder_name = os.path.join(folder_name, build_type)
+        if arch and not is_apple_os(conanfile):
+            folder_name = os.path.join(folder_name, arch)
+        new_folder = os.path.join(output_folder, folder_name)
+        rmdir(new_folder)
+        if is_apple_os(conanfile):
+            lipo_tree(conanfile, new_folder, [dep.package_folder], add=True)
+        else:
+            shutil.copytree(dep.package_folder, new_folder, symlinks=True)
+        dep.set_deploy_folder(new_folder)

extensions/deployers/lipo/lipo_add.py

@@ -0,0 +1,31 @@
+import os
+import shutil
+
+from conan.tools.files import copy
+from conans.util.files import rmdir
+from conan.tools.apple import is_apple_os
+
+from tools_lipo import lipo_tree
+
+
+def deploy(graph, output_folder, **kwargs):
+    # Note the kwargs argument is mandatory to be robust against future changes.
+    conanfile = graph.root.conanfile
+    conanfile.output.info(f"lipo deployer to {output_folder}")
+    for name, dep in graph.root.conanfile.dependencies.items():
+        if dep.package_folder is None:
+            continue
+        folder_name = os.path.join("full_deploy", dep.context, dep.ref.name, str(dep.ref.version))
+        build_type = dep.info.settings.get_safe("build_type")
+        arch = dep.info.settings.get_safe("arch")
+        if build_type:
+            folder_name = os.path.join(folder_name, build_type)
+        if arch and not is_apple_os(conanfile):
+            folder_name = os.path.join(folder_name, arch)
+        new_folder = os.path.join(output_folder, folder_name)
+        if is_apple_os(conanfile):
+            lipo_tree(conanfile, new_folder, [dep.package_folder], add=True)
+        else:
+            rmdir(new_folder)
+            shutil.copytree(dep.package_folder, new_folder, symlinks=True)
+        dep.set_deploy_folder(new_folder)

extensions/deployers/lipo/tools_lipo.py

@@ -0,0 +1,167 @@
+import os
+import shlex
+import shutil
+from subprocess import Popen
+
+from conan.errors import ConanException
+
+
+__all__ = ['is_macho_binary', 'lipo']
+
+# These are for optimization only, to avoid unnecessarily reading files.
+_binary_exts = ['.a', '.dylib']
+_regular_exts = [
+    '.h', '.hpp', '.hxx', '.c', '.cc', '.cxx', '.cpp', '.m', '.mm', '.txt', '.md', '.html', '.jpg', '.png'
+]
+
+
+def lipo(conanfile, command='create', output=None, inputs=(), arch_type=None, arch_types=()):
+    """
+    Run lipo for one or more files (see lipo manpage)
+    Arguments:
+      conanfile
+      command: one of
+        archs
+        create
+        detailed_info
+        extract
+        extract_family
+        info
+        remove
+        replace
+        thin
+        verify
+      output: output file (not used by all commands)
+      inputs: one or more input files
+      arch_type: architecture, if required
+      arch_types: multiple architectures, if required
+    """
+    archs = arch_types or ([arch_type] if arch_type else [])
+    if len(inputs) != 1 and command not in ('create', 'detailed_info', 'info', 'replace'):
+        raise ConanException(f'lipo {command} requires exactly one input file')
+    if len(archs) != 1 and command in ('thin',):
+        raise ConanException(f'lipo {command} requires exactly one arch_type')
+    if archs and command not in ('extract', 'extract_family', 'remove', 'replace', 'verify_arch'):
+        raise ConanException(f'lipo {command} does not require arch_type')
+    cmd = ['lipo'] + list(inputs) + [f"-{command}"]
+    if command in ('archs', 'detailed_info', 'info', 'verify_arch'):
+        return Popen(cmd)
+    elif command == 'verify_arch':
+        return Popen(cmd + list(archs))
+    if not output:
+        raise ConanException('lipo output is required')
+    cmd_output = ['-output', output]
+    if command == 'create':
+        conanfile.run(shlex.join(cmd + cmd_output))
+    elif command in ('extract', 'extract_family', 'remove'):
+        cmd = cmd[:-1] # remove command
+        for arch in archs:
+            cmd += [f"-{command}", arch]
+        conanfile.run(shlex.join(cmd + cmd_output))
+    elif command == 'replace':
+        if len(archs) + 1 != len(inputs):
+            raise ConanException(f'lipo {command} requires exactly one arch_type per input and one universal input')
+        cmd = ['lipo', inputs[0]]
+        for i in range(len(archs)):
+            cmd += [f"-{command}", archs[i], inputs[i + 1]]
+        conanfile.run(shlex.join(cmd + cmd_output))
+    elif command == 'thin':
+        conanfile.run(shlex.join(cmd + list(archs) + cmd_output))
+    else:
+        raise ConanException(f'lipo {command} is not a valid command')
+
+
+def is_macho_binary(filename):
+    """
+    Determines if filename is a Mach-O binary or fat binary
+    """
+    ext = os.path.splitext(filename)[1]
+    if ext in _binary_exts:
+        return True
+    if ext in _regular_exts:
+        return False
+    with open(filename, "rb") as f:
+        header = f.read(4)
+        if header == b'\xcf\xfa\xed\xfe':
+            # cffaedfe is Mach-O binary
+            return True
+        elif header == b'\xca\xfe\xba\xbe':
+            # cafebabe is Mach-O fat binary
+            return True
+        elif header == b'!<arch>\n':
+            # ar archive
+            return True
+    return False
+
+
+def copy_arch_file(conanfile, src, dst, top=None, arch_folders=(), add=False):
+    if os.path.isfile(src):
+        if top and arch_folders and is_macho_binary(src):
+            # Try to lipo all available archs on the first path.
+            src_components = src.split(os.path.sep)
+            top_components = top.split(os.path.sep)
+            if src_components[:len(top_components)] == top_components:
+                paths = [os.path.join(a, *(src_components[len(top_components):])) for a in arch_folders]
+                paths = [p for p in paths if os.path.isfile(p)]
+                if len(paths) > 1 or add:
+                    if add and os.path.exists(dst):
+                        lipo(conanfile, 'create', output=dst, inputs=[dst] + paths)
+                    else:
+                        lipo(conanfile, 'create', output=dst, inputs=paths)
+                    return
+        if os.path.exists(dst):
+            pass # don't overwrite existing files
+        else:
+            shutil.copy2(src, dst)
+
+
+# Modified copytree to copy new files to an existing tree.
+def graft_tree(src, dst, symlinks=False, copy_function=shutil.copy2, dirs_exist_ok=False):
+    names = os.listdir(src)
+    os.makedirs(dst, exist_ok=dirs_exist_ok)
+    errors = []
+    for name in names:
+        srcname = os.path.join(src, name)
+        dstname = os.path.join(dst, name)
+        try:
+            if symlinks and os.path.islink(srcname):
+                if os.path.exists(dstname):
+                    continue
+                linkto = os.readlink(srcname)
+                os.symlink(linkto, dstname)
+            elif os.path.isdir(srcname):
+                graft_tree(srcname, dstname, symlinks, copy_function, dirs_exist_ok)
+            else:
+                copy_function(srcname, dstname)
+            # What about devices, sockets etc.?
+        # catch the Error from the recursive graft_tree so that we can
+        # continue with other files
+        except shutil.Error as err:
+            errors.extend(err.args[0])
+        except OSError as why:
+            errors.append((srcname, dstname, str(why)))
+    try:
+        shutil.copystat(src, dst)
+    except OSError as why:
+        # can't copy file access times on Windows
+        if why.winerror is None: # pylint: disable=no-member
+            errors.extend((src, dst, str(why)))
+    if errors:
+        raise shutil.Error(errors)
+
+def lipo_tree(conanfile, dst_folder, arch_folders, add=False):
+    if add:
+        copy_function = lambda s, d: copy_arch_file(conanfile, s, d,
+                                                    top=folder,
+                                                    arch_folders=arch_folders,
+                                                    add=True)
+    else:
+        copy_function = lambda s, d: copy_arch_file(conanfile, s, d,
+                                                    top=folder,
+                                                    arch_folders=arch_folders)
+    for folder in arch_folders:
+        graft_tree(folder,
+                   dst_folder,
+                   symlinks=True,
+                   copy_function=copy_function,
+                   dirs_exist_ok=True)

tests/test_deploy_lipo.py

@@ -0,0 +1,59 @@
+import subprocess
+import tempfile
+import os
+
+import pytest
+
+from tools import save, run
+
+
+@pytest.fixture(autouse=True)
+def conan_test():
+    old_env = dict(os.environ)
+    env_vars = {"CONAN_HOME": tempfile.mkdtemp(suffix='conans')}
+    os.environ.update(env_vars)
+    current = tempfile.mkdtemp(suffix="conans")
+    cwd = os.getcwd()
+    os.chdir(current)
+    try:
+        yield
+    finally:
+        os.chdir(cwd)
+        os.environ.clear()
+        os.environ.update(old_env)
+
+def test_deploy_lipo():
+    repo = os.path.join(os.path.dirname(__file__), "..")
+    run(f"conan config install {repo}")
+    run("conan --help")
+
+    txt = textwrap.dedent("""
+        from conan import ConanFile
+        from conan.tools.cmake import cmake_layout
+
+        class Pkg(ConanFile):
+            # Note that we don't depend on arch
+            settings = "os", "compiler", "build_type"
+
+            requires = ("libtiff/4.5.0",)
+
+            def build(self):
+                # Don't use XcodeBuild because it passes a single -arch flag
+                build_type = self.settings.get_safe("build_type")
+                project = 'example.xcodeproj'
+                self.run('xcodebuild -configuration {} -project {} -alltargets'.format(build_type, project))
+        """)
+    save("conanfile.py", txt)
+
+    run("conan install . --deploy=lipo -s arch=x86_64")
+    run("conan install . --deploy=lipo_add -s arch=armv8")
+
+    # Check that a nested dependency is present and contains two architectures
+    depend_file = os.path.join("full_deploy", "host", "zlib", "1.2.13", "Release", "lib", "libz.a")
+    assert os.path.exists(depend_file)
+    with subprocess.Popen(["lipo", "-info", depend_file],
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE) as pipe:
+        info = pipe.stdout.readline()
+        assert info.find("x86_64") >= 0
+        assert info.find("arm64") >= 0