Install one copy of libs used by multiple extension modules

If you have a package that contains multiple C extensions that
depend on the same shared libraries, then `delocate-wheel` saves
duplicate copies of all of the shared libraries.

This makes for packages that are several times larger than
necessary. This patch causes `delocate-wheel` to sweep up all
of the dependencies and put them in a single directory, rather
than creating duplicates.

The convention is based on the proposal in pypa/auditwheel#89
and pypa/auditwheel#90.

delocate/delocating.py

@@ -10,6 +10,12 @@
 import warnings
 from subprocess import Popen, PIPE
 
+try:
+    from wheel.install import WHEEL_INFO_RE  # type: ignore
+except ImportError:  # As of Wheel 0.32.0
+    from wheel.wheelfile import WHEEL_INFO_RE  # type: ignore
+    WHEEL_INFO_RE = WHEEL_INFO_RE.match
+
 from .pycompat import string_types
 from .libsana import tree_libs, stripped_lib_dict, get_rp_stripper
 from .tools import (set_install_name, zip2dir, dir2zip, validate_signature,
@@ -227,7 +233,7 @@ def _copy_required(lib_path, copy_filt_func, copied_libs):
         # Haven't see this one before, add entry to copied_libs
         out_path = pjoin(lib_path, basename(required))
         if exists(out_path):
-            raise DelocationError(out_path + ' already exists')
+            continue
         shutil.copy(required, lib_path)
         copied2orig[out_path] = required
         copied_libs[required] = procd_requirings
@@ -370,15 +376,12 @@ def delocate_wheel(in_wheel,
         all_copied = {}
         wheel_dir = realpath(pjoin(tmpdir, 'wheel'))
         zip2dir(in_wheel, wheel_dir)
+        wheel_fname = basename(out_wheel)
+        lib_path = pjoin(wheel_dir, WHEEL_INFO_RE(wheel_fname).group('name') + lib_sdir)
         for package_path in find_package_dirs(wheel_dir):
-            lib_path = pjoin(package_path, lib_sdir)
             lib_path_exists = exists(lib_path)
             copied_libs = delocate_path(package_path, lib_path,
                                         lib_filt_func, copy_filt_func)
-            if copied_libs and lib_path_exists:
-                raise DelocationError(
-                    '{0} already exists in wheel but need to copy '
-                    '{1}'.format(lib_path, '; '.join(copied_libs)))
             if len(os.listdir(lib_path)) == 0:
                 shutil.rmtree(lib_path)
             # Check architectures

delocate/tests/test_delocating.py

@@ -3,8 +3,8 @@
 from __future__ import division, print_function
 
 import os
-from os.path import (join as pjoin, dirname, basename, relpath, realpath,
-                     splitext)
+from os.path import (join as pjoin, dirname, basename, getmtime, relpath,
+                     realpath, splitext)
 import shutil
 
 from ..delocating import (DelocationError, delocate_tree_libs, copy_recurse,
@@ -282,18 +282,17 @@ def test_copy_recurse_overwrite():
         os.makedirs('libcopy')
         test_lib, liba, libb, libc = _copy_fixpath(
             [TEST_LIB, LIBA, LIBB, LIBC], 'libcopy')
+        liba_mtime = getmtime(LIBA)
         # Filter system libs
         def filt_func(libname):
             return not libname.startswith('/usr/lib')
         os.makedirs('subtree')
         # libb depends on liba
         shutil.copy2(libb, 'subtree')
-        # If liba is already present, barf
+        # If liba is already present, we shouldn't try to copy it
         shutil.copy2(liba, 'subtree')
-        assert_raises(DelocationError, copy_recurse, 'subtree', filt_func)
-        # Works if liba not present
-        os.unlink(pjoin('subtree', 'liba.dylib'))
         copy_recurse('subtree', filt_func)
+        assert_equal(liba_mtime, getmtime(pjoin('subtree', 'liba.dylib')))
 
 
 def test_delocate_path():

delocate/tests/test_scripts.py

@@ -159,7 +159,7 @@ def _check_wheel(wheel_fname, lib_sdir):
     wheel_fname = abspath(wheel_fname)
     with InTemporaryDirectory():
         zip2dir(wheel_fname, 'plat_pkg')
-        dylibs = pjoin('plat_pkg', 'fakepkg1', lib_sdir)
+        dylibs = pjoin('plat_pkg', lib_sdir)
         assert_true(exists(dylibs))
         assert_equal(os.listdir(dylibs), ['libextfunc.dylib'])
 
@@ -171,38 +171,40 @@ def test_wheel():
         fixed_wheel, stray_lib = _fixed_wheel(tmpdir)
         code, stdout, stderr = run_command(
             ['delocate-wheel', fixed_wheel])
-        _check_wheel(fixed_wheel, '.dylibs')
+        _check_wheel(fixed_wheel, 'fakepkg1.dylibs')
         # Make another copy to test another output directory
         fixed_wheel, stray_lib = _fixed_wheel(tmpdir)
         code, stdout, stderr = run_command(
-            ['delocate-wheel', '-L', 'dynlibs_dir', fixed_wheel])
-        _check_wheel(fixed_wheel, 'dynlibs_dir')
+            ['delocate-wheel', '-L', '.dynlibs_dir', fixed_wheel])
+        _check_wheel(fixed_wheel, 'fakepkg1.dynlibs_dir')
         # Another output directory
         fixed_wheel, stray_lib = _fixed_wheel(tmpdir)
         code, stdout, stderr = run_command(
             ['delocate-wheel', '-w', 'fixed', fixed_wheel])
-        _check_wheel(pjoin('fixed', basename(fixed_wheel)), '.dylibs')
+        _check_wheel(pjoin('fixed', basename(fixed_wheel)), 'fakepkg1.dylibs')
         # More than one wheel
-        shutil.copy2(fixed_wheel, 'wheel_copy.ext')
+        fixed_wheel_copy = basename(
+            fixed_wheel).replace('fakepkg1', 'wheel_copy')
+        shutil.copy2(fixed_wheel, fixed_wheel_copy)
         code, stdout, stderr = run_command(
-            ['delocate-wheel', '-w', 'fixed2', fixed_wheel, 'wheel_copy.ext'])
+            ['delocate-wheel', '-w', 'fixed2', fixed_wheel, fixed_wheel_copy])
         assert_equal(stdout,
                      ['Fixing: ' + name
-                      for name in (fixed_wheel, 'wheel_copy.ext')])
-        _check_wheel(pjoin('fixed2', basename(fixed_wheel)), '.dylibs')
-        _check_wheel(pjoin('fixed2', 'wheel_copy.ext'), '.dylibs')
+                      for name in (fixed_wheel, fixed_wheel_copy)])
+        _check_wheel(pjoin('fixed2', basename(fixed_wheel)), 'fakepkg1.dylibs')
+        _check_wheel(pjoin('fixed2', fixed_wheel_copy), 'wheel_copy.dylibs')
         # Verbose - single wheel
         code, stdout, stderr = run_command(
             ['delocate-wheel', '-w', 'fixed3', fixed_wheel, '-v'])
-        _check_wheel(pjoin('fixed3', basename(fixed_wheel)), '.dylibs')
+        _check_wheel(pjoin('fixed3', basename(fixed_wheel)), 'fakepkg1.dylibs')
         wheel_lines1 = ['Fixing: ' + fixed_wheel,
                         'Copied to package .dylibs directory:',
                         stray_lib]
         assert_equal(stdout, wheel_lines1)
         code, stdout, stderr = run_command(
             ['delocate-wheel', '-v', '--wheel-dir', 'fixed4',
-             fixed_wheel, 'wheel_copy.ext'])
-        wheel_lines2 = ['Fixing: wheel_copy.ext',
+             fixed_wheel, fixed_wheel_copy])
+        wheel_lines2 = ['Fixing: ' + fixed_wheel_copy,
                         'Copied to package .dylibs directory:',
                         stray_lib]
         assert_equal(stdout, wheel_lines1 + wheel_lines2)
@@ -213,17 +215,18 @@ def test_fix_wheel_dylibs():
     with InTemporaryDirectory() as tmpdir:
         # Default in-place fix
         fixed_wheel, stray_lib = _fixed_wheel(tmpdir)
-        _rename_module(fixed_wheel, 'module.other', 'test.whl')
-        shutil.copyfile('test.whl', 'test2.whl')
+        _rename_module(fixed_wheel, 'module.other', fixed_wheel)
         # Default is to look in all files and therefore fix
         code, stdout, stderr = run_command(
-            ['delocate-wheel', 'test.whl'])
-        _check_wheel('test.whl', '.dylibs')
+            ['delocate-wheel', fixed_wheel])
+        _check_wheel(fixed_wheel, 'fakepkg1.dylibs')
         # Can turn this off to only look in dynamic lib exts
+        fixed_wheel, stray_lib = _fixed_wheel(tmpdir)
+        _rename_module(fixed_wheel, 'module.other', fixed_wheel)
         code, stdout, stderr = run_command(
-            ['delocate-wheel', 'test2.whl', '-d'])
-        with InWheel('test2.whl'):  # No fix
-            assert_false(exists(pjoin('fakepkg1', '.dylibs')))
+            ['delocate-wheel', fixed_wheel, '-d'])
+        with InWheel(fixed_wheel):  # No fix
+            assert_false(exists(pjoin('fakepkg1.dylibs')))
 
 
 def test_fix_wheel_archs():
@@ -234,7 +237,7 @@ def test_fix_wheel_archs():
         # Fixed wheel, architectures are OK
         code, stdout, stderr = run_command(
             ['delocate-wheel', fixed_wheel, '-k'])
-        _check_wheel(fixed_wheel, '.dylibs')
+        _check_wheel(fixed_wheel, 'fakepkg1.dylibs')
         # Broken with one architecture removed still OK without checking
         # But if we check, raise error
         fmt_str = 'Fixing: {0}\n{1} needs arch {2} missing from {3}'
@@ -251,7 +254,7 @@ def _fix_break_fix(arch):
             _fix_break(arch)
             code, stdout, stderr = run_command(
                 ['delocate-wheel', fixed_wheel])
-            _check_wheel(fixed_wheel, '.dylibs')
+            _check_wheel(fixed_wheel, 'fakepkg1.dylibs')
             # Checked
             _fix_break(arch)
             code, stdout, stderr = bytes_runner.run_command(

delocate/tests/test_wheelies.py

@@ -94,34 +94,30 @@ def test_fix_plat():
                      {_rp(stray_lib): {dep_mod: stray_lib}})
         zip2dir(fixed_wheel, 'plat_pkg')
         assert_true(exists(pjoin('plat_pkg', 'fakepkg1')))
-        dylibs = pjoin('plat_pkg', 'fakepkg1', '.dylibs')
+        dylibs = pjoin('plat_pkg', 'fakepkg1.dylibs')
         assert_true(exists(dylibs))
         assert_equal(os.listdir(dylibs), ['libextfunc.dylib'])
         # New output name
         fixed_wheel, stray_lib = _fixed_wheel(tmpdir)
-        assert_equal(delocate_wheel(fixed_wheel, 'fixed_wheel.ext'),
+        new_wheel = basename(fixed_wheel).replace('fakepkg1', 'fixed_wheel')
+        assert_equal(delocate_wheel(fixed_wheel, new_wheel),
                      {_rp(stray_lib): {dep_mod: stray_lib}})
-        zip2dir('fixed_wheel.ext', 'plat_pkg1')
+        zip2dir(new_wheel, 'plat_pkg1')
         assert_true(exists(pjoin('plat_pkg1', 'fakepkg1')))
-        dylibs = pjoin('plat_pkg1', 'fakepkg1', '.dylibs')
+        dylibs = pjoin('plat_pkg1', 'fixed_wheel.dylibs')
         assert_true(exists(dylibs))
         assert_equal(os.listdir(dylibs), ['libextfunc.dylib'])
         # Test another lib output directory
+        new_wheel = basename(fixed_wheel).replace('fakepkg1', 'fixed_wheel2')
         assert_equal(delocate_wheel(fixed_wheel,
-                                    'fixed_wheel2.ext',
-                                    'dylibs_dir'),
+                                    new_wheel,
+                                    '.dylibs_dir'),
                      {_rp(stray_lib): {dep_mod: stray_lib}})
-        zip2dir('fixed_wheel2.ext', 'plat_pkg2')
+        zip2dir(new_wheel, 'plat_pkg2')
         assert_true(exists(pjoin('plat_pkg2', 'fakepkg1')))
-        dylibs = pjoin('plat_pkg2', 'fakepkg1', 'dylibs_dir')
+        dylibs = pjoin('plat_pkg2', 'fixed_wheel2.dylibs_dir')
         assert_true(exists(dylibs))
         assert_equal(os.listdir(dylibs), ['libextfunc.dylib'])
-        # Test check for existing output directory
-        assert_raises(DelocationError,
-                      delocate_wheel,
-                      fixed_wheel,
-                      'broken_wheel.ext',
-                      'subpkg')
         # Test that `wheel unpack` works
         fixed_wheel, stray_lib = _fixed_wheel(tmpdir)
         assert_equal(delocate_wheel(fixed_wheel),
@@ -130,7 +126,7 @@ def test_fix_plat():
         # Check that copied libraries have modified install_name_ids
         zip2dir(fixed_wheel, 'plat_pkg3')
         base_stray = basename(stray_lib)
-        the_lib = pjoin('plat_pkg3', 'fakepkg1', '.dylibs', base_stray)
+        the_lib = pjoin('plat_pkg3', 'fakepkg1.dylibs', base_stray)
         inst_id = DLC_PREFIX + 'fakepkg1/' + base_stray
         assert_equal(get_install_id(the_lib), inst_id)
 
@@ -143,7 +139,7 @@ def test_script_permissions():
         wheel_name = pjoin('wheels', whl_name)
         script_name = pjoin('fakepkg1-1.0.data', 'scripts', 'fakescript.py')
         exe_name = pjoin('fakepkg1', 'ascript')
-        lib_path = pjoin('fakepkg1', '.dylibs')
+        lib_path = 'fakepkg1.dylibs'
         mtimes = {}
         with InWheel(wheel_name):
             assert not isdir(lib_path)
@@ -169,21 +165,23 @@ def test_fix_plat_dylibs():
     # Check default and non-default searches for dylibs
     with InTemporaryDirectory() as tmpdir:
         fixed_wheel, stray_lib = _fixed_wheel(tmpdir)
-        _rename_module(fixed_wheel, 'module.other', 'test.whl')
+        new_wheel = basename(fixed_wheel).replace('fakepkg1', 'test')
+        _rename_module(fixed_wheel, 'module.other', new_wheel)
         # With dylibs-only - only analyze files with exts '.dylib', '.so'
-        assert_equal(delocate_wheel('test.whl', lib_filt_func='dylibs-only'),
+        assert_equal(delocate_wheel(new_wheel, lib_filt_func='dylibs-only'),
                      {})
         # With func that doesn't find the module
         func = lambda fn : fn.endswith('.so')
-        assert_equal(delocate_wheel('test.whl', lib_filt_func=func), {})
+        assert_equal(delocate_wheel(new_wheel, lib_filt_func=func), {})
         # Default - looks in every file
-        shutil.copyfile('test.whl', 'test2.whl')  # for following test
+        new_wheel2 = basename(fixed_wheel).replace('fakepkg1', 'test2')
+        shutil.copyfile(new_wheel, new_wheel2)  # for following test
         dep_mod = pjoin('fakepkg1', 'subpkg', 'module.other')
-        assert_equal(delocate_wheel('test.whl'),
+        assert_equal(delocate_wheel(new_wheel),
                      {realpath(stray_lib): {dep_mod: stray_lib}})
         # With func that does find the module
         func = lambda fn : fn.endswith('.other')
-        assert_equal(delocate_wheel('test2.whl', lib_filt_func=func),
+        assert_equal(delocate_wheel(new_wheel2, lib_filt_func=func),
                      {realpath(stray_lib): {dep_mod: stray_lib}})
 
 
@@ -285,7 +283,7 @@ def test_patch_wheel():
 def test_fix_rpath():
     # Test wheels which have an @rpath dependency
     # Also verifies the delocated libraries signature
-    with InTemporaryDirectory():
+    with InTemporaryDirectory() as tmpdir:
         # The module was set to expect its dependency in the libs/ directory
         os.symlink(DATA_PATH, 'libs')
 
@@ -296,10 +294,11 @@ def test_fix_rpath():
             dep_mod = 'fakepkg/subpkg/module2.so'
         dep_path = '@rpath/libextfunc_rpath.dylib'
 
+        out_wheel = pjoin(tmpdir, basename(RPATH_WHEEL))
         assert_equal(
-            delocate_wheel(RPATH_WHEEL, 'tmp.whl'),
+            delocate_wheel(RPATH_WHEEL, out_wheel),
             {stray_lib: {dep_mod: dep_path}},
         )
-        with InWheel('tmp.whl'):
+        with InWheel(out_wheel):
             check_call(['codesign', '--verify',
-                        'fakepkg/.dylibs/libextfunc_rpath.dylib'])
+                        'fakepkg.dylibs/libextfunc_rpath.dylib'])