Two enhancements to PEXBuilder:

1. Allow it to optionally copy instead of hard-linking files into the
   environment.  This is useful for creating long-running environments
   without corrupting them by modifying the underlying files from under
   them.

2. Make precompiling .py sources optional, as mentioned in a TODO.
   Again, in long-running environments it may make more sense to save
   this time when building the environment and pay it when actually
   using the environment.

Testing Done:
Added tests for the new functionality.

CI passes: https://travis-ci.org/pantsbuild/pex/builds/66359462

Bugs closed: 123

Reviewed at https://rbcommons.com/s/twitter/r/2346/

CHANGES.rst

@@ -2,6 +2,13 @@
 CHANGES
 =======
 
+----------
+1.0.1.dev1
+----------
+
+* Allow PEXBuilder to optionally copy files into the PEX environment instead of hard-linking them.
+* Allow PEXBuilder to optionally skip precompilation of .py files into .pyc files.
+
 -----
 1.0.0
 -----

pex/pex_builder.py

@@ -55,7 +55,8 @@ class InvalidExecutableSpecification(Error): pass
 
   BOOTSTRAP_DIR = ".bootstrap"
 
-  def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, preamble=None):
+  def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, preamble=None,
+               copy=False):
     """Initialize a pex builder.
 
     :keyword path: The path to write the PEX as it is built.  If ``None`` is specified,
@@ -67,6 +68,8 @@ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, prea
     :keyword preamble: If supplied, execute this code prior to bootstrapping this PEX
       environment.
     :type preamble: str
+    :keyword copy: If False, attempt to create the pex environment via hard-linking, falling
+                   back to copying across devices. If True, always copy.
 
     .. versionchanged:: 0.8
       The temporary directory created when ``path`` is not specified is now garbage collected on
@@ -79,6 +82,7 @@ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, prea
     self._shebang = self._interpreter.identity.hashbang()
     self._logger = logging.getLogger(__name__)
     self._preamble = to_bytes(preamble or '')
+    self._copy = copy
     self._distributions = set()
 
   def _ensure_unfrozen(self, name='Operation'):
@@ -123,17 +127,17 @@ def info(self, value):
     self._ensure_unfrozen('Changing PexInfo')
     self._pex_info = value
 
-  # TODO(wickman) Add option to not compile/marshal sources.
-  def add_source(self, filename, env_filename):
+  def add_source(self, filename, env_filename, precompile_python=True):
     """Add a source to the PEX environment.
 
     :param filename: The source filename to add to the PEX.
-    :param env_filename: The destination filename in the PEX.  This path
+    :param env_filename: The destination filename in the PEX.
+    :param compile_python: If True, precompile .py files into .pyc files.
       must be a relative path.
     """
     self._ensure_unfrozen('Adding source')
-    self._chroot.link(filename, env_filename, "source")
-    if filename.endswith('.py'):
+    self._copy_or_link(filename, env_filename, 'source')
+    if precompile_python and filename.endswith('.py'):
       env_filename_pyc = os.path.splitext(env_filename)[0] + '.pyc'
       with open(filename) as fp:
         pyc_object = CodeMarshaller.from_py(fp.read(), env_filename)
@@ -147,7 +151,7 @@ def add_resource(self, filename, env_filename):
       must be a relative path.
     """
     self._ensure_unfrozen('Adding a resource')
-    self._chroot.link(filename, env_filename, "resource")
+    self._copy_or_link(filename, env_filename, "resource")
 
   def add_requirement(self, req):
     """Add a requirement to the PEX environment.
@@ -178,7 +182,7 @@ def set_executable(self, filename, env_filename=None):
     if self._chroot.get("executable"):
       raise self.InvalidExecutableSpecification(
           "Setting executable on a PEXBuilder that already has one!")
-    self._chroot.link(filename, env_filename, "executable")
+    self._copy_or_link(filename, env_filename, "executable")
     entry_point = env_filename
     entry_point.replace(os.path.sep, '.')
     self._pex_info.entry_point = entry_point.rpartition('.')[0]
@@ -242,7 +246,7 @@ def _add_dist_dir(self, path, dist_name):
         filename = os.path.join(root, f)
         relpath = os.path.relpath(filename, path)
         target = os.path.join(self._pex_info.internal_cache, dist_name, relpath)
-        self._chroot.link(filename, target)
+        self._copy_or_link(filename, target)
     return CacheHelper.dir_hash(path)
 
   def _add_dist_zip(self, path, dist_name):
@@ -326,6 +330,12 @@ def _prepare_main(self):
     self._chroot.write(self._preamble + b'\n' + BOOTSTRAP_ENVIRONMENT,
         '__main__.py', label='main')
 
+  def _copy_or_link(self, src, dst, label=None):
+    if self._copy:
+      self._chroot.copy(src, dst, label)
+    else:
+      self._chroot.link(src, dst, label)
+
   # TODO(wickman) Ideally we unqualify our setuptools dependency and inherit whatever is
   # bundled into the environment so long as it is compatible (and error out if not.)
   #

tests/test_pex_builder.py

@@ -2,6 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import os
+import stat
 import zipfile
 from contextlib import closing
 
@@ -65,3 +66,44 @@ def test_pex_builder_shebang():
     expected_preamble = b'#!foobar\n'
     with open(target, 'rb') as fp:
       assert fp.read(len(expected_preamble)) == expected_preamble
+
+
+def test_pex_builder_compilation():
+  with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
+    src = os.path.join(td1, 'exe.py')
+    with open(src, 'w') as fp:
+      fp.write(exe_main)
+
+    def build_and_check(path, precompile):
+      pb = PEXBuilder(path)
+      pb.add_source(src, 'exe.py', precompile_python=precompile)
+      pyc_exists = os.path.exists(os.path.join(path, 'exe.pyc'))
+      if precompile:
+        assert pyc_exists
+      else:
+        assert not pyc_exists
+
+    build_and_check(td2, False)
+    build_and_check(td3, True)
+
+
+def test_pex_builder_copy_or_link():
+  with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
+    src = os.path.join(td1, 'exe.py')
+    with open(src, 'w') as fp:
+      fp.write(exe_main)
+
+    def build_and_check(path, copy):
+      pb = PEXBuilder(path, copy=copy)
+      pb.add_source(src, 'exe.py')
+
+      s1 = os.stat(src)
+      s2 = os.stat(os.path.join(path, 'exe.py'))
+      is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
+      if copy:
+        assert not is_link
+      else:
+        assert is_link
+
+    build_and_check(td2, False)
+    build_and_check(td3, True)