Add new --no-use-system-time flag to use a deterministic timestamp in built PEX

pex/bin/pex.py

@@ -302,6 +302,17 @@ def configure_clp_pex_options(parser):
            'likely will not be reproducible, meaning that if you were to run `./pex -o` with the '
            'same inputs then the new pex would not be byte-for-byte identical to the original.')
 
+  group.add_option(
+      '--use-system-time', '--no-use-system-time',
+      dest='use_system_time',
+      default=True,
+      action='callback',
+      callback=parse_bool,
+      help='Use the current system time to generate timestamps for the new pex. Otherwise, Pex '
+           'will use midnight on January 1, 1980. By using system time, the generated pex '
+           'will not be reproducible, meaning that if you were to run `./pex -o` with the '
+           'same inputs then the new pex would not be byte-for-byte identical to the original.')
+
   parser.add_option_group(group)
 
 
@@ -694,7 +705,11 @@ def main(args=None):
       log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
       tmp_name = options.pex_name + '~'
       safe_delete(tmp_name)
-      pex_builder.build(tmp_name, bytecode_compile=options.compile)
+      pex_builder.build(
+        tmp_name,
+        bytecode_compile=options.compile,
+        deterministic_timestamp=not options.use_system_time
+      )
       os.rename(tmp_name, options.pex_name)
     else:
       if not _compatible_with_current_platform(options.platforms):

pex/common.py

@@ -15,9 +15,17 @@
 import threading
 import zipfile
 from collections import defaultdict
+from datetime import datetime
 from uuid import uuid4
 
 
+# We use the start of MS-DOS time, which is what zipfiles use (see section 4.4.6 of
+# https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT).
+DETERMINISTIC_DATETIME = datetime(
+  year=1980, month=1, day=1, hour=0, minute=0, second=0, tzinfo=None
+)
+
+
 def die(msg, exit_code=1):
   print(msg, file=sys.stderr)
   sys.exit(exit_code)
@@ -82,6 +90,34 @@ def teardown(self):
 class PermPreservingZipFile(zipfile.ZipFile, object):
   """A ZipFile that works around https://bugs.python.org/issue15795"""
 
+  @classmethod
+  def zip_info_from_file(cls, filename, arcname=None, date_time=None):
+    """Construct a ZipInfo for a file on the filesystem.
+
+    Usually this is provided directly as a method of ZipInfo, but it is not implemented in Python
+    2.7 so we re-implement it here. The main divergance we make from the original is adding a
+    parameter for the datetime (a time.struct_time), which allows us to use a deterministic
+    timestamp. See https://github.com/python/cpython/blob/master/Lib/zipfile.py#L495."""
+    st = os.stat(filename)
+    isdir = stat.S_ISDIR(st.st_mode)
+    if arcname is None:
+      arcname = filename
+    arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
+    while arcname[0] in (os.sep, os.altsep):
+      arcname = arcname[1:]
+    if isdir:
+      arcname += '/'
+    if date_time is None:
+      date_time = time.localtime(st.st_mtime)
+    zinfo = zipfile.ZipInfo(filename=arcname, date_time=date_time[:6])
+    zinfo.external_attr = (st.st_mode & 0xFFFF) << 16  # Unix attributes
+    if isdir:
+      zinfo.file_size = 0
+      zinfo.external_attr |= 0x10  # MS-DOS directory flag
+    else:
+      zinfo.file_size = st.st_size
+    return zinfo
+
   def _extract_member(self, member, targetpath, pwd):
     result = super(PermPreservingZipFile, self)._extract_member(member, targetpath, pwd)
     info = member if isinstance(member, zipfile.ZipInfo) else self.getinfo(member)
@@ -98,7 +134,7 @@ def _chmod(self, info, path):
 
 @contextlib.contextmanager
 def open_zip(path, *args, **kwargs):
-  """A contextmanager for zip files.  Passes through positional and kwargs to zipfile.ZipFile."""
+  """A contextmanager for zip files. Passes through positional and kwargs to zipfile.ZipFile."""
   with contextlib.closing(PermPreservingZipFile(path, *args, **kwargs)) as zip:
     yield zip
 
@@ -363,7 +399,15 @@ def __str__(self):
   def delete(self):
     shutil.rmtree(self.chroot)
 
-  def zip(self, filename, mode='w'):
+  def zip(self, filename, mode='w', deterministic_timestamp=False):
     with open_zip(filename, mode) as zf:
       for f in sorted(self.files()):
-        zf.write(os.path.join(self.chroot, f), arcname=f, compress_type=zipfile.ZIP_DEFLATED)
+        full_path = os.path.join(self.chroot, f)
+        zinfo = zf.zip_info_from_file(
+            filename=full_path,
+            arcname=f,
+            date_time=DETERMINISTIC_DATETIME.timetuple() if deterministic_timestamp else None
+        )
+        with open(full_path, 'rb') as open_f:
+          data = open_f.read()
+        zf.writestr(zinfo, data, compress_type=zipfile.ZIP_DEFLATED)

pex/pex_builder.py

@@ -428,11 +428,12 @@ def freeze(self, bytecode_compile=True):
       self._precompile_source()
     self._frozen = True
 
-  def build(self, filename, bytecode_compile=True):
+  def build(self, filename, bytecode_compile=True, deterministic_timestamp=False):
     """Package the PEX into a zipfile.
 
     :param filename: The filename where the PEX should be stored.
     :param bytecode_compile: If True, precompile .py files into .pyc files.
+    :param deterministic_timestamp: If True, will use our hardcoded time for zipfile timestamps.
 
     If the PEXBuilder is not yet frozen, it will be frozen by ``build``.  This renders the
     PEXBuilder immutable.
@@ -450,7 +451,7 @@ def build(self, filename, bytecode_compile=True):
     with open(filename + '~', 'ab') as pexfile:
       assert os.path.getsize(pexfile.name) == 0
       pexfile.write(to_bytes('%s\n' % self._shebang))
-    self._chroot.zip(filename + '~', mode='a')
+    self._chroot.zip(filename + '~', mode='a', deterministic_timestamp=deterministic_timestamp)
     if os.path.exists(filename):
       os.unlink(filename)
     os.rename(filename + '~', filename)

tests/test_integration.py

@@ -1353,7 +1353,7 @@ def assert_reproducible_build(args):
     # https://tox.readthedocs.io/en/latest/example/basic.html#special-handling-of-pythonhashseed.
     def create_pex(path, seed):
       result = run_pex_command(
-        args + ['-o', path, '--no-compile'],
+        args + ['-o', path, '--no-compile', '--no-use-system-time'],
         env=make_env(PYTHONHASHSEED=seed)
       )
       result.assert_success()
@@ -1381,7 +1381,6 @@ def create_pex(path, seed):
     assert filecmp.cmp(pex1, pex2, shallow=False)
 
 
-@pytest.mark.skip("Acceptance test for landing https://github.com/pantsbuild/pex/issues/716.")
 def test_reproducible_build_no_args():
   assert_reproducible_build([])
 
@@ -1397,7 +1396,6 @@ def test_reproducible_build_sdist_requirements():
   assert_reproducible_build(['pycparser==2.19', '--no-build'])
 
 
-@pytest.mark.skip("Acceptance test for landing https://github.com/pantsbuild/pex/issues/716.")
 def test_reproducible_build_m_flag():
   assert_reproducible_build(['-m', 'pydoc'])
 
@@ -1416,11 +1414,9 @@ def test_reproducible_build_c_flag():
     assert_reproducible_build([project_dir, '-c', 'my_app'])
 
 
-@pytest.mark.skip("Acceptance test for landing https://github.com/pantsbuild/pex/issues/716.")
 def test_reproducible_build_python_flag():
   assert_reproducible_build(['--python=python2.7'])
 
 
-@pytest.mark.skip("Acceptance test for landing https://github.com/pantsbuild/pex/issues/716.")
 def test_reproducible_build_python_shebang_flag():
   assert_reproducible_build(['--python-shebang=/usr/bin/python'])

tests/test_pex_builder.py

@@ -3,6 +3,7 @@
 
 import os
 import stat
+import zipfile
 
 import pytest
 
@@ -177,3 +178,12 @@ def build_and_check(path, copy):
 
     build_and_check(td2, False)
     build_and_check(td3, True)
+
+
+def test_pex_builder_deterministic_timestamp():
+  pb = PEXBuilder()
+  with temporary_dir() as td:
+    target = os.path.join(td, 'foo.pex')
+    pb.build(target, deterministic_timestamp=True)
+    with zipfile.ZipFile(target) as zf:
+      assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist())