Create .tar.gz package deterministically

Fixes #11981.

python/servo/command_base.py

@@ -6,13 +6,16 @@
 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
-
+import gzip
+import itertools
+import locale
 import os
 from os import path
 import contextlib
 import subprocess
 from subprocess import PIPE
 import sys
+import tarfile
 import platform
 
 import toml
@@ -33,6 +36,55 @@ def cd(new_path):
         os.chdir(previous_path)
 
 
+@contextlib.contextmanager
+def setlocale(name):
+    """Context manager for changing the current locale"""
+    saved_locale = locale.setlocale(locale.LC_ALL)
+    try:
+        yield locale.setlocale(locale.LC_ALL, name)
+    finally:
+        locale.setlocale(locale.LC_ALL, saved_locale)
+
+
+def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
+    """Create a .tar.gz archive in a deterministic (reproducible) manner.
+
+    See https://reproducible-builds.org/docs/archives/ for more details."""
+
+    def reset(tarinfo):
+        """Helper to reset owner/group and modification time for tar entries"""
+        tarinfo.uid = tarinfo.gid = 0
+        tarinfo.uname = tarinfo.gname = "root"
+        tarinfo.mtime = 0
+        return tarinfo
+
+    with cd(dir_to_archive):
+        current_dir = "."
+        file_list = [current_dir]
+        for root, dirs, files in os.walk(current_dir):
+            for name in itertools.chain(dirs, files):
+                file_list.append(os.path.join(root, name))
+
+        # Sort file entries with the fixed locale
+        with setlocale('C'):
+            file_list.sort(cmp=locale.strcoll)
+
+        # Package to a temporary file first because 'gzip' includes filename
+        # in the archive.
+        # TODO do this in a temporary folder after #11983 is fixed
+        temp_file = os.path.join(os.path.dirname(dest_archive),
+                                 'package-temp-file.tar.gz~')
+        # Use 'gzip' explicitly to set 'mtime' to 0
+        with gzip.GzipFile(temp_file, 'wb', mtime=0) as gzip_file:
+            with tarfile.open(fileobj=gzip_file, mode='w:') as tar_file:
+                for entry in file_list:
+                    arcname = entry
+                    if prepend_path is not None:
+                        arcname = os.path.normpath(os.path.join(prepend_path, arcname))
+                    tar_file.add(entry, filter=reset, recursive=False, arcname=arcname)
+        os.rename(temp_file, dest_archive)
+
+
 def host_triple():
     os_type = platform.system().lower()
     if os_type == "linux":

python/servo/package_commands.py

@@ -13,7 +13,6 @@
 import os.path as path
 import shutil
 import subprocess
-import tarfile
 
 from mach.registrar import Registrar
 from datetime import datetime
@@ -24,7 +23,13 @@
     Command,
 )
 
-from servo.command_base import CommandBase, cd, BuildNotFound, is_macosx
+from servo.command_base import (
+    archive_deterministically,
+    BuildNotFound,
+    cd,
+    CommandBase,
+    is_macosx,
+)
 from servo.post_build_commands import find_dep_path_newest
 
 
@@ -156,7 +161,10 @@ def package(self, release=False, dev=False, android=None, debug=False, debugger=
         else:
             dir_to_package = '/'.join(binary_path.split('/')[:-1])
             dir_to_root = '/'.join(binary_path.split('/')[:-3])
-            shutil.copytree(dir_to_root + '/resources', dir_to_package + '/resources')
+            resources_dir = dir_to_package + '/resources'
+            if os.path.exists(resources_dir):
+                delete(resources_dir)
+            shutil.copytree(dir_to_root + '/resources', resources_dir)
             browserhtml_path = find_dep_path_newest('browserhtml', binary_path)
             if browserhtml_path is None:
                 print("Could not find browserhtml package; perhaps you haven't built Servo.")
@@ -185,9 +193,9 @@ def package(self, release=False, dev=False, android=None, debug=False, debugger=
             time = datetime.utcnow().replace(microsecond=0).isoformat()
             time = time.replace(':', "-")
             tar_path += time + "-servo-tech-demo.tar.gz"
-            with tarfile.open(tar_path, "w:gz") as tar:
-                # arcname is to add by relative rather than absolute path
-                tar.add(dir_to_package, arcname='servo/')
+
+            archive_deterministically(dir_to_package, tar_path, prepend_path='servo/')
+
             print("Packaged Servo into " + tar_path)
 
     @Command('install',