From eafc5a95799b147f955c9df9121b420242272081 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Thu, 31 Dec 2020 20:39:22 +0100 Subject: [PATCH] modifications for reproducible builds * pass SOURCE_DATE_EPOCH etc. through to build env * clean tar entries & gzip mtime * utime() files before zipping * sort file lists before tar/zip * call zip w/ -X * make private_version deterministic * python3: add reproducible-buildinfo.diff patch (from Debian) Caveats: * still requires identical build path and umask * arm64-v8a requires ndk >= r22 to build completely reproducibly - due to a clang/llvm bug Makefile Example: NB: you need to export something like this in your build environment to get reproducible builds. export LC_ALL := C export TZ := UTC export SOURCE_DATE_EPOCH := $(shell git log -1 --pretty=%ct) export PYTHONHASHSEED := $(SOURCE_DATE_EPOCH) export BUILD_DATE := $(shell LC_ALL=C TZ=UTC date +'%b %e %Y' -d @$(SOURCE_DATE_EPOCH)) export BUILD_TIME := $(shell LC_ALL=C TZ=UTC date +'%H:%M:%S' -d @$(SOURCE_DATE_EPOCH)) --- pythonforandroid/archs.py | 6 ++++ .../bootstraps/common/build/build.py | 29 ++++++++++++++++--- pythonforandroid/recipes/python3/__init__.py | 11 +++++-- .../patches/reproducible-buildinfo.diff | 13 +++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 pythonforandroid/recipes/python3/patches/reproducible-buildinfo.diff diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index 160464f243..aa661fecce 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -234,6 +234,12 @@ def get_env(self, with_flags_in_cc=True): env['PATH'] = environ['PATH'] + # for reproducible builds + if 'SOURCE_DATE_EPOCH' in environ: + for k in 'LC_ALL TZ SOURCE_DATE_EPOCH PYTHONHASHSEED BUILD_DATE BUILD_TIME'.split(): + if k in environ: + env[k] = environ[k] + return env diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 7d54a6f02b..92e8db81d2 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 +from gzip import GzipFile +import hashlib import json from os.path import ( dirname, join, isfile, realpath, relpath, split, exists, basename ) -from os import listdir, makedirs, remove +from os import environ, listdir, makedirs, remove import os import shlex import shutil @@ -161,6 +163,13 @@ def select(fn): return False return not is_blacklist(fn) + def clean(tinfo): + """cleaning function (for reproducible builds)""" + tinfo.uid = tinfo.gid = 0 + tinfo.uname = tinfo.gname = '' + tinfo.mtime = 0 + return tinfo + # get the files and relpath file of all the directory we asked for files = [] for sd in source_dirs: @@ -168,9 +177,11 @@ def select(fn): compile_dir(sd, optimize_python=optimize_python) files += [(x, relpath(realpath(x), sd)) for x in listfiles(sd) if select(x)] + files.sort() # deterministic # create tar.gz of thoses files - tf = tarfile.open(tfn, 'w:gz', format=tarfile.USTAR_FORMAT) + gf = GzipFile(tfn, 'wb', mtime=0) # deterministic + tf = tarfile.open(None, 'w', gf, format=tarfile.USTAR_FORMAT) dirs = [] for fn, afn in files: dn = dirname(afn) @@ -189,8 +200,9 @@ def select(fn): tf.addfile(tinfo) # put the file - tf.add(fn, afn) + tf.add(fn, afn, filter=clean) tf.close() + gf.close() def compile_dir(dfn, optimize_python=True): @@ -521,9 +533,18 @@ def make_package(args): versioned_name=versioned_name) # String resources: + timestamp = time.time() + if 'SOURCE_DATE_EPOCH' in environ: + # for reproducible builds + timestamp = int(environ['SOURCE_DATE_EPOCH']) + private_version = "{} {} {}".format( + args.version, + args.numeric_version, + timestamp + ) render_args = { "args": args, - "private_version": str(time.time()) + "private_version": hashlib.sha1(private_version.encode()).hexdigest() } if get_bootstrap_name() == "sdl2": render_args["url_scheme"] = url_scheme diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index a904646cc4..1e2d53b560 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -3,7 +3,7 @@ import subprocess from multiprocessing import cpu_count -from os import environ +from os import environ, utime from os.path import dirname, exists, join from pathlib import Path from shutil import copy2 @@ -62,6 +62,7 @@ class Python3Recipe(TargetPythonRecipe): patches = [ 'patches/pyconfig_detection.patch', + 'patches/reproducible-buildinfo.diff', # Python 3.7.1 ('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")), @@ -387,8 +388,14 @@ def create_python_bundle(self, dirn, arch): with current_directory(join(self.get_build_dir(arch.arch), 'Lib')): stdlib_filens = list(walk_valid_filens( '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist)) + if 'SOURCE_DATE_EPOCH' in environ: + # for reproducible builds + stdlib_filens.sort() + timestamp = int(environ['SOURCE_DATE_EPOCH']) + for filen in stdlib_filens: + utime(filen, (timestamp, timestamp)) info("Zip {} files into the bundle".format(len(stdlib_filens))) - shprint(sh.zip, stdlib_zip, *stdlib_filens) + shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens) # copy the site-packages into place ensure_dir(join(dirn, 'site-packages')) diff --git a/pythonforandroid/recipes/python3/patches/reproducible-buildinfo.diff b/pythonforandroid/recipes/python3/patches/reproducible-buildinfo.diff new file mode 100644 index 0000000000..807d180a68 --- /dev/null +++ b/pythonforandroid/recipes/python3/patches/reproducible-buildinfo.diff @@ -0,0 +1,13 @@ +# DP: Build getbuildinfo.o with DATE/TIME values when defined + +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -785,6 +785,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ + -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ + -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ + -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ ++ $(if $(BUILD_DATE),-DDATE='"$(BUILD_DATE)"') \ ++ $(if $(BUILD_TIME),-DTIME='"$(BUILD_TIME)"') \ + -o $@ $(srcdir)/Modules/getbuildinfo.c + + Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile