Add build scripts and documentation for building releases

Adds builds scripts that are used to build releases deterministically.
Also adds documentation that explains the release process and what
the build scripts do.

.python-version

@@ -1 +1 @@
-3.5.6
+3.6.8

contrib/README.md

@@ -0,0 +1,21 @@
+# Assorted tools
+
+## `build_bin.sh`
+
+Creates a virtualenv with the locked dependencies using Poetry. Then uses pyinstaller to create a standalone binary for the OS type currently running.
+
+## `build_dist.sh`
+
+Creates a virtualenv with the locked dependencies using Poetry. Then uses Poetry to produce deterministic builds of the wheel and sdist for upload to PyPi
+
+`faketime` needs to be installed
+
+## `build_wine.sh`
+
+Sets up Wine with Python and everything needed to build Windows binaries. Creates a virtualenv with the locked dependencies using Poetry. Then uses pyinstaller to create a standalone Windows binary.
+
+`wine` needs to be installed
+
+## `generate_setup.sh`
+
+Builds the source distribution and extracts the setup.py from it.

contrib/build.Dockerfile

@@ -0,0 +1,48 @@
+FROM debian:stretch-slim
+
+SHELL ["/bin/bash", "-c"]
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt-get install -y \
+    apt-transport-https \
+    git \
+    make \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    wget \
+    curl \
+    llvm \
+    libncurses5-dev \
+    xz-utils \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
+    libusb-1.0-0-dev \
+    libudev-dev \
+    faketime
+
+RUN curl https://pyenv.run | bash
+ENV PATH="/root/.pyenv/bin:$PATH"
+COPY contrib/reproducible-python.diff /opt/reproducible-python.diff
+ENV PYTHON_CONFIGURE_OPTS="--enable-shared"
+ENV BUILD_DATE="Jan  1 2019"
+ENV BUILD_TIME="00:00:00"
+RUN eval "$(pyenv init -)" && eval "$(pyenv virtualenv-init -)" && cat /opt/reproducible-python.diff | pyenv install -kp 3.6.8
+
+RUN dpkg --add-architecture i386 
+RUN wget -nc https://dl.winehq.org/wine-builds/winehq.key
+RUN apt-key add winehq.key
+RUN echo "deb https://dl.winehq.org/wine-builds/debian/ stretch main" >> /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get install --install-recommends -y \
+    wine-stable-amd64 \
+    wine-stable-i386 \
+    wine-stable \
+    winehq-stable \
+    p7zip-full

contrib/build_bin.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+# Script for building standalone binary releases deterministically
+
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+pip install -U pip
+pip install poetry
+
+# Setup poetry and install the dependencies
+poetry install
+
+# We now need to remove debugging symbols and build id from the hidapi SO file
+so_dir=`dirname $(dirname $(poetry run which python))`/lib/python3.6/site-packages
+find ${so_dir} -name '*.so' -type f -execdir strip '{}' \;
+if [[ $OSTYPE != *"darwin"* ]]; then
+    find ${so_dir} -name '*.so' -type f -execdir strip -R .note.gnu.build-id '{}' \;
+fi
+
+# We also need to change the timestamps of all of the base library files
+lib_dir=`pyenv root`/versions/3.6.8/lib/python3.6
+TZ=UTC find ${lib_dir} -name '*.py' -type f -execdir touch -t "201901010000.00" '{}' \;
+
+# Make the standalone binary
+export PYTHONHASHSEED=42
+poetry run pyinstaller hwi.spec
+unset PYTHONHASHSEED

contrib/build_dist.sh

@@ -0,0 +1,15 @@
+#! /bin/bash
+# Script for building pypi distribution archives deterministically
+
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+pip install -U pip
+pip install poetry
+
+# Setup poetry and install the dependencies
+poetry install
+
+# Make the distribution archives for pypi
+poetry build -f wheel
+# faketime is needed to make sdist detereministic
+TZ=UTC faketime -f "2019-01-01 00:00:00" poetry build -f sdist

contrib/build_wine.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Script which sets up Wine and builds the Windows standalone binary
+
+set -e
+
+PYTHON_VERSION=3.6.8
+
+PYTHON_FOLDER="python3"
+PYHOME="c:/$PYTHON_FOLDER"
+PYTHON="wine $PYHOME/python.exe -OO -B"
+
+LIBUSB_URL=https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.7z
+LIBUSB_HASH="671f1a420757b4480e7fadc8313d6fb3cbb75ca00934c417c1efa6e77fb8779b"
+
+wine 'wineboot'
+
+# Install Python
+# Get the PGP keys
+wget -N -c "https://www.python.org/static/files/pubkeys.txt"
+gpg --import pubkeys.txt
+rm pubkeys.txt
+
+# Install python components
+for msifile in core dev exe lib pip tools; do
+    wget -N -c "https://www.python.org/ftp/python/$PYTHON_VERSION/amd64/${msifile}.msi"
+    wget -N -c "https://www.python.org/ftp/python/$PYTHON_VERSION/amd64/${msifile}.msi.asc"
+    gpg --verify "${msifile}.msi.asc" "${msifile}.msi"
+    wine msiexec /i "${msifile}.msi" /qb TARGETDIR=$PYHOME
+    rm $msifile.msi*
+done
+
+# Get libusb
+wget -N -c -O libusb.7z "$LIBUSB_URL"
+echo "$LIBUSB_HASH  libusb.7z" | sha256sum -c
+7za x -olibusb libusb.7z -aoa
+cp libusb/MS64/dll/libusb-1.0.dll ~/.wine/drive_c/python3/
+rm -r libusb*
+
+# Update pip
+$PYTHON -m pip install -U pip
+
+# Install Poetry and things needed for pyinstaller
+$PYTHON -m pip install poetry
+
+# We also need to change the timestamps of all of the base library files
+lib_dir=~/.wine/drive_c/python3/Lib
+TZ=UTC find ${lib_dir} -name '*.py' -type f -execdir touch -t "201901010000.00" '{}' \;
+
+# Do the build
+POETRY="wine $PYHOME/Scripts/poetry.exe"
+$POETRY install -E windist
+export PYTHONHASHSEED=42
+$POETRY run pyinstaller hwi.spec
+unset PYTHONHASHSEED

contrib/generate_setup.sh

@@ -0,0 +1,32 @@
+#! /bin/bash
+# Generates the setup.py file
+
+set -e
+
+# Setup poetry and install the dependencies
+poetry install
+
+# Build the source distribution
+poetry build -f sdist
+
+# Extract setup.py from the distribution
+unset -v tarball
+for file in dist/*
+do
+    if [[ $file -nt $tarball && $file == *".tar.gz" ]]
+    then
+        tarball=$file
+    fi
+done
+unset -v toextract
+for file in `tar -tf $tarball`
+do
+    if [[ $file == *"setup.py" ]]
+    then
+        toextract=$file
+    fi
+done
+tar -xf $tarball $toextract
+mv $toextract .
+dir=`echo $toextract | cut -f1 -d"/"`
+rm -r $dir

contrib/reproducible-python.diff

@@ -0,0 +1,13 @@
+# DP: Build getbuildinfo.o with DATE/TIME values when defined
+
+--- Makefile.pre.in
++++ Makefile.pre.in
+@@ -741,6 +741,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
+ 	      -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \
+ 	      -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \
+ 	      -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \
++	      $(if $(BUILD_DATE),-DDATE='"$(BUILD_DATE)"') \
++	      $(if $(BUILD_TIME),-DTIME='"$(BUILD_TIME)"') \
+ 	      -o $@ $(srcdir)/Modules/getbuildinfo.c
+
+ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile

docs/release-process.md

@@ -0,0 +1,51 @@
+# Release Process
+
+1. Bump version number in `pyproject.toml`, generate the setup.py file, and git tag release
+2. Build distribution archives for PyPi with `contrib/build_dist.sh`
+3. For MacOS and Linux, use `contrib/build_bin.sh`. This needs to be run on a MacOS machine for the MacOS binary and on a Linux machine for the linux one.
+4. For Windows, use `contrib/build_wine.sh` to build the Windows binary using wine
+5. Upload distribution archives to PyPi
+6. Upload distribution archives and standalone binaries to Github
+
+## Deterministic builds with Docker
+
+Create the docker image:
+
+```
+docker build --no-cache -t hwi-builder -f contrib/build.Dockerfile .
+```
+
+Build everything
+
+```
+docker run -it --name hwi-builder -v $PWD:/opt/hwi --rm  --workdir /opt/hwi hwi-builder /bin/bash -c "contrib/build_bin.sh && contrib/build_dist.sh && contrib/build_wine.sh"
+```
+
+## Building macOS binary
+
+Note that the macOS build is non-deterministic.
+
+First install [pyenv](https://github.com/pyenv/pyenv) using whichever method you prefer.
+
+Then a deterministic build of Python 3.6.8 needs to be installed. This can be done with the patch in `contrib/reproducible-python.diff`. First `cd` into HWI's source tree. Then use:
+
+```
+cat contrib/reproducible-python.diff | PYTHON_CONFIGURE_OPTS="--enable-framework" BUILD_DATE="Jan  1 2019" BUILD_TIME="00:00:00" pyenv install -kp 3.6.8
+```
+
+Make sure that python 3.6.8 is active
+
+```
+$ python --version
+Python 3.6.8
+```
+
+Now install [Poetry](https://github.com/sdispater/poetry) with `pip install poetry`
+
+Additional dependencies can be installed with:
+
+```
+brew install libusb
+```
+
+Build the binaries by using `contrib/build_bin.sh`.