diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 541ddd23f2..deb273963e 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos export HOST=x86_64-apple-darwin18 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools libtinfo5 python3-dev python3-setuptools xorriso" export XCODE_VERSION=12.1 diff --git a/ci/test/00_setup_env_win32.sh b/ci/test/00_setup_env_win32.sh index 725867eed4..016b07d074 100755 --- a/ci/test/00_setup_env_win32.sh +++ b/ci/test/00_setup_env_win32.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win32 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win32 (bionic is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win32 export HOST=i686-w64-mingw32 export PACKAGES="python3 nsis g++-mingw-w64-i686 wine-binfmt wine32" export RUN_UNIT_TESTS=true diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 74e9540199..a8db572aa8 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win64 export HOST=x86_64-w64-mingw32 export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" export GOAL="" diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index a2d0df1a8e..721dd02693 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -97,18 +97,14 @@ Perform basic security checks on a series of executables. symbol-check.py =============== -A script to check that the executables produced by gitian only contain -certain symbols and are only linked against allowed libraries. +A script to check that release executables only contain certain symbols and are only linked against +allowed libraries. For Linux this means checking for allowed gcc, glibc and libstdc++ version symbols. This makes sure they are still compatible with the minimum supported distribution versions. For macOS and Windows we check that the executables are only linked against libraries we allow. -Example usage after a gitian build: - - find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py - If no errors occur the return value will be 0 and the output will be empty. If there are any errors the return value will be 1 and output like this will be printed: diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py old mode 100755 new mode 100644 index 3ae8cab500..05b57b6967 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -6,8 +6,10 @@ A script to check that the executables produced by gitian only contain certain symbols and are only linked against allowed libraries. -Example usage: +NOTE: gitian is no longer used and this script is not currently being maintained. It may be used in the future +with Guix deterministic builds and so will be left in place for the time being. +Example usage: find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import subprocess @@ -124,7 +126,6 @@ class CPPFilt(object): ''' Demangle C++ symbol names. - Use a pipe to the 'c++filt' command. ''' def __init__(self): diff --git a/contrib/gitian-build.sh b/contrib/gitian-build.sh deleted file mode 100755 index 161d289f92..0000000000 --- a/contrib/gitian-build.sh +++ /dev/null @@ -1,392 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016 The Gridcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or https://opensource.org/licenses/mit-license.php. - -# What to do -sign=true -verify=false -build=false - -# Systems to build -linux=true -windows=true -#osx=true -osx=false - -# Other Basic variables -SIGNER=TheCharlatan -VERSION=3.5.9.2 -commit=false -url=https://github.com/TheCharlatan/Gridcoin-Research/ -proc=2 -mem=2000 -lxc=true -osslTarUrl=http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz -osslPatchUrl=https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch -scriptName=$(basename -- "$0") -signProg="gpg --detach-sign" -commitFiles=true - -# Help Message -read -d '' usage <<- EOF -Usage: $scriptName [-c|u|v|b|s|B|o|h|j|m|] signer version - -Run this script from the directory containing the gridcoin, gitian-builder, gitian.sigs, and gridcoin-detached-sigs. - -Arguments: -signer GPG signer to sign each build assert file -version Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified - -Options: --c|--commit Indicate that the version argument is for a commit or branch --u|--url Specify the URL of the repository. Default is https://github.com/gridcoin/Gridcoin-Research --v|--verify Verify the gitian build --b|--build Do a gitian build --s|--sign Make signed binaries for Windows and Mac OSX --B|--buildsign Build both signed and unsigned binaries --o|--os Specify which Operating Systems the build is for. Default is lwx. l for linux, w for windows, x for osx --j Number of processes to use. Default 2 --m Memory to allocate in MiB. Default 2000 ---kvm Use KVM instead of LXC ---setup Setup the gitian building environment. Uses KVM. If you want to use lxc, use the --lxc option. Only works on Debian-based systems (Ubuntu, Debian) ---detach-sign Create the assert file for detached signing. Will not commit anything. ---no-commit Do not commit anything to git --h|--help Print this help message -EOF - -# Get options and arguments -while :; do - case $1 in - # Verify - -v|--verify) - verify=true - ;; - # Build - -b|--build) - build=true - ;; - # Sign binaries - -s|--sign) - sign=true - ;; - # Build then Sign - -B|--buildsign) - sign=true - build=true - ;; - # PGP Signer - -S|--signer) - if [ -n "$2" ] - then - SIGNER=$2 - shift - else - echo 'Error: "--signer" requires a non-empty argument.' - exit 1 - fi - ;; - # Operating Systems - -o|--os) - if [ -n "$2" ] - then - linux=false - windows=false - osx=false - if [[ "$2" = *"l"* ]] - then - linux=true - fi - if [[ "$2" = *"w"* ]] - then - windows=true - fi - if [[ "$2" = *"x"* ]] - then - osx=true - fi - shift - else - echo 'Error: "--os" requires an argument containing an l (for linux), w (for windows), or x (for Mac OSX)' - exit 1 - fi - ;; - # Help message - -h|--help) - echo "$usage" - exit 0 - ;; - # Commit or branch - -c|--commit) - commit=true - ;; - # Number of Processes - -j) - if [ -n "$2" ] - then - proc=$2 - shift - else - echo 'Error: "-j" requires an argument' - exit 1 - fi - ;; - # Memory to allocate - -m) - if [ -n "$2" ] - then - mem=$2 - shift - else - echo 'Error: "-m" requires an argument' - exit 1 - fi - ;; - # URL - -u) - if [ -n "$2" ] - then - url=$2 - shift - else - echo 'Error: "-u" requires an argument' - exit 1 - fi - ;; - # kvm - --kvm) - lxc=false - ;; - # Detach sign - --detach-sign) - signProg="true" - commitFiles=false - ;; - # Commit files - --no-commit) - commitFiles=false - ;; - # Setup - --setup) - setup=true - ;; - *) # Default case: If no more options then break out of the loop. - break - esac - shift -done - -# Set up LXC -if [[ $lxc = true ]] -then - export USE_LXC=1 - export LXC_BRIDGE=lxcbr0 - sudo ifconfig lxcbr0 up 10.0.2.2 -fi - -# Check for OSX SDK -if [[ ! -e "gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz" && $osx == true ]] -then - echo "Cannot build for OSX, SDK does not exist. Will build for other OSes" - osx=false -fi - -# Get signer -if [[ -n "$1" ]] -then - SIGNER=$1 - shift -fi - -# Get version -if [[ -n "$1" ]] -then - VERSION=$1 - COMMIT=$VERSION - shift -fi - -# Check that a signer is specified -if [[ $SIGNER == "" ]] -then - echo "$scriptName: Missing signer." - echo "Try $scriptName --help for more information" - exit 1 -fi - -# Check that a version is specified -if [[ $VERSION == "" ]] -then - echo "$scriptName: Missing version." - echo "Try $scriptName --help for more information" - exit 1 -fi - -# Add a "v" if no -c -if [[ $commit = false ]] -then - COMMIT="v${VERSION}" -fi -echo ${COMMIT} - -# Setup build environment -if [[ $setup = true ]] -then - sudo apt-get install ruby apache2 git apt-cacher-ng python-vm-builder qemu-kvm qemu-utils - git clone https://github.com/gridcoin-community/gitian.sigs.git - git clone https://github.com/devrandom/gitian-builder.git - pushd ./gitian-builder - if [[ -n "$USE_LXC" ]] - then - sudo apt-get install lxc - bin/make-base-vm --suite trusty --arch amd64 --lxc - else - bin/make-base-vm --suite trusty --arch amd64 - fi - popd -fi - -# Set up build -pushd ./Gridcoin-Research -git fetch -git checkout ${COMMIT} -popd - -# Build -if [[ $build = true ]] -then - # Make output folder - mkdir -p ./gridcoin-binaries/${VERSION} - - # Build Dependencies - echo "" - echo "Building Dependencies" - echo "" - pushd ./gitian-builder - mkdir -p inputs - wget -N -P inputs $osslPatchUrl - wget -N -P inputs $osslTarUrl - make -C ../Gridcoin-Research/depends download SOURCES_PATH=`pwd`/cache/common - - # Linux - if [[ $linux = true ]] - then - echo "" - echo "Compiling ${VERSION} Linux" - echo "" - ./bin/gbuild -j ${proc} -m ${mem} --commit gridcoin=${COMMIT} --url gridcoin=${url} ../Gridcoin-Research/contrib/gitian-descriptors/gitian-linux.yml - ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../Gridcoin-Research/contrib/gitian-descriptors/gitian-linux.yml - mv build/out/gridcoin-*.tar.gz build/out/src/gridcoin-*.tar.gz ../gridcoin-binaries/${VERSION} - fi - # Windows - if [[ $windows = true ]] - then - echo "" - echo "Compiling ${VERSION} Windows" - echo "" - ./bin/gbuild -j ${proc} -m ${mem} --commit gridcoin=${COMMIT} --url gridcoin=${url} ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win.yml - ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-win-unsigned --destination ../gitian.sigs/ ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win.yml - mv build/out/gridcoin-*-win-unsigned.tar.gz inputs/gridcoin-win-unsigned.tar.gz - mv build/out/gridcoin-*.zip build/out/gridcoin-*.exe ../gridcoin-binaries/${VERSION} - fi - # Mac OSX - if [[ $osx = true ]] - then - echo "" - echo "Compiling ${VERSION} Mac OSX" - echo "" - ./bin/gbuild -j ${proc} -m ${mem} --commit gridcoin=${COMMIT} --url gridcoin=${url} ../Gridcoin-Research/contrib/gitian-descriptors/gitian-osx.yml - ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../Gridcoin-Research/contrib/gitian-descriptors/gitian-osx.yml - mv build/out/gridcoin-*-osx-unsigned.tar.gz inputs/gridcoin-osx-unsigned.tar.gz - mv build/out/gridcoin-*.tar.gz build/out/gridcoin-*.dmg ../gridcoin-binaries/${VERSION} - fi - popd - - if [[ $commitFiles = true ]] - then - # Commit to gitian.sigs repo - echo "" - echo "Committing ${VERSION} Unsigned Sigs" - echo "" - pushd gitian.sigs - git add ${VERSION}-linux/${SIGNER} - git add ${VERSION}-win-unsigned/${SIGNER} - git add ${VERSION}-osx-unsigned/${SIGNER} - git commit -a -m "Add ${VERSION} unsigned sigs for ${SIGNER}" - popd - fi -fi - -# Verify the build -if [[ $verify = true ]] -then - # Linux - pushd ./gitian-builder - echo "" - echo "Verifying v${VERSION} Linux" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-linux ../Gridcoin-Research/contrib/gitian-descriptors/gitian-linux.yml - # Windows - echo "" - echo "Verifying v${VERSION} Windows" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-unsigned ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win.yml - # Mac OSX - echo "" - echo "Verifying v${VERSION} Mac OSX" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-unsigned ../Gridcoin-Research/contrib/gitian-descriptors/gitian-osx.yml - # Signed Windows - echo "" - echo "Verifying v${VERSION} Signed Windows" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../Gridcoin-Research/contrib/gitian-descriptors/gitian-osx-signer.yml - # Signed Mac OSX - echo "" - echo "Verifying v${VERSION} Signed Mac OSX" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../Gridcoin-Research/contrib/gitian-descriptors/gitian-osx-signer.yml - popd -fi - -# Sign binaries -if [[ $sign = true ]] -then - - pushd ./gitian-builder - # Sign Windows - if [[ $windows = true ]] - then - echo "" - echo "Signing ${VERSION} Windows" - echo "" - ./bin/gbuild -i --commit signature=${COMMIT} ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win-signer.yml - ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-win-signed --destination ../gitian.sigs/ ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win-signer.yml - mv build/out/gridcoin-*win64-setup.exe ../gridcoin-binaries/${VERSION} - mv build/out/gridcoin-*win32-setup.exe ../gridcoin-binaries/${VERSION} - fi - # Sign Mac OSX - if [[ $osx = true ]] - then - echo "" - echo "Signing ${VERSION} Mac OSX" - echo "" - ./bin/gbuild -i --commit signature=${COMMIT} ../Gridcoin-Research/contrib/gitian-descriptors/gitian-osx-signer.yml - ./bin/gsign -p $signProg --signer $SIGNER --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../Gridcoin-Research/contrib/gitian-descriptors/gitian-osx-signer.yml - mv build/out/gridcoin-osx-signed.dmg ../gridcoin-binaries/${VERSION}/gridcoin-${VERSION}-osx.dmg - fi - popd - - if [[ $commitFiles = true ]] - then - # Commit Sigs - pushd gitian.sigs - echo "" - echo "Committing ${VERSION} Signed Sigs" - echo "" - git add ${VERSION}-win-signed/${SIGNER} - git add ${VERSION}-osx-signed/${SIGNER} - git commit -a -m "Add ${VERSION} signed binary sigs for ${SIGNER}" - popd - fi -fi diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml deleted file mode 100644 index 0bb77f7263..0000000000 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ /dev/null @@ -1,193 +0,0 @@ ---- -name: "gridcoin-linux-0.16" -enable_cache: true -suites: -- "trusty" -architectures: -- "amd64" -packages: -- "curl" -- "g++-aarch64-linux-gnu" -- "g++-4.8-aarch64-linux-gnu" -- "gcc-4.8-aarch64-linux-gnu" -- "binutils-aarch64-linux-gnu" -- "g++-arm-linux-gnueabihf" -- "g++-4.8-arm-linux-gnueabihf" -- "gcc-4.8-arm-linux-gnueabihf" -- "binutils-arm-linux-gnueabihf" -- "g++-4.8-multilib" -- "gcc-4.8-multilib" -- "binutils-gold" -- "git-core" -- "pkg-config" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "ca-certificates" -- "python" -- "bison" -remotes: -- "url": "https://github.com/gridcoin/Gridcoin-Research.git" - "dir": "Gridcoin-Research" -files: [] -script: | - - WRAP_DIR=$HOME/wrapped - HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" - FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="date ar ranlib nm" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS=-static-libstdc++ - - export QT_RCC_TEST=1 - export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" - export TZ="UTC" - export BUILD_DIR=`pwd` - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes - mkdir -p $EXTRA_INCLUDES_BASE - - # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, - # but we can't write there. Instead, create a link here and force it to be included in the - # search paths by wrapping gcc/g++. - - mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu - rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm - ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm - - for prog in gcc g++; do - rm -f ${WRAP_DIR}/${prog} - cat << EOF > ${WRAP_DIR}/${prog} - #!/bin/bash - REAL="`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1`" - for var in "\$@" - do - if [ "\$var" = "-m32" ]; then - export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" - export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" - break - fi - done - \$REAL \$@ - EOF - chmod +x ${WRAP_DIR}/${prog} - done - - cd bitcoin || exit - BASEPREFIX=`pwd`/depends - # Build dependencies for each host - for i in $HOSTS; do - EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" - if [ -d "$EXTRA_INCLUDES" ]; then - export HOST_ID_SALT="$EXTRA_INCLUDES" - fi - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - unset HOST_ID_SALT - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=`echo bitcoin-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` - # Correct tar file order - mkdir -p temp - pushd temp - tar xf ../$SOURCEDIST - find bitcoin-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST - popd - - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h - - ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir -p distsrc-${i} - cd distsrc-${i} || exit - INSTALLPATH=`pwd`/installed/${DISTNAME} - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ - - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - - #TODO: This is a quick hack that disables symbol checking for arm. - # Instead, we should investigate why these are popping up. - # For aarch64, we'll need to bump up the min GLIBC version, as the abi - # support wasn't introduced until 2.17. - case $i in - aarch64-*) : ;; - arm-*) : ;; - *) make ${MAKEOPTS} -C src check-symbols ;; - esac - - make install DESTDIR=${INSTALLPATH} - cd installed || exit - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; - find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; - find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz - cd ../../ - rm -rf distsrc-${i} - done - mkdir -p $OUTDIR/src - mv $SOURCEDIST $OUTDIR/src diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml deleted file mode 100644 index a797dbedbd..0000000000 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: "gridcoin-dmg-signer" -suites: -- "trusty" -architectures: -- "amd64" -packages: -- "faketime" -remotes: -- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" - "dir": "signature" -files: -- "bitcoin-osx-unsigned.tar.gz" -script: | - WRAP_DIR=$HOME/wrapped - mkdir -p ${WRAP_DIR} - export PATH=`pwd`:$PATH - FAKETIME_PROGS="dmg genisoimage" - - # Create global faketime wrappers - for prog in ${FAKETIME_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - - UNSIGNED=gridcoin-osx-unsigned.tar.gz - SIGNED=gridcoin-osx-signed.dmg - - tar -xf ${UNSIGNED} - OSX_VOLNAME="$(cat osx_volname)" - ./detached-sig-apply.sh ${UNSIGNED} signature/osx - ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -apple -o uncompressed.dmg signed-app - ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml deleted file mode 100644 index fdab37f0cf..0000000000 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ /dev/null @@ -1,166 +0,0 @@ ---- -name: "gridcoin-osx-0.16" -enable_cache: true -suites: -- "trusty" -architectures: -- "amd64" -packages: -- "ca-certificates" -- "curl" -- "g++" -- "git-core" -- "pkg-config" -- "autoconf" -- "librsvg2-bin" -- "libtiff-tools" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "cmake" -- "imagemagick" -- "libcap-dev" -- "libz-dev" -- "libbz2-dev" -- "python" -- "python-dev" -- "python-setuptools" -- "fonts-tuffy" -remotes: -- "url": "https://github.com/gridcoin/Gridcoin-Research.git" - "dir": "Gridcoin-Research" -files: -- "MacOSX10.14.sdk.tar.gz" -script: | - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin11" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" - FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg genisoimage" - - export QT_RCC_TEST=1 - export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" - export TZ="UTC" - export BUILD_DIR=`pwd` - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - export ZERO_AR_DATE=1 - - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd bitcoin || exit - BASEPREFIX=`pwd`/depends - - mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz - - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=`echo bitcoin-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` - - # Correct tar file order - mkdir -p temp - pushd temp - tar xf ../$SOURCEDIST - find bitcoin-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST - popd - - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h - - ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir -p distsrc-${i} - cd distsrc-${i} || exit - INSTALLPATH=`pwd`/installed/${DISTNAME} - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ - - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} - make ${MAKEOPTS} - make install-strip DESTDIR=${INSTALLPATH} - - make osx_volname - make deploydir - OSX_VOLNAME="$(cat osx_volname)" - mkdir -p unsigned-app-${i} - cp osx_volname unsigned-app-${i}/ - cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} - cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate - cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff - mv dist unsigned-app-${i} - pushd unsigned-app-${i} - find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz - popd - - make deploy - ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg - - cd installed || exit - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME} | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - cd ../../ - done - mkdir -p $OUTDIR/src - mv $SOURCEDIST $OUTDIR/src - mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml deleted file mode 100644 index 618315c360..0000000000 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: "gridcoin-win-signer" -suites: -- "trusty" -architectures: -- "amd64" -packages: -- "libssl-dev" -- "autoconf" -remotes: -- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" - "dir": "signature" -files: -- "osslsigncode-1.7.1.tar.gz" -- "osslsigncode-Backports-to-1.7.1.patch" -- "bitcoin-win-unsigned.tar.gz" -script: | - BUILD_DIR=`pwd` - SIGDIR=${BUILD_DIR}/signature/win - UNSIGNED_DIR=${BUILD_DIR}/unsigned - - echo "f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 osslsigncode-1.7.1.tar.gz" | sha256sum -c - echo "a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 osslsigncode-Backports-to-1.7.1.patch" | sha256sum -c - - mkdir -p ${UNSIGNED_DIR} - tar -C ${UNSIGNED_DIR} -xf bitcoin-win-unsigned.tar.gz - - tar xf osslsigncode-1.7.1.tar.gz - cd osslsigncode-1.7.1 || exit - patch -p1 < ${BUILD_DIR}/osslsigncode-Backports-to-1.7.1.patch - - ./configure --without-gsf --without-curl --disable-dependency-tracking - make - find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do - INFILE="`basename "${i}"`" - OUTFILE="`echo "${INFILE}" | sed s/-unsigned//`" - ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem" - done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml deleted file mode 100644 index 6709ed152c..0000000000 --- a/contrib/gitian-descriptors/gitian-win.yml +++ /dev/null @@ -1,179 +0,0 @@ ---- -name: "gridcoin-win-0.16" -enable_cache: true -suites: -- "trusty" -architectures: -- "amd64" -packages: -- "curl" -- "g++" -- "git-core" -- "pkg-config" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "mingw-w64" -- "g++-mingw-w64" -- "nsis" -- "zip" -- "ca-certificates" -- "python" -remotes: -- "url": "https://github.com/gridcoin/Gridcoin-Research.git" - "dir": "gridcoin" -files: [] -script: | - WRAP_DIR=$HOME/wrapped - HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" - FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip objcopy" - FAKETIME_PROGS="date makensis zip" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" - - export QT_RCC_TEST=1 - export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" - export TZ="UTC" - export BUILD_DIR=`pwd` - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - function create_per-host_linker_wrapper { - # This is only needed for trusty, as the mingw linker leaks a few bytes of - # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 - for i in $HOSTS; do - mkdir -p ${WRAP_DIR}/${i} - for prog in collect2; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}/${prog} - REAL=$(${i}-gcc -print-prog-name=${prog}) - echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} - echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} - chmod +x ${WRAP_DIR}/${i}/${prog} - done - for prog in gcc g++; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_linker_wrapper "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd bitcoin || exit - BASEPREFIX=`pwd`/depends - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_linker_wrapper "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=`echo bitcoin-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` - - # Correct tar file order - mkdir -p temp - pushd temp - tar xf ../$SOURCEDIST - find bitcoin-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST - mkdir -p $OUTDIR/src - cp ../$SOURCEDIST $OUTDIR/src - popd - - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h - - ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir -p distsrc-${i} - cd distsrc-${i} || exit - INSTALLPATH=`pwd`/installed/${DISTNAME} - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ - - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make deploy - make install DESTDIR=${INSTALLPATH} - rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe - cp -f bitcoin-*setup*.exe $OUTDIR/ - cd installed || exit - mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; - find ${DISTNAME}/lib -type f -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; - find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip - find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}-debug.zip - cd ../../ - rm -rf distsrc-${i} - done - cp -rf contrib/windeploy $BUILD_DIR - cd $BUILD_DIR/windeploy || exit - mkdir unsigned - cp $OUTDIR/bitcoin-*setup-unsigned.exe unsigned/ - find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz - mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip - mv ${OUTDIR}/${DISTNAME}-i686-*-debug.zip ${OUTDIR}/${DISTNAME}-win32-debug.zip - mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip - mv ${OUTDIR}/${DISTNAME}-i686-*.zip ${OUTDIR}/${DISTNAME}-win32.zip diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 90f4362672..0750e75547 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -65,6 +65,11 @@ make make deploy ``` +## NOTE: the below deterministic macOS DMG notes below are retained for historical purposes. +Since gitian has been removed, we are deciding the best path forward for deterministic builds. +Bitcoin is heading in the direction of Guix. Gridcoin may do the same, but this has not been +decided yet by the Gridcoin development team and community. + ## Deterministic macOS DMG Notes Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple `binutils` (`ld`, `ar`, etc) and DMG authoring tools. diff --git a/doc/README.md b/doc/README.md index 22a58da543..bff735ff6e 100644 --- a/doc/README.md +++ b/doc/README.md @@ -27,7 +27,6 @@ The following are developer notes on how to build Gridcoin on your native platfo - [Unix Build Notes](build-unix.md) - [Windows Build Notes](build-windows.md) - [OpenBSD Build Notes](build-openbsd.md) -- [Gitian Building Guide](gitian-building.md) Running ------- diff --git a/doc/gitian-building.md b/doc/gitian-building.md deleted file mode 100644 index db6a46ef16..0000000000 --- a/doc/gitian-building.md +++ /dev/null @@ -1,480 +0,0 @@ -Gitian building -================ - -*Setup instructions for a Gitian build of Bitcoin Core using a Debian VM or physical system.* - -Gitian is the deterministic build process that is used to build the Bitcoin -Core executables. It provides a way to be reasonably sure that the -executables are really built from the source on GitHub. It also makes sure that -the same, tested dependencies are used and statically built into the executable. - -Multiple developers build the source code by following a specific descriptor -("recipe"), cryptographically sign the result, and upload the resulting signature. -These results are compared and only if they match, the build is accepted and uploaded -to bitcoin.org. - -More independent Gitian builders are needed, which is why this guide exists. -It is preferred you follow these steps yourself instead of using someone else's -VM image to avoid 'contaminating' the build. - -Table of Contents ------------------- - -- [Create a new VirtualBox VM](#create-a-new-virtualbox-vm) -- [Connecting to the VM](#connecting-to-the-vm) -- [Setting up Debian for Gitian building](#setting-up-debian-for-gitian-building) -- [Installing Gitian](#installing-gitian) -- [Setting up the Gitian image](#setting-up-the-gitian-image) -- [Getting and building the inputs](#getting-and-building-the-inputs) -- [Building Bitcoin Core](#building-bitcoin-core) -- [Building an alternative repository](#building-an-alternative-repository) -- [Signing externally](#signing-externally) -- [Uploading signatures](#uploading-signatures) - -Preparing the Gitian builder host ---------------------------------- - -The first step is to prepare the host environment that will be used to perform the Gitian builds. -This guide explains how to set up the environment, and how to start the builds. - -Debian Linux was chosen as the host distribution because it has a lightweight install (in contrast to Ubuntu) and is readily available. -Any kind of virtualization can be used, for example: -- [VirtualBox](https://www.virtualbox.org/) (covered by this guide) -- [KVM](http://www.linux-kvm.org/page/Main_Page) -- [LXC](https://linuxcontainers.org/), see also [Gitian host docker container](https://github.com/gdm85/tenku/tree/master/docker/gitian-bitcoin-host/README.md). - -You can also install Gitian on actual hardware instead of using virtualization. - -Create a new VirtualBox VM ---------------------------- -In the VirtualBox GUI click "New" and choose the following parameters in the wizard: - -![](gitian-building/create_new_vm.png) - -- Type: Linux, Debian (64-bit) - -![](gitian-building/create_vm_memsize.png) - -- Memory Size: at least 3000MB, anything less and the build might not complete. - -![](gitian-building/create_vm_hard_disk.png) - -- Hard Disk: Create a virtual hard disk now - -![](gitian-building/create_vm_hard_disk_file_type.png) - -- Hard Disk file type: Use the default, VDI (VirtualBox Disk Image) - -![](gitian-building/create_vm_storage_physical_hard_disk.png) - -- Storage on physical hard disk: Dynamically Allocated - -![](gitian-building/create_vm_file_location_size.png) - -- File location and size: at least 40GB; as low as 20GB *may* be possible, but better to err on the safe side -- Click `Create` - -After creating the VM, we need to configure it. - -- Click the `Settings` button, then go to the `Network` tab. Adapter 1 should be attached to `NAT`. - -![](gitian-building/network_settings.png) - -- Click `Advanced`, then `Port Forwarding`. We want to set up a port through which we can reach the VM to get files in and out. -- Create a new rule by clicking the plus icon. - -![](gitian-building/port_forwarding_rules.png) - -- Set up the new rule the following way: - - Name: `SSH` - - Protocol: `TCP` - - Leave Host IP empty - - Host Port: `22222` - - Leave Guest IP empty - - Guest Port: `22` - -- Click `Ok` twice to save. - -Get the [Debian 8.x net installer](http://cdimage.debian.org/mirror/cdimage/archive/8.5.0/amd64/iso-cd/debian-8.5.0-amd64-netinst.iso) (a more recent minor version should also work, see also [Debian Network installation](https://www.debian.org/CD/netinst/)). -This DVD image can be [validated](https://www.debian.org/CD/verify) using a SHA256 hashing tool, for example on -Unixy OSes by entering the following in a terminal: - - echo "ad4e8c27c561ad8248d5ebc1d36eb172f884057bfeb2c22ead823f59fa8c3dff debian-8.5.0-amd64-netinst.iso" | sha256sum -c - # (must return OK) - -Then start the VM. On the first launch you will be asked for a CD or DVD image. Choose the downloaded ISO. - -![](gitian-building/select_startup_disk.png) - -Installing Debian ------------------- - -This section will explain how to install Debian on the newly created VM. - -- Choose the non-graphical installer. We do not need the graphical environment; it will only increase installation time and disk usage. - -![](gitian-building/debian_install_1_boot_menu.png) - -**Note**: Navigating in the Debian installer: -To keep a setting at the default and proceed, just press `Enter`. -To select a different button, press `Tab`. - -- Choose locale and keyboard settings (doesn't matter, you can just go with the defaults or select your own information) - -![](gitian-building/debian_install_2_select_a_language.png) -![](gitian-building/debian_install_3_select_location.png) -![](gitian-building/debian_install_4_configure_keyboard.png) - -- The VM will detect network settings using DHCP, this should all proceed automatically -- Configure the network: - - Hostname `debian`. - - Leave domain name empty. - -![](gitian-building/debian_install_5_configure_the_network.png) -![](gitian-building/debian_install_6_domain_name.png) - -- Choose a root password and enter it twice (remember it for later) - -![](gitian-building/debian_install_6a_set_up_root_password.png) - -- Name the new user `debian` (the full name doesn't matter, you can leave it empty) -- Set the account username as `debian` - -![](gitian-building/debian_install_7_set_up_user_fullname.png) -![](gitian-building/debian_install_8_set_up_username.png) - -- Choose a user password and enter it twice (remember it for later) - -![](gitian-building/debian_install_9_user_password.png) - -- The installer will set up the clock using a time server; this process should be automatic -- Set up the clock: choose a time zone (depends on the locale settings that you picked earlier; specifics don't matter) - -![](gitian-building/debian_install_10_configure_clock.png) - -- Disk setup - - Partitioning method: Guided - Use the entire disk - -![](gitian-building/debian_install_11_partition_disks.png) - - - Select disk to partition: SCSI1 (0,0,0) - -![](gitian-building/debian_install_12_choose_disk.png) - - - Partition Disks -> *All files in one partition* - -![](gitian-building/all_files_in_one_partition.png) - - - Finish partitioning and write changes to disk -> *Yes* (`Tab`, `Enter` to select the `Yes` button) - -![](gitian-building/debian_install_14_finish.png) -![](gitian-building/debian_install_15_write_changes.png) - -- The base system will be installed, this will take a minute or so -- Choose a mirror (any will do) - -![](gitian-building/debian_install_16_choose_a_mirror.png) - -- Enter proxy information (unless you are on an intranet, leave this empty) - -![](gitian-building/debian_install_18_proxy_settings.png) - -- Wait a bit while 'Select and install software' runs -- Participate in popularity contest -> *No* -- Choose software to install. We need just the base system. -- Make sure only 'SSH server' and 'Standard System Utilities' are checked -- Uncheck 'Debian Desktop Environment' and 'Print Server' - -![](gitian-building/debian_install_19_software_selection.png) - -- Install the GRUB boot loader to the master boot record? -> Yes - -![](gitian-building/debian_install_20_install_grub.png) - -- Device for boot loader installation -> ata-VBOX_HARDDISK - -![](gitian-building/debian_install_21_install_grub_bootloader.png) - -- Installation Complete -> *Continue* -- After installation, the VM will reboot and you will have a working Debian VM. Congratulations! - -![](gitian-building/debian_install_22_finish_installation.png) - - -After Installation -------------------- -The next step in the guide involves logging in as root via SSH. -SSH login for root users is disabled by default, so we'll enable that now. - -Login to the VM using username `root` and the root password you chose earlier. -You'll be presented with a screen similar to this. - -![](gitian-building/debian_root_login.png) - -Type: - -``` -sed -i 's/^PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config -``` -and press enter. Then, -``` -/etc/init.d/ssh restart -``` -and enter to restart SSH. Logout by typing 'logout' and pressing 'enter'. - -Connecting to the VM ----------------------- - -After the VM has booted you can connect to it using SSH, and files can be copied from and to the VM using a SFTP utility. -Connect to `localhost`, port `22222` (or the port configured when installing the VM). -On Windows you can use [putty](https://www.chiark.greenend.org.uk/~sgtatham/putty/download.html) and [WinSCP](https://winscp.net/eng/index.php). - -For example, to connect as `root` from a Linux command prompt use - - $ ssh root@localhost -p 22222 - The authenticity of host '[localhost]:22222 ([127.0.0.1]:22222)' can't be established. - RSA key fingerprint is ae:f5:c8:9f:17:c6:c7:1b:c2:1b:12:31:1d:bb:d0:c7. - Are you sure you want to continue connecting (yes/no)? yes - Warning: Permanently added '[localhost]:22222' (RSA) to the list of known hosts. - root@localhost's password: (enter root password configured during install) - - The programs included with the Debian GNU/Linux system are free software; - the exact distribution terms for each program are described in the - individual files in /usr/share/doc/*/copyright. - - Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent - permitted by applicable law. - root@debian:~# - -Replace `root` with `debian` to log in as user. - -Setting up Debian for Gitian building --------------------------------------- - -In this section we will be setting up the Debian installation for Gitian building. - -First we need to log in as `root` to set up dependencies and make sure that our -user can use the sudo command. Type/paste the following in the terminal: - -```bash -apt-get install git ruby sudo apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make ubuntu-archive-keyring curl -adduser debian sudo -``` - -Then set up LXC and the rest with the following, which is a complex jumble of settings and workarounds: - -```bash -# the version of lxc-start in Debian needs to run as root, so make sure -# that the build script can execute it without providing a password -echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc -echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-execute" >> /etc/sudoers.d/gitian-lxc -# make /etc/rc.local script that sets up bridge between guest and host -echo '#!/bin/sh -e' > /etc/rc.local -echo 'brctl addbr br0' >> /etc/rc.local -echo 'ifconfig br0 10.0.3.2/24 up' >> /etc/rc.local -echo 'iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE' >> /etc/rc.local -echo 'echo 1 > /proc/sys/net/ipv4/ip_forward' >> /etc/rc.local -echo 'exit 0' >> /etc/rc.local -# make sure that USE_LXC is always set when logging in as debian, -# and configure LXC IP addresses -echo 'export USE_LXC=1' >> /home/debian/.profile -echo 'export GITIAN_HOST_IP=10.0.3.2' >> /home/debian/.profile -echo 'export LXC_GUEST_IP=10.0.3.5' >> /home/debian/.profile -reboot -``` - -At the end the VM is rebooted to make sure that the changes take effect. The steps in this -section only need to be performed once. - -Installing Gitian ------------------- - -Re-login as the user `debian` that was created during installation. -The rest of the steps in this guide will be performed as that user. - -There is no `python-vm-builder` package in Debian, so we need to install it from source ourselves, - -```bash -wget https://archive.ubuntu.com/ubuntu/pool/universe/v/vm-builder/vm-builder_0.12.4+bzr494.orig.tar.gz -echo "76cbf8c52c391160b2641e7120dbade5afded713afaa6032f733a261f13e6a8e vm-builder_0.12.4+bzr494.orig.tar.gz" | sha256sum -c -# (verification -- must return OK) -tar -zxvf vm-builder_0.12.4+bzr494.orig.tar.gz -cd vm-builder-0.12.4+bzr494 -sudo python setup.py install -cd .. -``` - -**Note**: When sudo asks for a password, enter the password for the user *debian* not for *root*. - -Clone the git repositories for bitcoin and Gitian. - -```bash -git clone https://github.com/devrandom/gitian-builder.git -git clone https://github.com/bitcoin/bitcoin -git clone https://github.com/bitcoin-core/gitian.sigs.git -``` - -Setting up the Gitian image -------------------------- - -Gitian needs a virtual image of the operating system to build in. -Currently this is Ubuntu Trusty x86_64. -This image will be copied and used every time that a build is started to -make sure that the build is deterministic. -Creating the image will take a while, but only has to be done once. - -Execute the following as user `debian`: - -```bash -cd gitian-builder -bin/make-base-vm --lxc --arch amd64 --suite trusty -``` - -There will be a lot of warnings printed during the build of the image. These can be ignored. - -**Note**: When sudo asks for a password, enter the password for the user *debian* not for *root*. - -Getting and building the inputs --------------------------------- - -At this point you have two options, you can either use the automated script (found in [contrib/gitian-build.sh](/contrib/gitian-build.sh)) or you could manually do everything by following this guide. If you're using the automated script, then run it with the "--setup" command. Afterwards, run it with the "--build" command (example: "contrib/gitian-build.sh -b signer 0.13.0"). Otherwise ignore this. - -Follow the instructions in [doc/release-process.md](release-process.md#fetch-and-create-inputs-first-time-or-when-dependency-versions-change) -in the bitcoin repository under 'Fetch and create inputs' to install sources which require -manual intervention. Also optionally follow the next step: 'Seed the Gitian sources cache -and offline git repositories' which will fetch the remaining files required for building -offline. - -Building Bitcoin Core ----------------- - -To build Bitcoin Core (for Linux, OS X and Windows) just follow the steps under 'perform -Gitian builds' in [doc/release-process.md](release-process.md#perform-gitian-builds) in the bitcoin repository. - -This may take some time as it will build all the dependencies needed for each descriptor. -These dependencies will be cached after a successful build to avoid rebuilding them when possible. - -At any time you can check the package installation and build progress with - -```bash -tail -f var/install.log -tail -f var/build.log -``` - -Output from `gbuild` will look something like - - Initialized empty Git repository in /home/debian/gitian-builder/inputs/bitcoin/.git/ - remote: Counting objects: 57959, done. - remote: Total 57959 (delta 0), reused 0 (delta 0), pack-reused 57958 - Receiving objects: 100% (57959/57959), 53.76 MiB | 484.00 KiB/s, done. - Resolving deltas: 100% (41590/41590), done. - From https://github.com/bitcoin/bitcoin - ... (new tags, new branch etc) - --- Building for trusty amd64 --- - Stopping target if it is up - Making a new image copy - stdin: is not a tty - Starting target - Checking if target is up - Preparing build environment - Updating apt-get repository (log in var/install.log) - Installing additional packages (log in var/install.log) - Grabbing package manifest - stdin: is not a tty - Creating build script (var/build-script) - lxc-start: Connection refused - inotify event with no name (mask 32768) - Running build script (log in var/build.log) - -Building an alternative repository ------------------------------------ - -If you want to do a test build of a pull on GitHub it can be useful to point -the Gitian builder at an alternative repository, using the same descriptors -and inputs. - -For example: -```bash -URL=https://github.com/laanwj/bitcoin.git -COMMIT=2014_03_windows_unicode_path -./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml -./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml -./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml -``` - -Building fully offline ------------------------ - -For building fully offline including attaching signatures to unsigned builds, the detached-sigs repository -and the bitcoin git repository with the desired tag must both be available locally, and then gbuild must be -told where to find them. It also requires an apt-cacher-ng which is fully-populated but set to offline mode, or -manually disabling gitian-builder's use of apt-get to update the VM build environment. - -To configure apt-cacher-ng as an offline cacher, you will need to first populate its cache with the relevant -files. You must additionally patch target-bin/bootstrap-fixup to set its apt sources to something other than -plain archive.ubuntu.com: us.archive.ubuntu.com works. - -So, if you use LXC: - -```bash -export PATH="$PATH":/path/to/gitian-builder/libexec -export USE_LXC=1 -cd /path/to/gitian-builder -./libexec/make-clean-vm --suite trusty --arch amd64 - -LXC_ARCH=amd64 LXC_SUITE=trusty on-target -u root apt-get update -LXC_ARCH=amd64 LXC_SUITE=trusty on-target -u root \ - -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install \ - $( sed -ne '/^packages:/,/[^-] .*/ {/^- .*/{s/"//g;s/- //;p}}' ../bitcoin/contrib/gitian-descriptors/*|sort|uniq ) -LXC_ARCH=amd64 LXC_SUITE=trusty on-target -u root apt-get -q -y purge grub -LXC_ARCH=amd64 LXC_SUITE=trusty on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade -``` - -And then set offline mode for apt-cacher-ng: - -``` -/etc/apt-cacher-ng/acng.conf -[...] -Offlinemode: 1 -[...] - -service apt-cacher-ng restart -``` - -Then when building, override the remote URLs that gbuild would otherwise pull from the Gitian descriptors:: -```bash - -cd /some/root/path/ -git clone https://github.com/bitcoin-core/bitcoin-detached-sigs.git - -BTCPATH=/some/root/path/bitcoin -SIGPATH=/some/root/path/bitcoin-detached-sigs - -./bin/gbuild --url bitcoin=${BTCPATH},signature=${SIGPATH} ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml -``` - -Signing externally -------------------- - -If you want to do the PGP signing on another device, that's also possible; just define `SIGNER` as mentioned -and follow the steps in the build process as normal. - - gpg: skipped "laanwj": secret key not available - -When you execute `gsign` you will get an error from GPG, which can be ignored. Copy the resulting `.assert` files -in `gitian.sigs` to your signing machine and do - -```bash - gpg --detach-sign ${VERSION}-linux/${SIGNER}/bitcoin-linux-build.assert - gpg --detach-sign ${VERSION}-win/${SIGNER}/bitcoin-win-build.assert - gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/bitcoin-osx-build.assert -``` - -This will create the `.sig` files that can be committed together with the `.assert` files to assert your -Gitian build. - -Uploading signatures ---------------------- - -After building and signing you can push your signatures (both the `.assert` and `.assert.sig` files) to the -[bitcoin-core/gitian.sigs](https://github.com/bitcoin-core/gitian.sigs/) repository, or if that's not possible create a pull -request. You can also mail the files to Wladimir (laanwj@gmail.com) and he will commit them. diff --git a/doc/gitian-building/all_files_in_one_partition.png b/doc/gitian-building/all_files_in_one_partition.png deleted file mode 100644 index 5f3a4f3e93..0000000000 Binary files a/doc/gitian-building/all_files_in_one_partition.png and /dev/null differ diff --git a/doc/gitian-building/create_new_vm.png b/doc/gitian-building/create_new_vm.png deleted file mode 100644 index 4df464be6a..0000000000 Binary files a/doc/gitian-building/create_new_vm.png and /dev/null differ diff --git a/doc/gitian-building/create_vm_file_location_size.png b/doc/gitian-building/create_vm_file_location_size.png deleted file mode 100644 index abfa36e48d..0000000000 Binary files a/doc/gitian-building/create_vm_file_location_size.png and /dev/null differ diff --git a/doc/gitian-building/create_vm_hard_disk.png b/doc/gitian-building/create_vm_hard_disk.png deleted file mode 100644 index c25894acb6..0000000000 Binary files a/doc/gitian-building/create_vm_hard_disk.png and /dev/null differ diff --git a/doc/gitian-building/create_vm_hard_disk_file_type.png b/doc/gitian-building/create_vm_hard_disk_file_type.png deleted file mode 100644 index 31fb37cf0d..0000000000 Binary files a/doc/gitian-building/create_vm_hard_disk_file_type.png and /dev/null differ diff --git a/doc/gitian-building/create_vm_memsize.png b/doc/gitian-building/create_vm_memsize.png deleted file mode 100644 index eefc0da65e..0000000000 Binary files a/doc/gitian-building/create_vm_memsize.png and /dev/null differ diff --git a/doc/gitian-building/create_vm_storage_physical_hard_disk.png b/doc/gitian-building/create_vm_storage_physical_hard_disk.png deleted file mode 100644 index 24b5fb4421..0000000000 Binary files a/doc/gitian-building/create_vm_storage_physical_hard_disk.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_10_configure_clock.png b/doc/gitian-building/debian_install_10_configure_clock.png deleted file mode 100644 index 04f75f56cd..0000000000 Binary files a/doc/gitian-building/debian_install_10_configure_clock.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_11_partition_disks.png b/doc/gitian-building/debian_install_11_partition_disks.png deleted file mode 100644 index 6665d87d4d..0000000000 Binary files a/doc/gitian-building/debian_install_11_partition_disks.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_12_choose_disk.png b/doc/gitian-building/debian_install_12_choose_disk.png deleted file mode 100644 index 217969bdb1..0000000000 Binary files a/doc/gitian-building/debian_install_12_choose_disk.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_14_finish.png b/doc/gitian-building/debian_install_14_finish.png deleted file mode 100644 index d8734e1d3b..0000000000 Binary files a/doc/gitian-building/debian_install_14_finish.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_15_write_changes.png b/doc/gitian-building/debian_install_15_write_changes.png deleted file mode 100644 index 38dad89145..0000000000 Binary files a/doc/gitian-building/debian_install_15_write_changes.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_16_choose_a_mirror.png b/doc/gitian-building/debian_install_16_choose_a_mirror.png deleted file mode 100644 index d6def1e611..0000000000 Binary files a/doc/gitian-building/debian_install_16_choose_a_mirror.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_18_proxy_settings.png b/doc/gitian-building/debian_install_18_proxy_settings.png deleted file mode 100644 index 2adb0668b6..0000000000 Binary files a/doc/gitian-building/debian_install_18_proxy_settings.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_19_software_selection.png b/doc/gitian-building/debian_install_19_software_selection.png deleted file mode 100644 index bcfbe902f4..0000000000 Binary files a/doc/gitian-building/debian_install_19_software_selection.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_1_boot_menu.png b/doc/gitian-building/debian_install_1_boot_menu.png deleted file mode 100644 index e95fbb3d55..0000000000 Binary files a/doc/gitian-building/debian_install_1_boot_menu.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_20_install_grub.png b/doc/gitian-building/debian_install_20_install_grub.png deleted file mode 100644 index b3171e6abd..0000000000 Binary files a/doc/gitian-building/debian_install_20_install_grub.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_21_install_grub_bootloader.png b/doc/gitian-building/debian_install_21_install_grub_bootloader.png deleted file mode 100644 index c8399c5dca..0000000000 Binary files a/doc/gitian-building/debian_install_21_install_grub_bootloader.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_22_finish_installation.png b/doc/gitian-building/debian_install_22_finish_installation.png deleted file mode 100644 index 789033c8a7..0000000000 Binary files a/doc/gitian-building/debian_install_22_finish_installation.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_2_select_a_language.png b/doc/gitian-building/debian_install_2_select_a_language.png deleted file mode 100644 index dabc19f61e..0000000000 Binary files a/doc/gitian-building/debian_install_2_select_a_language.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_3_select_location.png b/doc/gitian-building/debian_install_3_select_location.png deleted file mode 100644 index 4db2e75d01..0000000000 Binary files a/doc/gitian-building/debian_install_3_select_location.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_4_configure_keyboard.png b/doc/gitian-building/debian_install_4_configure_keyboard.png deleted file mode 100644 index e0b758e14a..0000000000 Binary files a/doc/gitian-building/debian_install_4_configure_keyboard.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_5_configure_the_network.png b/doc/gitian-building/debian_install_5_configure_the_network.png deleted file mode 100644 index fa840521cb..0000000000 Binary files a/doc/gitian-building/debian_install_5_configure_the_network.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_6_domain_name.png b/doc/gitian-building/debian_install_6_domain_name.png deleted file mode 100644 index a95faa835c..0000000000 Binary files a/doc/gitian-building/debian_install_6_domain_name.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_6a_set_up_root_password.png b/doc/gitian-building/debian_install_6a_set_up_root_password.png deleted file mode 100644 index 581d49fa6f..0000000000 Binary files a/doc/gitian-building/debian_install_6a_set_up_root_password.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_7_set_up_user_fullname.png b/doc/gitian-building/debian_install_7_set_up_user_fullname.png deleted file mode 100644 index c1bcce0e98..0000000000 Binary files a/doc/gitian-building/debian_install_7_set_up_user_fullname.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_8_set_up_username.png b/doc/gitian-building/debian_install_8_set_up_username.png deleted file mode 100644 index 9fd7923c8a..0000000000 Binary files a/doc/gitian-building/debian_install_8_set_up_username.png and /dev/null differ diff --git a/doc/gitian-building/debian_install_9_user_password.png b/doc/gitian-building/debian_install_9_user_password.png deleted file mode 100644 index eb059551f1..0000000000 Binary files a/doc/gitian-building/debian_install_9_user_password.png and /dev/null differ diff --git a/doc/gitian-building/debian_root_login.png b/doc/gitian-building/debian_root_login.png deleted file mode 100644 index da7cefd23c..0000000000 Binary files a/doc/gitian-building/debian_root_login.png and /dev/null differ diff --git a/doc/gitian-building/network_settings.png b/doc/gitian-building/network_settings.png deleted file mode 100644 index e2a1d1a730..0000000000 Binary files a/doc/gitian-building/network_settings.png and /dev/null differ diff --git a/doc/gitian-building/port_forwarding_rules.png b/doc/gitian-building/port_forwarding_rules.png deleted file mode 100644 index 5f678ebe71..0000000000 Binary files a/doc/gitian-building/port_forwarding_rules.png and /dev/null differ diff --git a/doc/gitian-building/select_startup_disk.png b/doc/gitian-building/select_startup_disk.png deleted file mode 100644 index 0b4f1d520c..0000000000 Binary files a/doc/gitian-building/select_startup_disk.png and /dev/null differ diff --git a/doc/release-process.md b/doc/release-process.md old mode 100644 new mode 100755 index 28aa6d9b38..2597f16fce --- a/doc/release-process.md +++ b/doc/release-process.md @@ -1,252 +1,57 @@ -Release Process -==================== +## Gridcoin Release Process -Before every release candidate: +### Preparaton + - git checkout staging + - git pull + - git merge testnet + - Update changelog and commit + - Change version in configure.ac to [version] and set CLIENT_VERSION_IS_RELEASE to 'true' and commit + - git push -* Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_IS_RELEASE` to `true`) -* Write release notes to changelog +### Merge to master + - git checkout master + - git pull + - git merge --no-ff -S staging # Merge and sign. Copy changelog into commit message, but remove any brackets as that is a git comment -Before every major release: +The changelog can be conveniently generated by the following: -* Update hardcoded, see [this pull request](https://github.com/gridcoin/Gridcoin-Research/pull/783) for an example. -* Update version of `contrib/gitian-descriptors/*.yml`: usually one'd want to do this on master after branching off the release - but be sure to at least do it before a new major release +#### Gridcoin maintainers/release engineers, suggestion for writing release notes -### First time / New builders - -If you're using the automated script (found in [contrib/gitian-build.sh](/contrib/gitian-build.sh)), then at this point you should run it with the "--setup" command. Otherwise ignore this. - -Check out the source code in the following directory hierarchy. - - cd /path/to/your/toplevel/build - git clone https://github.com/gridcoin-community/gitian.sigs.git - git clone https://github.com/bitcoin-core/bitcoin-detached-sigs.git - git clone https://github.com/devrandom/gitian-builder.git - git clone https://github.com/gridcoin/Gridcoin-Research.git - -### Gridcoin maintainers/release engineers, suggestion for writing release notes - -Write release notes. git shortlog helps a lot, for example: + - Write release notes. git shortlog helps a lot, for example: git shortlog --no-merges v(current version, e.g. 0.7.2)..v(new version, e.g. 0.8.0) -Generate list of authors: + - Generate list of authors: git log --format='%aN' "$*" | sort -ui | sed -e 's/^/- /' -Tag version (or release candidate) in git - - git tag -s v(new version, e.g. 0.8.0) - -### Setup and perform Gitian builds - -If you're using the automated script (found in [contrib/gitian-build.sh](/contrib/gitian-build.sh)), then at this point you should run it with the "--build" command. Otherwise ignore this. - -Setup Gitian descriptors: - - pushd ./Gridcoin-Research - export SIGNER=(your Gitian key, ie bluematt, sipa, etc) - export VERSION=(new version, e.g. 0.8.0) - git fetch - git checkout v${VERSION} - popd - -Ensure your gitian.sigs are up-to-date if you wish to gverify your builds against other Gitian signatures. - - pushd ./gitian.sigs - git pull - popd - -Ensure gitian-builder is up-to-date: - - pushd ./gitian-builder - git pull - popd - -### Fetch and create inputs: (first time, or when dependency versions change) - - pushd ./gitian-builder - mkdir -p inputs - wget -P inputs https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch - wget -P inputs http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz - popd - -Create the OS X SDK tarball, see the [OS X readme](README_osx.md) for details, and copy it into the inputs directory. - -### Optional: Seed the Gitian sources cache and offline git repositories - -By default, Gitian will fetch source files as needed. To cache them ahead of time: - - pushd ./gitian-builder - make -C ../Gridcoin-Research/depends download SOURCES_PATH=`pwd`/cache/common - popd - -Only missing files will be fetched, so this is safe to re-run for each build. - -NOTE: Offline builds must use the --url flag to ensure Gitian fetches only from local URLs. For example: - - pushd ./gitian-builder - ./bin/gbuild --url gridcoin=/path/to/gridcoin,signature=/path/to/sigs {rest of arguments} - popd - -The gbuild invocations below DO NOT DO THIS by default. - -### Build and sign Gridcoin Wallet for Linux, Windows, and OS X: - - pushd ./gitian-builder - ./bin/gbuild --num-make 2 --memory 3000 --commit gridcoin=v${VERSION} ../Gridcoin-Research/contrib/gitian-descriptors/gitian-linux.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../Gridcoin-Research/contrib/gitian-descriptors/gitian-linux.yml - mv build/out/gridcoin-*.tar.gz build/out/src/gridcoin-*.tar.gz ../ - - ./bin/gbuild --num-make 2 --memory 3000 --commit gridcoin=v${VERSION} ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-win-unsigned --destination ../gitian.sigs/ ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win.yml - mv build/out/gridcoin-*-win-unsigned.tar.gz inputs/gridcoin-win-unsigned.tar.gz - mv build/out/gridcoin-*.zip build/out/gridcoin-*.exe ../ - - ./bin/gbuild --num-make 2 --memory 3000 --commit gridcoin=v${VERSION} ../gridcoin/contrib/gitian-descriptors/gitian-osx.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../gridcoin/contrib/gitian-descriptors/gitian-osx.yml - mv build/out/gridcoin-*-osx-unsigned.tar.gz inputs/gridcoin-osx-unsigned.tar.gz - mv build/out/gridcoin-*.tar.gz build/out/gridcoin-*.dmg ../ - popd - -Build output expected: - - 1. source tarball (`gridcoin-${VERSION}.tar.gz`) - 2. linux 32-bit and 64-bit dist tarballs (`gridcoin-${VERSION}-linux[32|64].tar.gz`) - 3. Windows 32-bit and 64-bit unsigned installers and dist zips (`gridcoin-${VERSION}-win[32|64]-setup-unsigned.exe`, `gridcoin-${VERSION}-win[32|64].zip`) - 4. OS X unsigned installer and dist tarball (`gridcoin-${VERSION}-osx-unsigned.dmg`, `gridcoin-${VERSION}-osx64.tar.gz`) - 5. Gitian signatures (in `gitian.sigs/${VERSION}-/(your Gitian key)/`) - -### Verify other gitian builders signatures to your own. (Optional) - -Add other gitian builders keys to your gpg keyring, and/or refresh keys. - - gpg --import gridcoin/contrib/gitian-keys/*.pgp - gpg --refresh-keys - -Verify the signatures - - pushd ./gitian-builder - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-linux ../Gridcoin-Research/contrib/gitian-descriptors/gitian-linux.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-unsigned ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-unsigned ../Gridcoin-Research/contrib/gitian-descriptors/gitian-osx.yml - popd - -### Next steps: - -Commit your signature to gitian.sigs: - - pushd gitian.sigs - git add ${VERSION}-linux/${SIGNER} - git add ${VERSION}-win-unsigned/${SIGNER} - git add ${VERSION}-osx-unsigned/${SIGNER} - git commit -a - git push # Assuming you can push to the gitian.sigs tree - popd - -Codesigner only: Create Windows/OS X detached signatures: -- Only one person handles codesigning. Everyone else should skip to the next step. -- Only once the Windows/OS X builds each have 3 matching signatures may they be signed with their respective release keys. - -Codesigner only: Sign the osx binary: - - transfer gridcoin-osx-unsigned.tar.gz to osx for signing - tar xf gridcoin-osx-unsigned.tar.gz - ./detached-sig-create.sh -s "Key ID" - Enter the keychain password and authorize the signature - Move signature-osx.tar.gz back to the gitian host - -Codesigner only: Sign the Windows binaries: - - tar xf gridcoin-win-unsigned.tar.gz - ./detached-sig-create.sh -key /path/to/codesign.key - Enter the passphrase for the key when prompted - signature-win.tar.gz will be created - -Codesigner only: Commit the detached codesign payloads: - - cd ~/gridcoin-detached-sigs - checkout the appropriate branch for this release series - rm -rf * - tar xf signature-osx.tar.gz - tar xf signature-win.tar.gz - git add -a - git commit -m "point to ${VERSION}" - git tag -s v${VERSION} HEAD - git push the current branch and new tag - -Non-codesigners: wait for Windows/OS X detached signatures: - -- Once the Windows/OS X builds each have 3 matching signatures, they will be signed with their respective release keys. -- Detached signatures will then be committed to the [gridcoin-detached-sigs](https://github.com/bitcoin-core/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. - -Create (and optionally verify) the signed OS X binary: - - pushd ./gitian-builder - ./bin/gbuild -i --commit signature=v${VERSION} ../gridcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../gridcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../gridcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - mv build/out/gridcoin-osx-signed.dmg ../gridcoin-${VERSION}-osx.dmg - popd - -Create (and optionally verify) the signed Windows binaries: - - pushd ./gitian-builder - ./bin/gbuild -i --commit signature=v${VERSION} ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win-signer.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-win-signed --destination ../gitian.sigs/ ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win-signer.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-signed ../Gridcoin-Research/contrib/gitian-descriptors/gitian-win-signer.yml - mv build/out/gridcoin-*win64-setup.exe ../gridcoin-${VERSION}-win64-setup.exe - mv build/out/gridcoin-*win32-setup.exe ../gridcoin-${VERSION}-win32-setup.exe - popd - -Commit your signature for the signed OS X/Windows binaries: - - pushd gitian.sigs - git add ${VERSION}-osx-signed/${SIGNER} - git add ${VERSION}-win-signed/${SIGNER} - git commit -a - git push # Assuming you can push to the gitian.sigs tree - popd - -### After 3 or more people have gitian-built and their results match: - -- Create `SHA256SUMS.asc` for the builds, and GPG-sign it: - -```bash -sha256sum * > SHA256SUMS -``` - -The list of files should be: -``` -gridcoin-${VERSION}-aarch64-linux-gnu.tar.gz -gridcoin-${VERSION}-arm-linux-gnueabihf.tar.gz -gridcoin-${VERSION}-i686-pc-linux-gnu.tar.gz -gridcoin-${VERSION}-x86_64-linux-gnu.tar.gz -gridcoin-${VERSION}-osx64.tar.gz -gridcoin-${VERSION}-osx.dmg -gridcoin-${VERSION}.tar.gz -gridcoin-${VERSION}-win32-setup.exe -gridcoin-${VERSION}-win32.zip -gridcoin-${VERSION}-win64-setup.exe -gridcoin-${VERSION}-win64.zip -``` -The `*-debug*` files generated by the gitian build contain debug symbols -for troubleshooting by developers. It is assumed that anyone that is interested -in debugging can run gitian to generate the files for themselves. To avoid -end-user confusion about which file to pick, as well as save storage -space *do not upload these to the gridcoin.us server, nor put them in the torrent*. - -- GPG-sign it, delete the unsigned file: -``` -gpg --digest-algo sha256 --clearsign SHA256SUMS # outputs SHA256SUMS.asc -rm SHA256SUMS -``` - -- Announce the release: - - - Update title of #gridcoin on Freenode IRC - - - Twitter, reddit /r/Gridcoin and Slack - - - Create a [new GitHub release](https://github.com/gridcoin/Gridcoin-Research/releases/new) with a link to the archived release notes. - - - Celebrate + - git checkout master-test + - git merge master + - (Do test builds of master-test to do sanify checks of release. This can be used for actual release build of the Windows builds because it is at the same commit.) + - git checkout master + - git push + +### Tag + - git tag -s [version] # Paste previous commit message into this commit message + - git push --tags + +### Release + - Go to https://github.com/gridcoin-community/Gridcoin-Research/releases/new + - Tag version: [version] + - Release title: [version], + - Copy/paste changelog into Description + - Publish + +### Finalize + - git checkout development + - git merge master + - Edit configure.ac, increment version for development. Commit it. + - git push + + - git checkout testnet + - git merge development + - git push + +### Announce the release: + - Reddit /r/Gridcoin, Discord, Slack, Hive, and Twitter + - Update title of #gridcoin on IRC diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index fc71b583b7..f95b890812 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -14,10 +14,6 @@ disabled=( SC2086 # Double quote to prevent globbing and word splitting. SC2162 # read without -r will mangle backslashes. ) -disabled_gitian=( - SC2094 # Make sure not to read and write the same file in the same pipeline. - SC2129 # Consider using { cmd1; cmd2; } >> file instead of individual redirects. -) EXIT_CODE=0 @@ -32,22 +28,4 @@ if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/ EXIT_CODE=1 fi -if ! command -v yq > /dev/null; then - echo "Skipping Gitian descriptor scripts checking since yq is not installed." - exit $EXIT_CODE -fi - -EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" -for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') -do - script=$(basename "$descriptor") - # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. - echo "#!/bin/bash" > $script - yq -r .script "$descriptor" >> $script - if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE_GITIAN" $script; then - EXIT_CODE=1 - fi - rm $script -done - exit $EXIT_CODE