cd: generate release binaries using GitHub actions

.github/workflows/cd.yml

@@ -0,0 +1,35 @@
+name: CD
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        script-id: [win32, win64, mac]
+    env:
+      FILE_ENV: ./cd/00_setup_env_${{ matrix.script-id }}.sh
+      OS_NAME: linux
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+              ./depends/built
+              ./ci/scratch/.ccache
+          key: ${{ runner.os }}-${{ matrix.script-id }}
+      - name: build
+        run: |
+          ./cd/run_all.sh
+      - name: release
+        uses: softprops/action-gh-release@5e3f23f92c903aac25270f66388fdcb366c5b549
+        with:
+          files: '/tmp/release/*'

Makefile.am

@@ -66,6 +66,8 @@ $(BITCOIN_WIN_INSTALLER): all-recursive
 	@test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \
 	  echo error: could not build $@
 	@echo built $@
+	@rm release/gridcoinresearch.exe release/gridcoinresearchd.exe
+	@mv $@ $(top_builddir)/release/$@
 
 $(OSX_APP)/Contents/PkgInfo:
 	$(MKDIR_P) $(@D)
@@ -124,7 +126,9 @@ $(OSX_TEMP_ISO): $(APP_DIST_EXTRAS)
 	$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH))
 
 $(OSX_DMG): $(OSX_TEMP_ISO)
+	$(MKDIR_P) $(top_builddir)/release
 	$(DMG) dmg "$<" "$@"
+	@mv $@ $(top_builddir)/release/$(PACKAGE)-$(PACKAGE_VERSION)-macos.dmg
 
 dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG)
 	sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@

cd/00_setup_env.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+# The root dir.
+# The ci system copies this folder.
+# This is where the depends build is done.
+BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../ >/dev/null 2>&1 && pwd )
+export BASE_ROOT_DIR
+
+echo "Setting specific values in env"
+if [ -n "${FILE_ENV}" ]; then
+  set -o errexit;
+  # shellcheck disable=SC1090
+  source "${FILE_ENV}"
+fi
+
+echo "Fallback to default values in env (if not yet set)"
+# The number of parallel jobs to pass down to make and test_runner.py
+export MAKEJOBS=${MAKEJOBS:--j4}
+# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
+# This folder only exists on the ci host.
+export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch}
+# What host to compile for. See also ./depends/README.md
+# Tests that need cross-compilation export the appropriate HOST.
+# Tests that run natively guess the host
+export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
+# Whether to prefer BusyBox over GNU utilities
+export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
+export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed}
+export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04}
+# See man 7 debconf
+export DEBIAN_FRONTEND=noninteractive
+export CCACHE_SIZE=${CCACHE_SIZE:-100M}
+export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
+export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
+# The cache dir.
+# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
+export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
+# The depends dir.
+# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
+export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
+# Folder where the build result is put (bin and lib).
+export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
+# Folder where the build is done (dist and out-of-tree build).
+export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
+export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
+export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}
+export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
+export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"}

cd/00_setup_env_mac.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_macos_cross
+export DOCKER_NAME_TAG=ubuntu:20.04
+export HOST=x86_64-apple-darwin18
+export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools libtinfo5 python3-dev python3-setuptools xorriso"
+export XCODE_VERSION=12.1
+export XCODE_BUILD_ID=12A7403
+export GRIDCOIN_CONFIG="--with-gui --enable-reduce-exports"

cd/00_setup_env_win32.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_win32
+export DOCKER_NAME_TAG=ubuntu:20.04
+export HOST=i686-w64-mingw32
+export PACKAGES="python3 nsis g++-mingw-w64-i686 wine-binfmt wine32"
+export GOAL="deploy"
+export GRIDCOIN_CONFIG="--enable-reduce-exports --with-gui=qt5"
+export DPKG_ADD_ARCH="i386"

cd/00_setup_env_win64.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_win64
+export DOCKER_NAME_TAG=ubuntu:20.04
+export HOST=x86_64-w64-mingw32
+export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
+export GRIDCOIN_CONFIG="--enable-reduce-exports --with-gui=qt5"

cd/03_before_install.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+BEGIN_FOLD () {
+  echo ""
+  CURRENT_FOLD_NAME=$1
+  echo "::group::${CURRENT_FOLD_NAME}"
+}
+
+END_FOLD () {
+  RET=$?
+    echo "::endgroup::"
+  if [ $RET != 0 ]; then
+    echo "${CURRENT_FOLD_NAME} failed with status code ${RET}"
+  fi
+}
+

cd/04_install.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+if [[ $DOCKER_NAME_TAG == centos* ]]; then
+  export LC_ALL=en_US.utf8
+fi
+if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
+  export LC_ALL=C
+fi
+
+# Create folders that are mounted into the docker
+mkdir -p "${CCACHE_DIR}"
+mkdir -p "/tmp/release"
+
+env | grep -E '^(GRIDCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env
+
+export P_CI_DIR="$PWD"
+
+mkdir -p /tmp/release
+
+if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
+  echo "Creating $DOCKER_NAME_TAG container to run in"
+  ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
+
+  DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
+                  --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
+                  --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
+                  --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \
+                  --mount type=bind,src=/tmp/release,dst=/release \
+                  -w $BASE_ROOT_DIR \
+                  --env-file /tmp/env \
+                  --name $CONTAINER_NAME \
+                  $DOCKER_NAME_TAG)
+  export DOCKER_CI_CMD_PREFIX="docker exec $DOCKER_ID"
+else
+  echo "Running on host system without docker wrapper"
+fi
+
+DOCKER_EXEC () {
+  $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*"
+}
+export -f DOCKER_EXEC
+
+if [ -n "$DPKG_ADD_ARCH" ]; then
+  DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH"
+fi
+
+if [[ $DOCKER_NAME_TAG == centos* ]]; then
+  BEGIN_FOLD yum
+  ${CI_RETRY_EXE} DOCKER_EXEC yum -y install epel-release
+  ${CI_RETRY_EXE} DOCKER_EXEC yum -y install $DOCKER_PACKAGES $PACKAGES
+  END_FOLD
+elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
+  BEGIN_FOLD apt
+  ${CI_RETRY_EXE} DOCKER_EXEC apt-get update
+  ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
+  END_FOLD
+fi
+
+if [ "$OS_NAME" == "macos" ]; then
+  top -l 1 -s 0 | awk ' /PhysMem/ {print}'
+  echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
+else
+  DOCKER_EXEC free -m -h
+  DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
+  DOCKER_EXEC echo $(lscpu | grep Endian)
+  DOCKER_EXEC echo "Free disk space:"
+  DOCKER_EXEC df -h
+fi
+
+if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
+  echo "Create $BASE_ROOT_DIR"
+  DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR
+fi
+
+if [ "$USE_BUSY_BOX" = "true" ]; then
+  echo "Setup to use BusyBox utils"
+  DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/
+  # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)
+  # find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version)
+  # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed)
+  # shellcheck disable=SC1010
+  DOCKER_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) $BASE_SCRATCH_DIR/bins/\$util\; done
+  # Print BusyBox version
+  DOCKER_EXEC patch --help
+fi

cd/05_before_script.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources
+
+OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
+OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}"
+
+if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then
+  sudo curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
+fi
+
+if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then
+  DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
+fi
+if [[ $HOST = *-mingw32 ]]; then
+  DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)
+fi
+if [ -z "$NO_DEPENDS" ]; then
+  if [[ $DOCKER_NAME_TAG == centos* ]]; then
+    # CentOS has problems building the depends if the config shell is not explicitly set
+    # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to
+    #  an error as the first command is executed)
+    SHELL_OPTS="CONFIG_SHELL=/bin/bash"
+  else
+    SHELL_OPTS="CONFIG_SHELL="
+  fi
+  DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
+fi
+if [[ $HOST = x86_64-apple-darwin* ]]; then
+  DOCKER_EXEC "cd src/ && ../contrib/nomacro.pl"
+fi

cd/06_script_a.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+GRIDCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
+DOCKER_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE"
+
+DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/gridcoin-$HOST"
+export P_CI_DIR="${BASE_BUILD_DIR}/gridcoin-$HOST"
+
+BEGIN_FOLD autogen
+if [ -n "$CONFIG_SHELL" ]; then
+  DOCKER_EXEC "$CONFIG_SHELL" -c "{BASE_ROOT_DIR}/autogen.sh"
+else
+  DOCKER_EXEC ${BASE_ROOT_DIR}/autogen.sh
+fi
+END_FOLD
+
+BEGIN_FOLD configure
+DOCKER_EXEC ${BASE_ROOT_DIR}/configure $GRIDCOIN_CONFIG_ALL $GRIDCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false)
+END_FOLD
+
+BEGIN_FOLD build
+DOCKER_EXEC make $MAKEJOBS deploy || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make deploy V=1 ; false )
+END_FOLD
+
+BEGIN_FOLD cache_stats
+DOCKER_EXEC "ccache --version | head -n 1 && ccache --show-stats"
+DOCKER_EXEC du -sh "${DEPENDS_DIR}"/*/
+END_FOLD

cd/06_script_b.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2021 The Gridcoin developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+DOCKER_EXEC mv release/* /release
+
+if [[ $HOST = *-apple-* ]]; then
+	KV=$(cat $BASE_ROOT_DIR/depends/hosts/darwin.mk | grep "OSX_MIN_VERSION=")
+	VER=${KV#OSX_MIN_VERSION=}
+	for f in /tmp/release/*.dmg; do
+		mv $f ${f%.dmg}-min-$VER.dmg
+	done
+fi
+
+cd /tmp/release/
+for f in *; do
+    sha256sum $f > $f.SHA256
+done

cd/run_all.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+set -o errexit; source ./cd/00_setup_env.sh
+set -o errexit; source ./cd/03_before_install.sh
+set -o errexit; source ./cd/04_install.sh
+set -o errexit; source ./cd/05_before_script.sh
+set -o errexit; source ./cd/06_script_a.sh
+set -o errexit; source ./cd/06_script_b.sh