diff --git a/.github/workflows/bincheck.yml b/.github/workflows/bincheck.yml new file mode 100644 index 000000000000..45f85c07b2c5 --- /dev/null +++ b/.github/workflows/bincheck.yml @@ -0,0 +1,26 @@ +name: bincheck + +on: + push: + tags: [ v* ] + +jobs: + + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: sudo apt-get update && sudo apt-get install -y qemu-system-x86 + - run: cd build/release/bincheck && ./test-linux ${{ github.ref_name }} ${{ github.sha }} + + darwin: + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - run: cd build/release/bincheck && ./test-macos ${{ github.ref_name }} ${{ github.sha }} + + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - run: cd build/release/bincheck && bash test-windows ${{ github.ref_name }} ${{ github.sha }} diff --git a/build/release/bincheck/.gitignore b/build/release/bincheck/.gitignore new file mode 100644 index 000000000000..a57ea47eddf9 --- /dev/null +++ b/build/release/bincheck/.gitignore @@ -0,0 +1,10 @@ +# files created by running cockroach +cockroach-pid +cockroach-data/ + +# files created running bincheck +cockroach.tar.gz +aes-128.key +mnt/ +actual +expected diff --git a/build/release/bincheck/README.md b/build/release/bincheck/README.md new file mode 100644 index 000000000000..775ca7a3bb23 --- /dev/null +++ b/build/release/bincheck/README.md @@ -0,0 +1,67 @@ +# bincheck + +bincheck verifies the sanity of CockroachDB release binaries. At present, the +sanity checks are: + + * starting a one-node server and running a simple SQL query, and + * verifying the output of `cockroach version`. + +## Testing a new release + +bincheck action is triggered when a new tag with `v` prefix is created. Check +https://github.com/cockroachdb/cockroach/actions after a release is published. + + +## The nitty-gritty + +### Overview + +For the MacOS and Windows binaries, the mechanics involved are simple. We ask +GitHub Actions to spin us up a MacOS or Windows runner, download the +appropriate pre-built `cockroach` binary, and run our sanity checks. + +Linux is more complicated, not out of necessity, but out of ambition. We co-opt +the Linux verification step to additionally test support for pre-SSE4.2 +CPUs. This requires emulating such a CPU, and Linux is the only +operating system that is feasible to run under emulation. Windows and MacOS have +prohibitively slow boot times, and, perhaps more importantly, Windows and MacOS +install media are not freely available. + +So, with the help of [Buildroot], an embedded Linux build manager, this +repository ships an [8.7MB Linux distribution][linux-image] that's capable of +running under [QEMU] and launching our pre-built CockroachDB binaries. To verify +the Linux binaries, we first boot this Linux distribution on an emulated +pre-SSE4.2 CPU (`qemu-system-x86_64 -cpu qemu64,-sse4.2`), then run our sanity +checks from inside this VM. + +SSE4.2 support is particularly important in CockroachDB, +since RocksDB internally uses a [CRC32C checksum][crc32c] to protect against +data corruption. SSE4.2 includes hardware support for computing CRC32C +checksums, but is only available on CPUs released after November 2008. Producing +a binary that can dynamically switch between the SSE4.2 and non-SSE4.2 +implementations at runtime [has proven difficult][issue-15589]. + + +### Updating the Linux image + +After [installing Buildroot][buildroot-install]: + +```shell +$ make qemu_x86_64_glibc_defconfig BR2_EXTERNAL=buildroot +$ make menuconfig # Only if configuration changes are necessary. +$ make +$ cp output/images/bzImage images/qemu_x86_64_glibc_bzImage +``` + +At the time of writing, `qemu_x86_64_glibc_defconfig` instructed Buildroot to +build a Linux 4.9 kernel, install Bash and OpenSSH into userland, and configure +sshd to boot at startup and allow passwordless `root` authentication. +`/etc/fstab` is modified to mount the first hard drive at `/bincheck`, assuming +it's a FAT volume. + +[buildroot-install]: https://buildroot.org/download.html +[issue-15589]: https://github.com/cockroachdb/cockroach/issues/15589 +[linux-image]: ./images/qemu_x86_64_glibc_bzImage +[Buildroot]: https://buildroot.org +[CRC32C]: http://www.evanjones.ca/crc32c.html +[QEMU]: http://qemu.org diff --git a/build/release/bincheck/bincheck b/build/release/bincheck/bincheck new file mode 100755 index 000000000000..8b3f1db1297f --- /dev/null +++ b/build/release/bincheck/bincheck @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export COCKROACH_INCONSISTENT_TIME_ZONES=true + +# Verify arguments. +if [[ $# -ne 3 ]] +then + echo "usage: $0 COCKROACH-BINARY EXPECTED-VERSION EXPECTED-SHA" >&2 + exit 1 +fi + +readonly cockroach="$1" +readonly version="$2" +readonly sha="$3" +readonly urlfile=cockroach-url + +# Display build information. +echo "" +"$cockroach" version +echo "" + +# Start a CockroachDB server, wait for it to become ready, and arrange for it to +# be force-killed when the script exits. +rm -f "$urlfile" + +# Generate encryption key. +echo "Generating encryption key:" +"$cockroach" gen encryption-key aes-128.key +echo "" + +# Start node with encryption enabled. +"$cockroach" start-single-node --insecure --listening-url-file="$urlfile" --enterprise-encryption=path=cockroach-data,key=aes-128.key,old-key=plain & + +trap "kill -9 $! &> /dev/null" EXIT +for i in {0..3} +do + [[ -f "$urlfile" ]] && break + backoff=$((2 ** i)) + echo "server not yet available; sleeping for $backoff seconds" + sleep $backoff +done + +# Verify the output of a simple SQL query. +"$cockroach" sql --insecure < expected < actual +diff -u expected actual + +# Attempt a clean shutdown for good measure. We'll force-kill in the atexit +# handler if this fails. +"$cockroach" quit --insecure +trap - EXIT + +# Verify reported version matches expected version. +echo "$version" > expected +"$cockroach" version | grep 'Build Tag' | cut -f2 -d: | tr -d ' ' > actual +diff -u expected actual + +# Verify reported SHA matches expected SHA. +echo "$sha" > expected +"$cockroach" version | grep 'Build Commit ID' | cut -f2 -d: | tr -d ' ' > actual +diff -u expected actual diff --git a/build/release/bincheck/buildroot.patch b/build/release/bincheck/buildroot.patch new file mode 100644 index 000000000000..abaa744445ba --- /dev/null +++ b/build/release/bincheck/buildroot.patch @@ -0,0 +1,35 @@ +diff --git a/package/ncurses/Config.in b/package/ncurses/Config.in +index 92be164..b3333f1 100644 +--- a/package/ncurses/Config.in ++++ b/package/ncurses/Config.in +@@ -23,4 +23,9 @@ config BR2_PACKAGE_NCURSES_TARGET_PROGS + help + Include ncurses programs in target (clear, reset, tput, ...) + ++config BR2_PACKAGE_NCURSES_TERMLIB ++ bool "separate terminfo library" ++ help ++ Build a separate terminfo library (libtinfo) ++ + endif +diff --git a/package/ncurses/ncurses.mk b/package/ncurses/ncurses.mk +index 94c8c9a..c1b61aa 100644 +--- a/package/ncurses/ncurses.mk ++++ b/package/ncurses/ncurses.mk +@@ -27,6 +27,7 @@ NCURSES_CONF_OPTS = \ + --enable-pc-files \ + --with-pkg-config-libdir="/usr/lib/pkgconfig" \ + $(if $(BR2_PACKAGE_NCURSES_TARGET_PROGS),,--without-progs) \ ++ $(if $(BR2_PACKAGE_NCURSES_TERMLIB),--with-termlib) \ + --without-manpages + + ifeq ($(BR2_STATIC_LIBS),y) +@@ -65,7 +66,7 @@ NCURSES_TERMINFO_FILES = \ + ifeq ($(BR2_PACKAGE_NCURSES_WCHAR),y) + NCURSES_CONF_OPTS += --enable-widec + NCURSES_LIB_SUFFIX = w +-NCURSES_LIBS = ncurses menu panel form ++NCURSES_LIBS = ncurses menu panel form $(if $(BR2_PACKAGE_NCURSES_TERMLIB),tinfo) + + define NCURSES_LINK_LIBS_STATIC + $(foreach lib,$(NCURSES_LIBS:%=lib%), \ diff --git a/build/release/bincheck/buildroot/Config.in b/build/release/bincheck/buildroot/Config.in new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/build/release/bincheck/buildroot/board/qemu/common/fs-overlay/bincheck/.keep b/build/release/bincheck/buildroot/board/qemu/common/fs-overlay/bincheck/.keep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/build/release/bincheck/buildroot/board/qemu/common/fs-overlay/etc/ssh/sshd_config b/build/release/bincheck/buildroot/board/qemu/common/fs-overlay/etc/ssh/sshd_config new file mode 100644 index 000000000000..5ae17a5a4d05 --- /dev/null +++ b/build/release/bincheck/buildroot/board/qemu/common/fs-overlay/etc/ssh/sshd_config @@ -0,0 +1,3 @@ +PasswordAuthentication yes +PermitEmptyPasswords yes +PermitRootLogin yes diff --git a/build/release/bincheck/buildroot/board/qemu/common/post-build.sh b/build/release/bincheck/buildroot/board/qemu/common/post-build.sh new file mode 100755 index 000000000000..6a4758d7d3ea --- /dev/null +++ b/build/release/bincheck/buildroot/board/qemu/common/post-build.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +fstab=$TARGET_DIR/etc/fstab +grep -q /bincheck "$fstab" || cat >> "$fstab" < cockroach.tar.gz + tar zxf cockroach.tar.gz -C mnt --strip-components=1 + else + curl -sSfL "${binary_url}" > cockroach.zip + 7z e -omnt cockroach.zip + fi + + echo "Downloaded ${binary_url}" +} diff --git a/build/release/bincheck/images/qemu_x86_64_glibc_bzImage b/build/release/bincheck/images/qemu_x86_64_glibc_bzImage new file mode 100644 index 000000000000..35a36250119f Binary files /dev/null and b/build/release/bincheck/images/qemu_x86_64_glibc_bzImage differ diff --git a/build/release/bincheck/test-linux b/build/release/bincheck/test-linux new file mode 100755 index 000000000000..1415c83befe3 --- /dev/null +++ b/build/release/bincheck/test-linux @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail +source ./download_binary.sh + +if [[ $# -ne 2 ]] +then + echo "usage: $0 EXPECTED-VERSION EXPECTED-SHA" >&2 + exit 1 +fi + +COCKROACH_VERSION=$1 +COCKROACH_SHA=$2 + +download_and_extract "$COCKROACH_VERSION" "linux-amd64.tgz" + +ssh() { + command ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + root@localhost -p 2222 "$@" +} + +qemu-system-x86_64 \ + -cpu qemu64,-sse4.2 \ + -m 1G \ + -kernel images/qemu_x86_64_glibc_bzImage \ + -net nic,model=virtio -net user,hostfwd=tcp::2222-:22 \ + -drive file=fat:rw:mnt,format=raw \ + -nographic & + +trap "kill -9 $! &> /dev/null" EXIT + +for i in {0..4} +do + ssh true && break + backoff=$((2 ** i)) + echo "VM not yet available; sleeping for $backoff seconds" + sleep $backoff +done + +ssh /bin/bash -s /bincheck/cockroach "$COCKROACH_VERSION" "$COCKROACH_SHA" < bincheck diff --git a/build/release/bincheck/test-macos b/build/release/bincheck/test-macos new file mode 100755 index 000000000000..a1469ea61c2e --- /dev/null +++ b/build/release/bincheck/test-macos @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail +source ./download_binary.sh + +# Verify arguments. +if [[ $# -ne 2 ]] +then + echo "usage: $0 EXPECTED-VERSION EXPECTED-SHA" >&2 + exit 1 +fi + +COCKROACH_VERSION=$1 +COCKROACH_SHA=$2 + +download_and_extract "$COCKROACH_VERSION" "darwin-10.9-amd64.tgz" +./bincheck ./mnt/cockroach "$COCKROACH_VERSION" "$COCKROACH_SHA" diff --git a/build/release/bincheck/test-windows b/build/release/bincheck/test-windows new file mode 100644 index 000000000000..5a219182c90e --- /dev/null +++ b/build/release/bincheck/test-windows @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail +source ./download_binary.sh + +if [[ $# -ne 2 ]] +then + echo "usage: $0 EXPECTED-VERSION EXPECTED-SHA" >&2 + exit 1 +fi + +COCKROACH_VERSION=$1 +COCKROACH_SHA=$2 + +download_and_extract "$COCKROACH_VERSION" "windows-6.2-amd64.zip" +./bincheck ./mnt/cockroach.exe "$COCKROACH_VERSION" "$COCKROACH_SHA"