Merge pull request #77038 from rail/bincheck

release: enable bincheck for tag pushes

.github/workflows/bincheck.yml

@@ -0,0 +1,26 @@
+name: bincheck
+
+on:
+  push:
+    tags: [ v* ]
+
+jobs:
+
+  linux:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: sudo apt-get update && sudo apt-get install -y qemu-system-x86
+    - run: cd build/release/bincheck && ./test-linux ${{ github.ref_name }} ${{ github.sha }}
+
+  darwin:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: cd build/release/bincheck && ./test-macos ${{ github.ref_name }} ${{ github.sha }}
+
+  windows:
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: cd build/release/bincheck && bash test-windows ${{ github.ref_name }} ${{ github.sha }}

build/release/bincheck/.gitignore

@@ -0,0 +1,10 @@
+# files created by running cockroach
+cockroach-pid
+cockroach-data/
+
+# files created running bincheck
+cockroach.tar.gz
+aes-128.key
+mnt/
+actual
+expected

build/release/bincheck/README.md

@@ -0,0 +1,67 @@
+# bincheck
+
+bincheck verifies the sanity of CockroachDB release binaries. At present, the
+sanity checks are:
+
+  * starting a one-node server and running a simple SQL query, and
+  * verifying the output of `cockroach version`.
+
+## Testing a new release
+
+bincheck action is triggered when a new tag with `v` prefix is created. Check
+https://github.com/cockroachdb/cockroach/actions after a release is published.
+
+
+## The nitty-gritty
+
+### Overview
+
+For the MacOS and Windows binaries, the mechanics involved are simple. We ask
+GitHub Actions to spin us up a MacOS or Windows runner, download the
+appropriate pre-built `cockroach` binary, and run our sanity checks.
+
+Linux is more complicated, not out of necessity, but out of ambition. We co-opt
+the Linux verification step to additionally test support for pre-SSE4.2
+CPUs<sup>†</sup>. This requires emulating such a CPU, and Linux is the only
+operating system that is feasible to run under emulation. Windows and MacOS have
+prohibitively slow boot times, and, perhaps more importantly, Windows and MacOS
+install media are not freely available.
+
+So, with the help of [Buildroot], an embedded Linux build manager, this
+repository ships an [8.7MB Linux distribution][linux-image] that's capable of
+running under [QEMU] and launching our pre-built CockroachDB binaries. To verify
+the Linux binaries, we first boot this Linux distribution on an emulated
+pre-SSE4.2 CPU (`qemu-system-x86_64 -cpu qemu64,-sse4.2`), then run our sanity
+checks from inside this VM.
+
+<sup>†</sup><small>SSE4.2 support is particularly important in CockroachDB,
+since RocksDB internally uses a [CRC32C checksum][crc32c] to protect against
+data corruption. SSE4.2 includes hardware support for computing CRC32C
+checksums, but is only available on CPUs released after November 2008. Producing
+a binary that can dynamically switch between the SSE4.2 and non-SSE4.2
+implementations at runtime [has proven difficult][issue-15589].
+</small>
+
+### Updating the Linux image
+
+After [installing Buildroot][buildroot-install]:
+
+```shell
+$ make qemu_x86_64_glibc_defconfig BR2_EXTERNAL=buildroot
+$ make menuconfig  # Only if configuration changes are necessary.
+$ make
+$ cp output/images/bzImage images/qemu_x86_64_glibc_bzImage
+```
+
+At the time of writing, `qemu_x86_64_glibc_defconfig` instructed Buildroot to
+build a Linux 4.9 kernel, install Bash and OpenSSH into userland, and configure
+sshd to boot at startup and allow passwordless `root` authentication.
+`/etc/fstab` is modified to mount the first hard drive at `/bincheck`, assuming
+it's a FAT volume.
+
+[buildroot-install]: https://buildroot.org/download.html
+[issue-15589]: https://github.com/cockroachdb/cockroach/issues/15589
+[linux-image]: ./images/qemu_x86_64_glibc_bzImage
+[Buildroot]: https://buildroot.org
+[CRC32C]: http://www.evanjones.ca/crc32c.html
+[QEMU]: http://qemu.org

build/release/bincheck/bincheck

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export COCKROACH_INCONSISTENT_TIME_ZONES=true
+
+# Verify arguments.
+if [[ $# -ne 3 ]]
+then
+  echo "usage: $0 COCKROACH-BINARY EXPECTED-VERSION EXPECTED-SHA" >&2
+  exit 1
+fi
+
+readonly cockroach="$1"
+readonly version="$2"
+readonly sha="$3"
+readonly urlfile=cockroach-url
+
+# Display build information.
+echo ""
+"$cockroach" version
+echo ""
+
+# Start a CockroachDB server, wait for it to become ready, and arrange for it to
+# be force-killed when the script exits.
+rm -f "$urlfile"
+
+# Generate encryption key.
+echo "Generating encryption key:"
+"$cockroach" gen encryption-key aes-128.key
+echo ""
+
+# Start node with encryption enabled.
+"$cockroach" start-single-node --insecure --listening-url-file="$urlfile" --enterprise-encryption=path=cockroach-data,key=aes-128.key,old-key=plain &
+
+trap "kill -9 $! &> /dev/null" EXIT
+for i in {0..3}
+do
+  [[ -f "$urlfile" ]] && break
+  backoff=$((2 ** i))
+  echo "server not yet available; sleeping for $backoff seconds"
+  sleep $backoff
+done
+
+# Verify the output of a simple SQL query.
+"$cockroach" sql --insecure <<EOF
+  CREATE DATABASE bank;
+  CREATE TABLE bank.accounts (id INT PRIMARY KEY, balance DECIMAL);
+  INSERT INTO bank.accounts VALUES (1, 1000.50);
+EOF
+cat > expected <<EOF
+id	balance
+1	1000.50
+EOF
+"$cockroach" sql --insecure -e 'SELECT * FROM bank.accounts' > actual
+diff -u expected actual
+
+# Attempt a clean shutdown for good measure. We'll force-kill in the atexit
+# handler if this fails.
+"$cockroach" quit --insecure
+trap - EXIT
+
+# Verify reported version matches expected version.
+echo "$version" > expected
+"$cockroach" version | grep 'Build Tag' | cut -f2 -d: | tr -d ' ' > actual
+diff -u expected actual
+
+# Verify reported SHA matches expected SHA.
+echo "$sha" > expected
+"$cockroach" version | grep 'Build Commit ID' | cut -f2 -d: | tr -d ' ' > actual
+diff -u expected actual

build/release/bincheck/buildroot.patch

@@ -0,0 +1,35 @@
+diff --git a/package/ncurses/Config.in b/package/ncurses/Config.in
+index 92be164..b3333f1 100644
+--- a/package/ncurses/Config.in
++++ b/package/ncurses/Config.in
+@@ -23,4 +23,9 @@ config BR2_PACKAGE_NCURSES_TARGET_PROGS
+ 	help
+ 	  Include ncurses programs in target (clear, reset, tput, ...)
+
++config BR2_PACKAGE_NCURSES_TERMLIB
++	bool "separate terminfo library"
++	help
++	  Build a separate terminfo library (libtinfo)
++
+ endif
+diff --git a/package/ncurses/ncurses.mk b/package/ncurses/ncurses.mk
+index 94c8c9a..c1b61aa 100644
+--- a/package/ncurses/ncurses.mk
++++ b/package/ncurses/ncurses.mk
+@@ -27,6 +27,7 @@ NCURSES_CONF_OPTS = \
+ 	--enable-pc-files \
+ 	--with-pkg-config-libdir="/usr/lib/pkgconfig" \
+ 	$(if $(BR2_PACKAGE_NCURSES_TARGET_PROGS),,--without-progs) \
++	$(if $(BR2_PACKAGE_NCURSES_TERMLIB),--with-termlib) \
+ 	--without-manpages
+
+ ifeq ($(BR2_STATIC_LIBS),y)
+@@ -65,7 +66,7 @@ NCURSES_TERMINFO_FILES = \
+ ifeq ($(BR2_PACKAGE_NCURSES_WCHAR),y)
+ NCURSES_CONF_OPTS += --enable-widec
+ NCURSES_LIB_SUFFIX = w
+-NCURSES_LIBS = ncurses menu panel form
++NCURSES_LIBS = ncurses menu panel form $(if $(BR2_PACKAGE_NCURSES_TERMLIB),tinfo)
+
+ define NCURSES_LINK_LIBS_STATIC
+ 	$(foreach lib,$(NCURSES_LIBS:%=lib%), \

build/release/bincheck/buildroot/board/qemu/common/fs-overlay/etc/ssh/sshd_config

@@ -0,0 +1,3 @@
+PasswordAuthentication yes
+PermitEmptyPasswords yes
+PermitRootLogin yes

build/release/bincheck/buildroot/board/qemu/common/post-build.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+fstab=$TARGET_DIR/etc/fstab
+grep -q /bincheck "$fstab" || cat >> "$fstab" <<EOF
+/dev/sda1	/bincheck	vfat	defaults	0	0
+EOF

build/release/bincheck/buildroot/board/qemu/x86_64_glibc/linux.config

@@ -0,0 +1,52 @@
+CONFIG_SYSVIPC=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_INITRAMFS_SOURCE="${BR_BINARIES_DIR}/rootfs.cpio"
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_SMP=y
+CONFIG_HYPERVISOR_GUEST=y
+CONFIG_PARAVIRT=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_INET=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_VIRTIO_BLK=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_SCSI_VIRTIO=y
+CONFIG_ATA=y
+CONFIG_ATA_PIIX=y
+CONFIG_NETDEVICES=y
+CONFIG_VIRTIO_NET=y
+CONFIG_NE2K_PCI=y
+CONFIG_8139CP=y
+CONFIG_INPUT_EVDEV=y
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_HW_RANDOM_VIRTIO=m
+CONFIG_DRM=y
+CONFIG_DRM_QXL=y
+CONFIG_DRM_BOCHS=y
+CONFIG_DRM_VIRTIO_GPU=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HDA_INTEL=y
+CONFIG_SND_HDA_GENERIC=y
+CONFIG_USB=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_UHCI_HCD=y
+CONFIG_USB_STORAGE=y
+CONFIG_VIRTIO_PCI=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_INPUT=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
+CONFIG_EXT4_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=y

build/release/bincheck/buildroot/configs/qemu_x86_64_glibc_defconfig

@@ -0,0 +1,27 @@
+# Configuration derived from upstream qemu_x86_64_defconfig.
+
+# Architecture.
+BR2_x86_64=y
+
+# Toolchain.
+BR2_TOOLCHAIN_EXTERNAL=y
+
+# System.
+BR2_SYSTEM_BIN_SH_BASH=y
+BR2_SYSTEM_DHCP="eth0"
+BR2_TARGET_GENERIC_GETTY_PORT="tty1"
+
+# Filesystem.
+BR2_TARGET_ROOTFS_INITRAMFS=y
+BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_COCKROACH_BINCHECK_PATH)/board/qemu/common/fs-overlay"
+BR2_ROOTFS_POST_BUILD_SCRIPT="$(BR2_EXTERNAL_COCKROACH_BINCHECK_PATH)/board/qemu/common/post-build.sh"
+
+# Kernel.
+BR2_LINUX_KERNEL=y
+BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y
+BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="$(BR2_EXTERNAL_COCKROACH_BINCHECK_PATH)/board/qemu/x86_64_glibc/linux.config"
+
+# Userland.
+BR2_PACKAGE_BUSYBOX_SHOW_OTHERS=y
+BR2_PACKAGE_OPENSSH=y
+BR2_PACKAGE_NCURSES_TERMLIB=y

build/release/bincheck/buildroot/external.desc

@@ -0,0 +1,2 @@
+name: COCKROACH_BINCHECK
+desc: Linux images used to verify pre-built CockroachDB binaries.

build/release/bincheck/download_binary.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+download_and_extract() {
+  cockroach_version=$1
+  binary_suffix=$2
+  binary_source="https://binaries.cockroachdb.com"
+  binary_url="${binary_source}/cockroach-${cockroach_version}.${binary_suffix}"
+
+  mkdir -p mnt
+
+  # Check if this is a tarball or zip.
+  if [[ "${binary_suffix}" == *.tgz ]]; then
+    curl -sSfL "${binary_url}" > cockroach.tar.gz
+    tar zxf cockroach.tar.gz -C mnt --strip-components=1
+  else
+    curl -sSfL "${binary_url}" > cockroach.zip
+    7z e -omnt cockroach.zip
+  fi
+
+  echo "Downloaded ${binary_url}"
+}

build/release/bincheck/test-linux

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+source ./download_binary.sh
+
+if [[ $# -ne 2 ]]
+then
+  echo "usage: $0 EXPECTED-VERSION EXPECTED-SHA" >&2
+  exit 1
+fi
+
+COCKROACH_VERSION=$1
+COCKROACH_SHA=$2
+
+download_and_extract "$COCKROACH_VERSION" "linux-amd64.tgz"
+
+ssh() {
+  command ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+    root@localhost -p 2222 "$@"
+}
+
+qemu-system-x86_64 \
+  -cpu qemu64,-sse4.2 \
+  -m 1G \
+  -kernel images/qemu_x86_64_glibc_bzImage \
+  -net nic,model=virtio -net user,hostfwd=tcp::2222-:22 \
+  -drive file=fat:rw:mnt,format=raw \
+  -nographic &
+
+trap "kill -9 $! &> /dev/null" EXIT
+
+for i in {0..4}
+do
+  ssh true && break
+  backoff=$((2 ** i))
+  echo "VM not yet available; sleeping for $backoff seconds"
+  sleep $backoff
+done
+
+ssh /bin/bash -s /bincheck/cockroach "$COCKROACH_VERSION" "$COCKROACH_SHA" < bincheck

build/release/bincheck/test-macos

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+source ./download_binary.sh
+
+# Verify arguments.
+if [[ $# -ne 2 ]]
+then
+  echo "usage: $0 EXPECTED-VERSION EXPECTED-SHA" >&2
+  exit 1
+fi
+
+COCKROACH_VERSION=$1
+COCKROACH_SHA=$2
+
+download_and_extract "$COCKROACH_VERSION" "darwin-10.9-amd64.tgz"
+./bincheck ./mnt/cockroach "$COCKROACH_VERSION" "$COCKROACH_SHA"