From 9e5f614c88f1ff63a8c2de285f28a26904cd0412 Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Thu, 7 Mar 2019 22:24:48 +0000 Subject: [PATCH] feat(system): Include opentrons api server from external tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Adds a BR2_EXTERNAL tree from the monorepo (https://github.com/Opentrons/opentrons/pull/3217) and pulls in the opentrons api server to the build. - Fixes some issues in systemd services and targets. Codebuild rewrites broken (and maybe non-broken) absolute links to fit inside its chroot jail, which was breaking the absolute links that aren’t supposed to be valid until they’re on the robot - Add a VERSIONS file taken from a combination of the buildroot version (from git) and opentrons api server versions (from package.json) and save to /etc/VERSIONS.json - VERSIONS.json is a new artifact and also zipped up with the full images and update images when uploaded to s3 Closes https://github.com/Opentrons/opentrons/issues/2877 --- .gitignore | 1 + Dockerfile | 2 +- board/opentrons/ot2/post-build.sh | 6 ++ board/opentrons/ot2/post-image.sh | 8 +++ .../opentrons/ot2/rootfs-overlay/etc/dropbear | 2 +- .../ot2/rootfs-overlay/etc/nginx/nginx.conf | 2 +- .../opentrons-init-connection.service | 1 - .../opentrons-init-connections.service | 1 + .../etc/systemd/system/default.target | 2 +- .../system/opentrons-init-connections.service | 4 +- .../system/opentrons-run-boot-scripts.service | 4 +- .../jupyter-notebook.service | 2 +- .../opentrons-run-boot-scripts.service | 2 +- board/opentrons/ot2/write_version.py | 56 +++++++++++++++++++ buildspec.yml | 10 ++-- configs/ot2_defconfig | 3 + in_docker.sh | 4 +- opentrons-build.sh | 9 ++- 18 files changed, 100 insertions(+), 19 deletions(-) delete mode 120000 board/opentrons/ot2/rootfs-overlay/etc/systemd/system/NetworkManager.service.wants/opentrons-init-connection.service create mode 120000 board/opentrons/ot2/rootfs-overlay/etc/systemd/system/NetworkManager.service.wants/opentrons-init-connections.service create mode 100644 board/opentrons/ot2/write_version.py diff --git a/.gitignore b/.gitignore index bb02d9f572..179466dc29 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ *.rej *~ *.pyc +.env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c30f04a892..a161fe906f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG filter_output VOLUME /buildroot RUN apt-get -y update &&\ - apt-get -y install build-essential wget file cpio python rsync unzip bc libncurses-dev + apt-get -y install build-essential wget file cpio python rsync unzip bc libncurses-dev git RUN wget http://crosstool-ng.org/download/crosstool-ng/crosstool-ng-1.23.0.tar.xz &&\ tar xf ./crosstool-* &&\ diff --git a/board/opentrons/ot2/post-build.sh b/board/opentrons/ot2/post-build.sh index 8b1ca3ed70..05cd1e04b0 100755 --- a/board/opentrons/ot2/post-build.sh +++ b/board/opentrons/ot2/post-build.sh @@ -35,3 +35,9 @@ sed -i s/kernel=zImage/kernel=u-boot.bin/ ${BINARIES_DIR}/rpi-firmware/config.tx # write common pubkey to authorized keys # TODO: DO NOT DO THIS IN RELEASE BUILDS cat ${TARGET_DIR}/var/home/.ssh/robot_key.pub > ${TARGET_DIR}/var/home/.ssh/authorized_keys + +# Load the out-of-container env to get stuff from codebuild +export $(cat /buildroot/.env | xargs) + +python ./board/opentrons/ot2/write_version.py ${BINARIES_DIR}/opentrons-api-version.json ${BINARIES_DIR}/VERSION.json +cp ${BINARIES_DIR}/VERSION.json ${TARGET_DIR}/etc/VERSION.json diff --git a/board/opentrons/ot2/post-image.sh b/board/opentrons/ot2/post-image.sh index bbdb29e4c7..c2e946fa18 100755 --- a/board/opentrons/ot2/post-image.sh +++ b/board/opentrons/ot2/post-image.sh @@ -7,6 +7,8 @@ BOARD_NAME="$(basename ${BOARD_DIR})" GENIMAGE_CFG="${BOARD_DIR}/genimage-${BOARD_NAME}.cfg" GENIMAGE_TMP="${BUILD_DIR}/genimage.tmp" +sed -i -e 's/console=ttyAMA0,115200//' "${BINARIES_DIR}/rpi-firmware/cmdline.txt" + for arg in "$@" do case "${arg}" in @@ -61,4 +63,10 @@ genimage \ --outputpath "${BINARIES_DIR}" \ --config "${GENIMAGE_CFG}" +rm -f ${BINARIES_DIR}/ot2-system.zip +zip -j ${BINARIES_DIR}/ot2-system.zip ${BINARIES_DIR}/rootfs.ext4 ${BINARIES_DIR}/VERSION.json +rm -f ${BINARIES_DIR}/ot2-fullimage.zip +zip -j ${BINARIES_DIR}/ot2-fullimage.zip ${BINARIES_DIR}/sdcard.img ${BINARIES_DIR}/VERSION.json + + exit $? diff --git a/board/opentrons/ot2/rootfs-overlay/etc/dropbear b/board/opentrons/ot2/rootfs-overlay/etc/dropbear index 33a215760f..91ecc14f91 120000 --- a/board/opentrons/ot2/rootfs-overlay/etc/dropbear +++ b/board/opentrons/ot2/rootfs-overlay/etc/dropbear @@ -1 +1 @@ -/var/run/dropbear \ No newline at end of file +../var/run/dropbear \ No newline at end of file diff --git a/board/opentrons/ot2/rootfs-overlay/etc/nginx/nginx.conf b/board/opentrons/ot2/rootfs-overlay/etc/nginx/nginx.conf index cc3744e44e..847442802d 100644 --- a/board/opentrons/ot2/rootfs-overlay/etc/nginx/nginx.conf +++ b/board/opentrons/ot2/rootfs-overlay/etc/nginx/nginx.conf @@ -29,7 +29,7 @@ http { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; proxy_read_timeout 1h; - proxy_pass http://unix:/tmp/aiohttp.sock; + proxy_pass http://unix:/run/aiohttp.sock; } location /server { diff --git a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/NetworkManager.service.wants/opentrons-init-connection.service b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/NetworkManager.service.wants/opentrons-init-connection.service deleted file mode 120000 index cd1a0378aa..0000000000 --- a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/NetworkManager.service.wants/opentrons-init-connection.service +++ /dev/null @@ -1 +0,0 @@ -/etc/systemd/system/opentrons-init-connections.service \ No newline at end of file diff --git a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/NetworkManager.service.wants/opentrons-init-connections.service b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/NetworkManager.service.wants/opentrons-init-connections.service new file mode 120000 index 0000000000..d84bcf868b --- /dev/null +++ b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/NetworkManager.service.wants/opentrons-init-connections.service @@ -0,0 +1 @@ +../opentrons-init-connections.service \ No newline at end of file diff --git a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/default.target b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/default.target index a3be96ddd8..7dc13bde07 120000 --- a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/default.target +++ b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/default.target @@ -1 +1 @@ -/etc/systemd/system/opentrons.target \ No newline at end of file +./opentrons.target \ No newline at end of file diff --git a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-init-connections.service b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-init-connections.service index 90ebcce9fd..57a384eae2 100644 --- a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-init-connections.service +++ b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-init-connections.service @@ -1,8 +1,8 @@ [Unit] Description=Opentrons: Ensure system wired connections Before=NetworkManager.service -Requires=local-fs.target -After=local-fs.target +Requires=basic.target +After=basic.target [Service] Type=oneshot diff --git a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-run-boot-scripts.service b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-run-boot-scripts.service index 6c872b710e..481f3a477b 100644 --- a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-run-boot-scripts.service +++ b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-run-boot-scripts.service @@ -1,7 +1,7 @@ [Unit] Description=Opentrons: Run user-supplied boot scripts -Requires=local-fs.target -After=local-fs.target +Requires=basic.target +After=basic.target [Service] Type=oneshot diff --git a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/jupyter-notebook.service b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/jupyter-notebook.service index 104f3368c6..ce93748e03 120000 --- a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/jupyter-notebook.service +++ b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/jupyter-notebook.service @@ -1 +1 @@ -/etc/systemd/system/jupyter-notebook.service \ No newline at end of file +../jupyter-notebook.service \ No newline at end of file diff --git a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/opentrons-run-boot-scripts.service b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/opentrons-run-boot-scripts.service index f4f13c6eaa..402ff62b56 120000 --- a/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/opentrons-run-boot-scripts.service +++ b/board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/opentrons-run-boot-scripts.service @@ -1 +1 @@ -/etc/systemd/system/opentrons-run-boot-scripts.service \ No newline at end of file +../opentrons-run-boot-scripts.service \ No newline at end of file diff --git a/board/opentrons/ot2/write_version.py b/board/opentrons/ot2/write_version.py new file mode 100644 index 0000000000..8d62182ce7 --- /dev/null +++ b/board/opentrons/ot2/write_version.py @@ -0,0 +1,56 @@ +import json +import os +import sys +import subprocess +import argparse + +try: + br_version = subprocess.check_output( + ['git', 'describe', '--tags', '--always'], + stderr=subprocess.STDOUT).decode().strip() +except subprocess.CalledProcessError as cpe: + print("{}: {}: {}".format(cpe.cmd, cpe.returncode, cpe.stdout)) + print("Defaulting to (unknown)") + br_version = 'unknown' + +try: + br_sha = subprocess.check_output( + ['git', 'rev-parse', 'HEAD'], + stderr=subprocess.STDOUT).decode().strip() +except subprocess.CalledProcessError as cpe: + print("{}: {}: {}".format(cpe.cmd, cpe.returncode, cpe.stdout)) + print("Defaulting to (unknown)") + br_sha = 'unknown' + +try: + br_branch_from_git = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + stderr=subprocess.STDOUT).decode().strip() +except subprocess.CalledProcessError as cpe: + print("{}: {}: {}".format(cpe.cmd, cpe.returncode, cpe.stdout)) + print("Defaulting to (unknown)") + br_branch = 'unknown' + +br_branch = os.getenv('CODEBUILD_SOURCE_VERSION', br_branch_from_git) +build_id = os.getenv('CODEBUILD_BUILD_ID', 'dev') + +parser = argparse.ArgumentParser(description='Write a version file to buildroot') +parser.add_argument('in_versions', metavar='IN_VERSIONS', + type=argparse.FileType('r'), nargs='+', + help='A list of files to read versions from. These should ' + 'be json files with non-overlapping keys that will be' + ' unified (along with the buildroot version from git)' + ' into the final product.') +parser.add_argument('outfile', metavar='OUT', type=argparse.FileType('w'), + help='The file to write to') +args = parser.parse_args() + +version_dict = {'buildroot_version': br_version, + 'buildroot_sha': br_sha, + 'buildroot_branch': br_branch, + 'buildroot_buildid': build_id} + +for f in args.in_versions: + version_dict.update(json.load(f)) + +json.dump(version_dict, args.outfile) diff --git a/buildspec.yml b/buildspec.yml index 438ecccf4a..e53e7fe2f7 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -6,11 +6,11 @@ phases: - "docker build -t opentrons-buildroot ." build: commands: - - "docker run --mount type=bind,source=$(pwd),destination=/buildroot opentrons-buildroot ot2_defconfig" - - "docker run --mount type=bind,source=$(pwd),destination=/buildroot opentrons-buildroot all" + - ./opentrons-build.sh artifacts: files: - - output/images/sdcard.img - - output/images/rootfs.tar.gz - - output/images/rootfs.ext4 + - output/images/ot2-system.zip + - output/images/ot2-fullimage.zip + - output/images/VERSION.json + name: ot2-system discard-paths: yes diff --git a/configs/ot2_defconfig b/configs/ot2_defconfig index 4248c0036d..432158bf3a 100644 --- a/configs/ot2_defconfig +++ b/configs/ot2_defconfig @@ -70,3 +70,6 @@ BR2_PACKAGE_HOST_DOSFSTOOLS=y BR2_PACKAGE_HOST_GENEXT2FS=y BR2_PACKAGE_HOST_GENIMAGE=y BR2_PACKAGE_HOST_MTOOLS=y + +BR2_PACKAGE_PYTHON_OPENTRONS_API=y +BR2_PACKAGE_HOST_ZIP=y diff --git a/in_docker.sh b/in_docker.sh index 17937d9e6e..c3a914b3e4 100755 --- a/in_docker.sh +++ b/in_docker.sh @@ -29,8 +29,8 @@ fi if [ -z $filter ] then echo "Unfiltered make" - make -C /buildroot $@ + BR2_EXTERNAL=/opentrons make -C /buildroot $@ else echo "Filtered make" - (make -C /buildroot $@ 2>/buildroot/warnings.txt | awk '/^make/;{print $0 >>"/buildroot/buildlog.txt"}') || cat warnings.txt + (BR2_EXTERNAL=/opentrons make -C /buildroot $@ 2>/buildroot/warnings.txt | awk '/^make/;{print $0 >>"/buildroot/buildlog.txt"}') || cat warnings.txt fi diff --git a/opentrons-build.sh b/opentrons-build.sh index 83b8b7cf58..dbe69b1573 100755 --- a/opentrons-build.sh +++ b/opentrons-build.sh @@ -19,7 +19,11 @@ # are provided, it will run the docker container once with the arguments set -e -x -o pipefail -DOCKER_BIND="--mount type=bind,source=$(pwd),destination=/buildroot" +DOCKER_BR_BIND_DIR="/buildroot" +DOCKER_OT_BIND_DIR="/opentrons" +DOCKER_BIND_BR="--mount type=bind,source=$(pwd),destination=${DOCKER_BR_BIND_DIR}" +DOCKER_BIND_OT="--mount type=bind,source=$(pwd)/../opentrons,destination=${DOCKER_OT_BIND_DIR}" +DOCKER_BIND="${DOCKER_BIND_BR} ${DOCKER_BIND_OT}" heads=${@:1:$(($# - 1))} tail=${@:$#} @@ -32,6 +36,9 @@ imgname=opentrons-buildroot-$(git describe --all --dirty --always) docker build ${filter_arg} -t ${imgname} . +# Save codebuild-relevant env vars to get them inside docker +env | grep 'CODEBUILD\|AWS' >.env + case $# in 0) docker run ${DOCKER_BIND} ${imgname} ot2_defconfig