feat(system): Include opentrons api server from external tree

- Adds a BR2_EXTERNAL tree from the
  monorepo (Opentrons/opentrons#3217) and pulls in the opentrons api server to the build.

- Fixes some issues in systemd services and targets. Codebuild rewrites
  broken (and maybe non-broken) absolute links to fit inside its chroot jail,
  which was breaking the absolute links that aren’t supposed to be valid until
  they’re on the robot

- Add a VERSIONS file taken from a combination of the buildroot version (from
  git) and opentrons api server versions (from package.json) and save to
  /etc/VERSIONS.json

- VERSIONS.json is a new artifact and also zipped up with the full images and
  update images when uploaded to s3

Closes Opentrons/opentrons#2877

.gitignore

@@ -13,3 +13,4 @@
 *.rej
 *~
 *.pyc
+.env

Dockerfile

@@ -5,7 +5,7 @@ ARG filter_output
 VOLUME /buildroot
 
 RUN apt-get -y update &&\
-  apt-get -y install build-essential wget file cpio python rsync unzip bc libncurses-dev
+  apt-get -y install build-essential wget file cpio python rsync unzip bc libncurses-dev git
 
 RUN wget http://crosstool-ng.org/download/crosstool-ng/crosstool-ng-1.23.0.tar.xz &&\
   tar xf ./crosstool-* &&\

board/opentrons/ot2/post-build.sh

@@ -35,3 +35,9 @@ sed -i s/kernel=zImage/kernel=u-boot.bin/ ${BINARIES_DIR}/rpi-firmware/config.tx
 # write common pubkey to authorized keys
 # TODO: DO NOT DO THIS IN RELEASE BUILDS
 cat ${TARGET_DIR}/var/home/.ssh/robot_key.pub > ${TARGET_DIR}/var/home/.ssh/authorized_keys
+
+# Load the out-of-container env to get stuff from codebuild
+export $(cat /buildroot/.env | xargs)
+
+python ./board/opentrons/ot2/write_version.py ${BINARIES_DIR}/opentrons-api-version.json ${BINARIES_DIR}/VERSION.json
+cp ${BINARIES_DIR}/VERSION.json ${TARGET_DIR}/etc/VERSION.json

board/opentrons/ot2/post-image.sh

@@ -7,6 +7,8 @@ BOARD_NAME="$(basename ${BOARD_DIR})"
 GENIMAGE_CFG="${BOARD_DIR}/genimage-${BOARD_NAME}.cfg"
 GENIMAGE_TMP="${BUILD_DIR}/genimage.tmp"
 
+sed -i -e 's/console=ttyAMA0,115200//' "${BINARIES_DIR}/rpi-firmware/cmdline.txt"
+
 for arg in "$@"
 do
 	case "${arg}" in
@@ -61,4 +63,10 @@ genimage                           \
 	--outputpath "${BINARIES_DIR}" \
 	--config "${GENIMAGE_CFG}"
 
+rm -f ${BINARIES_DIR}/ot2-system.zip
+zip -j ${BINARIES_DIR}/ot2-system.zip ${BINARIES_DIR}/rootfs.ext4 ${BINARIES_DIR}/VERSION.json
+rm -f ${BINARIES_DIR}/ot2-fullimage.zip
+zip -j ${BINARIES_DIR}/ot2-fullimage.zip ${BINARIES_DIR}/sdcard.img ${BINARIES_DIR}/VERSION.json
+
+
 exit $?

board/opentrons/ot2/rootfs-overlay/etc/dropbear

@@ -1 +1 @@
-/var/run/dropbear
+../var/run/dropbear

board/opentrons/ot2/rootfs-overlay/etc/nginx/nginx.conf

@@ -29,7 +29,7 @@ http {
             proxy_set_header         Upgrade $http_upgrade;
             proxy_set_header         Connection "upgrade";
             proxy_read_timeout       1h;
-            proxy_pass               http://unix:/tmp/aiohttp.sock;
+            proxy_pass               http://unix:/run/aiohttp.sock;
         }
 
         location /server {

board/opentrons/ot2/rootfs-overlay/etc/systemd/system/NetworkManager.service.wants/opentrons-init-connections.service

@@ -0,0 +1 @@
+../opentrons-init-connections.service

board/opentrons/ot2/rootfs-overlay/etc/systemd/system/default.target

@@ -1 +1 @@
-/etc/systemd/system/opentrons.target
+./opentrons.target

board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-init-connections.service

@@ -1,8 +1,8 @@
 [Unit]
 Description=Opentrons: Ensure system wired connections
 Before=NetworkManager.service
-Requires=local-fs.target
-After=local-fs.target
+Requires=basic.target
+After=basic.target
 
 [Service]
 Type=oneshot

board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons-run-boot-scripts.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Opentrons: Run user-supplied boot scripts
-Requires=local-fs.target
-After=local-fs.target
+Requires=basic.target
+After=basic.target
 
 [Service]
 Type=oneshot

board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/jupyter-notebook.service

@@ -1 +1 @@
-/etc/systemd/system/jupyter-notebook.service
+../jupyter-notebook.service

board/opentrons/ot2/rootfs-overlay/etc/systemd/system/opentrons.target.wants/opentrons-run-boot-scripts.service

@@ -1 +1 @@
-/etc/systemd/system/opentrons-run-boot-scripts.service
+../opentrons-run-boot-scripts.service

board/opentrons/ot2/write_version.py

@@ -0,0 +1,56 @@
+import json
+import os
+import sys
+import subprocess
+import argparse
+
+try:
+    br_version = subprocess.check_output(
+        ['git', 'describe', '--tags', '--always'],
+        stderr=subprocess.STDOUT).decode().strip()
+except subprocess.CalledProcessError as cpe:
+    print("{}: {}: {}".format(cpe.cmd, cpe.returncode, cpe.stdout))
+    print("Defaulting to (unknown)")
+    br_version = 'unknown'
+
+try:
+    br_sha = subprocess.check_output(
+        ['git', 'rev-parse', 'HEAD'],
+        stderr=subprocess.STDOUT).decode().strip()
+except subprocess.CalledProcessError as cpe:
+    print("{}: {}: {}".format(cpe.cmd, cpe.returncode, cpe.stdout))
+    print("Defaulting to (unknown)")
+    br_sha = 'unknown'
+
+try:
+    br_branch_from_git = subprocess.check_output(
+        ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
+        stderr=subprocess.STDOUT).decode().strip()
+except subprocess.CalledProcessError as cpe:
+    print("{}: {}: {}".format(cpe.cmd, cpe.returncode, cpe.stdout))
+    print("Defaulting to (unknown)")
+    br_branch = 'unknown'
+
+br_branch = os.getenv('CODEBUILD_SOURCE_VERSION', br_branch_from_git)
+build_id = os.getenv('CODEBUILD_BUILD_ID', 'dev')
+
+parser = argparse.ArgumentParser(description='Write a version file to buildroot')
+parser.add_argument('in_versions', metavar='IN_VERSIONS',
+                    type=argparse.FileType('r'), nargs='+',
+                    help='A list of files to read versions from. These should '
+                         'be json files with non-overlapping keys that will be'
+                         ' unified (along with the buildroot version from git)'
+                         ' into the final product.')
+parser.add_argument('outfile', metavar='OUT', type=argparse.FileType('w'),
+                    help='The file to write to')
+args = parser.parse_args()
+
+version_dict = {'buildroot_version': br_version,
+                'buildroot_sha': br_sha,
+                'buildroot_branch': br_branch,
+                'buildroot_buildid': build_id}
+
+for f in args.in_versions:
+    version_dict.update(json.load(f))
+
+json.dump(version_dict, args.outfile)

buildspec.yml

@@ -6,11 +6,11 @@ phases:
       - "docker build -t opentrons-buildroot ."
   build:
     commands:
-      - "docker run --mount type=bind,source=$(pwd),destination=/buildroot opentrons-buildroot ot2_defconfig"
-      - "docker run --mount type=bind,source=$(pwd),destination=/buildroot opentrons-buildroot all"
+      - ./opentrons-build.sh
 artifacts:
   files:
-    - output/images/sdcard.img
-    - output/images/rootfs.tar.gz
-    - output/images/rootfs.ext4
+    - output/images/ot2-system.zip
+    - output/images/ot2-fullimage.zip
+    - output/images/VERSION.json
+  name: ot2-system
   discard-paths: yes

configs/ot2_defconfig

@@ -70,3 +70,6 @@ BR2_PACKAGE_HOST_DOSFSTOOLS=y
 BR2_PACKAGE_HOST_GENEXT2FS=y
 BR2_PACKAGE_HOST_GENIMAGE=y
 BR2_PACKAGE_HOST_MTOOLS=y
+
+BR2_PACKAGE_PYTHON_OPENTRONS_API=y
+BR2_PACKAGE_HOST_ZIP=y

in_docker.sh

@@ -29,8 +29,8 @@ fi
 if [ -z $filter ]
 then
     echo "Unfiltered make"
-    make -C /buildroot $@
+    BR2_EXTERNAL=/opentrons make -C /buildroot $@
 else
     echo "Filtered make"
-    (make -C /buildroot $@ 2>/buildroot/warnings.txt | awk '/^make/;{print $0 >>"/buildroot/buildlog.txt"}') || cat warnings.txt
+    (BR2_EXTERNAL=/opentrons make -C /buildroot $@ 2>/buildroot/warnings.txt | awk '/^make/;{print $0 >>"/buildroot/buildlog.txt"}') || cat warnings.txt
 fi

opentrons-build.sh

@@ -19,7 +19,11 @@
 # are provided, it will run the docker container once with the arguments
 set -e -x -o pipefail
 
-DOCKER_BIND="--mount type=bind,source=$(pwd),destination=/buildroot"
+DOCKER_BR_BIND_DIR="/buildroot"
+DOCKER_OT_BIND_DIR="/opentrons"
+DOCKER_BIND_BR="--mount type=bind,source=$(pwd),destination=${DOCKER_BR_BIND_DIR}"
+DOCKER_BIND_OT="--mount type=bind,source=$(pwd)/../opentrons,destination=${DOCKER_OT_BIND_DIR}"
+DOCKER_BIND="${DOCKER_BIND_BR} ${DOCKER_BIND_OT}"
 heads=${@:1:$(($# - 1))}
 tail=${@:$#}
 
@@ -32,6 +36,9 @@ imgname=opentrons-buildroot-$(git describe --all --dirty --always)
 
 docker build ${filter_arg} -t ${imgname} .
 
+# Save codebuild-relevant env vars to get them inside docker
+env | grep 'CODEBUILD\|AWS' >.env
+
 case $# in
     0)
         docker run ${DOCKER_BIND} ${imgname} ot2_defconfig