feat: static compilation of drivers

.gitignore

@@ -1,10 +1,9 @@
-docs/
-lib/
-bin/
-.shards/
 *.dwarf
 .DS_*
+.shards/
 .tool-versions
-repositories
-binaries
+bin/
 build_app
+docs/
+lib/
+repositories/

Dockerfile

@@ -1,86 +1,73 @@
 ARG CRYSTAL_VERSION=1.0.0
-ARG PLACE_COMMIT="DEV"
 
-FROM crystallang/crystal:${CRYSTAL_VERSION} as build
+# Build `digest_cli`
+###############################################################################
+FROM crystallang/crystal:${CRYSTAL_VERSION}-alpine-build as build-digest
 
 WORKDIR /app
 
-RUN apt-get update && \
-    apt-get install -y apt-transport-https && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive \
-    apt install --no-install-recommends -y \
-        bash \
-        ca-certificates \
-        curl \
-        llvm-10 llvm-10-dev \
-        libssh2-1 libssh2-1-dev \
-        libyaml-dev \
-    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-# Add trusted CAs for communicating with external services
-RUN update-ca-certificates
-
-# Create binary directories
-RUN mkdir -p repositories bin/drivers
-# Install watchexec
-RUN curl -sLO https://github.com/watchexec/watchexec/releases/download/cli-v1.16.0/watchexec-1.16.0-x86_64-unknown-linux-gnu.deb && \
-    dpkg -i watchexec-1.16.0-x86_64-unknown-linux-gnu.deb && \
-    rm -rf ./*.deb
-
-RUN mkdir -p /app/bin/drivers
-
-# Install shards before adding source.
-COPY shard.yml /app
-COPY shard.lock /app
-RUN shards install --ignore-crystal-version
-
-# Build digest tool before copying rest of source for better caching.
-COPY src/digest_cli.cr /app/src/digest_cli.cr
+RUN apk add --update --no-cache \
+            bash \
+            yaml-static
+
+# Build a missing llvm artefact, `llvm_ext.o`
+COPY scripts/build_llvm_ext.sh build_llvm_ext.sh
+RUN ./build_llvm_ext.sh
+
+COPY shard.* .
+RUN shards install --production --ignore-crystal-version
+
+COPY src/digest_cli.cr src/digest_cli.cr
+
+# Build the required target
 RUN CRYSTAL_PATH=lib:/usr/share/crystal/src/ \
     LLVM_CONFIG=$(/usr/share/crystal/src/llvm/ext/find-llvm-config) \
-    shards build digest_cli -Dpreview_mt --ignore-crystal-version --no-debug --production
+    shards build --static --no-debug --error-trace --ignore-crystal-version --production digest_cli && \
+    rm /usr/share/crystal/src/llvm/ext/llvm_ext.o
+
+# Build `build`
+###############################################################################
+FROM crystallang/crystal:${CRYSTAL_VERSION}-alpine as build
+
+ARG CRYSTAL_VERSION=1.0.0
+ARG PLACE_COMMIT="DEV"
 
-# Add the rest of the source last for efficient caching
-COPY scripts /app/scripts
+WORKDIR /app
+
+COPY shard.* .
+RUN shards install --production --ignore-crystal-version
+
+# Add source last for efficient caching
 COPY src /app/src
 
+# Build the required target
 RUN PLACE_COMMIT=${PLACE_COMMIT} \
+    PLACE_VERSION=${PLACE_VERSION} \
     UNAME_AT_COMPILE_TIME=true \
-    shards build --error-trace -Dpreview_mt --release --ignore-crystal-version --production build
+    CRYSTAL_VERSION=${CRYSTAL_VERSION}} \
+    shards build --no-debug --release --error-trace --ignore-crystal-version --production build
+
+COPY --from=build-digest /app/bin/digest_cli /app/bin
 
 ###############################################################################
 
 ENV HOME="/app"
 ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-ENV CRYSTAL_VERSION=$CRYSTAL_VERSION
-
-# Create a non-privileged user
 ARG IMAGE_UID="10001"
 ENV UID=$IMAGE_UID
 ENV USER=appuser
+
+# Create a non-privileged user
 RUN adduser \
     --disabled-password \
     --gecos "" \
-    --home "/nonexistent" \
+    --home "${HOME}" \
     --shell "/sbin/nologin" \
-    --no-create-home \
     --uid "${UID}" \
     "${USER}"
 
 RUN chown appuser -R /app
 
-# Install asdf version manager
-SHELL ["/bin/bash", "-l", "-c"]
-RUN git clone --depth 1 https://github.com/asdf-vm/asdf.git $HOME/.asdf --branch v0.8.0 && \
-    $HOME/.asdf/bin/asdf plugin-add crystal https://github.com/asdf-community/asdf-crystal.git && \
-    echo -e '\n. $HOME/.asdf/asdf.sh' >> ~/.bashrc && \
-    echo -e '\n. $HOME/.asdf/asdf.sh' >> ~/.profile && \
-    source ~/.bashrc
-
-USER appuser:appuser
-RUN chown appuser -R /app
-
 EXPOSE 3000
-HEALTHCHECK CMD wget -qO- http://localhost:3000/api/build/v1
-CMD ["/app/scripts/entrypoint.sh", "--server", "-b", "0.0.0.0", "-p", "3000"]
+HEALTHCHECK CMD /app/bin/build server --curl http://localhost:3000/api/build/v1
+CMD ["/app/scripts/entrypoint.sh", server, "-b", "0.0.0.0", "-p", "3000"]

Dockerfile.test

@@ -1,58 +1,82 @@
+ARG CRYSTAL_VERSION=1.0.0
+
+# Build `digest_cli`
+###############################################################################
+
+FROM crystallang/crystal:${CRYSTAL_VERSION}-alpine-build as build-digest
+
 ARG CRYSTAL_VERSION=1.0.0
 ARG PLACE_COMMIT="DEV"
 
-FROM crystallang/crystal:${CRYSTAL_VERSION} as build
+WORKDIR /app
+
+# Install the latest version of LibSSH2, ping, curl, git
+RUN apk add --update --no-cache \
+        bash \
+        yaml-static
+
+# Build a missing llvm artefact.
+COPY scripts/build_llvm_ext.sh build_llvm_ext.sh
+RUN ./build_llvm_ext.sh
+
+COPY shard.lock .
+COPY shard.yml .
+
+RUN shards install --ignore-crystal-version
+
+COPY src/digest_cli.cr src/digest_cli.cr
+
+ENV CRYSTAL_VERSION=$CRYSTAL_VERSION
+
+RUN CRYSTAL_PATH=lib:/usr/share/crystal/src/ \
+    LLVM_CONFIG=$(/usr/share/crystal/src/llvm/ext/find-llvm-config) \
+    PLACE_COMMIT=${PLACE_COMMIT} \
+    UNAME_AT_COMPILE_TIME=true \
+    shards build --static --error-trace --ignore-crystal-version digest_cli && \
+    rm /usr/share/crystal/src/llvm/ext/llvm_ext.o
+
+FROM crystallang/crystal:${CRYSTAL_VERSION}-alpine
 
 WORKDIR /app
 
-RUN apt-get update && \
-    apt-get install -y apt-transport-https && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive \
-    apt install --no-install-recommends -y \
+ENV CRYSTAL_VERSION=$CRYSTAL_VERSION
+
+# Install the latest version of LibSSH2, ping, curl, git
+RUN apk add --update --no-cache \
         bash \
         ca-certificates \
-        curl \
-        llvm-10 llvm-10-dev \
-        libssh2-1 libssh2-1-dev \
-        libyaml-dev \
-    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+        iputils \
+        libssh2-static \
+        yaml-static
+
+RUN apk add --update --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing watchexec
 
 # Add trusted CAs for communicating with external services
 RUN update-ca-certificates
 
-# Install watchexec
-RUN curl -sLO https://github.com/watchexec/watchexec/releases/download/cli-v1.16.0/watchexec-1.16.0-x86_64-unknown-linux-gnu.deb && \
-    dpkg -i watchexec-1.16.0-x86_64-unknown-linux-gnu.deb && \
-    rm -rf ./*.deb
+# Awaiting asdf patch for static compiler builds
+#
+# SHELL ["/bin/bash", "-l", "-c"]
+#
+# # Install asdf version manager
+# RUN git clone https://github.com/asdf-vm/asdf.git $HOME/.asdf --branch v0.8.0
+# RUN $HOME/.asdf/bin/asdf plugin-add crystal https://github.com/asdf-community/asdf-crystal.git
+#
+# RUN echo -e '\n. $HOME/.asdf/asdf.sh' >> ~/.bashrc && \
+#     echo -e '\n. $HOME/.asdf/asdf.sh' >> ~/.profile && \
+#     source ~/.bashrc
 
-# Add trusted CAs for communicating with external services
-RUN update-ca-certificates
+RUN mkdir -p /app/bin/drivers
 
-# Install shards before adding source.
-COPY shard.yml /app
-COPY shard.lock /app
+COPY shard.yml .
+COPY shard.lock .
 RUN shards install --ignore-crystal-version
 
-# Build digest tool before copying rest of source for better caching.
-COPY src/digest_cli.cr /app/src/digest_cli.cr
-RUN CRYSTAL_PATH=lib:/usr/share/crystal/src/ \
-    LLVM_CONFIG=$(/usr/share/crystal/src/llvm/ext/find-llvm-config) \
-    shards build digest_cli -Dpreview_mt --ignore-crystal-version --no-debug --production
-
 COPY scripts /app/scripts
-COPY src /app/src
 
-ENV HOME="/app"
-ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-
-SHELL ["/bin/bash", "-l", "-c"]
-# Install asdf version manager
-RUN git clone --depth 1 https://github.com/asdf-vm/asdf.git $HOME/.asdf --branch v0.8.0 && \
-    $HOME/.asdf/bin/asdf plugin-add crystal https://github.com/asdf-community/asdf-crystal.git && \
-    echo -e '\n. $HOME/.asdf/asdf.sh' >> $HOME/.bashrc && \
-    echo -e '\n. $HOME/.asdf/asdf.sh' >> $HOME/.profile && \
-    source ~/.bashrc
+COPY --from=build-digest /app/bin/digest_cli /app/bin
 
+# These provide certificate chain validation where communicating with external services over TLS
+ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
 
 CMD /app/scripts/entrypoint-test.sh

scripts/build_llvm_ext.sh

@@ -0,0 +1,6 @@
+#! /usr/bin/env bash
+
+CRYSTAL_PATH=/usr/share/crystal/src
+LLVM_CONFIG=$(/usr/share/crystal/src/llvm/ext/find-llvm-config)
+
+cc -c -g -O3 "$CRYSTAL_PATH/llvm/ext/llvm_ext.cc"  -o "$CRYSTAL_PATH/llvm/ext/llvm_ext.o" $($LLVM_CONFIG --cxxflags)

scripts/entrypoint-test.sh

@@ -2,10 +2,11 @@
 
 set -eu
 
-source $HOME/.asdf/asdf.sh
-
-asdf install crystal 1.0.0
-asdf global crystal 1.0.0
+# TODO: add once asdf patched for glibc crystal
+# source $HOME/.asdf/asdf.sh
+#
+# asdf install crystal 1.0.0
+# asdf global crystal 1.0.0
 
 if [ -z ${GITHUB_ACTION+x} ]
 then
@@ -39,14 +40,14 @@ do
   esac
 done
 
-if [[ "$multithreaded" == "true" ]]; then
+if [[ "${multithreaded}" == "true" ]]; then
   args="-Dpreview_mt"
 else
   args=""
 fi
 
-if [[ "$watch" == "true" ]]; then
+if [[ "${watch}" == "true" ]]; then
   CRYSTAL_WORKERS=$(nproc) watchexec -e cr -c -r -w src -w spec -- scripts/crystal-spec.sh -v ${args}
 else
   CRYSTAL_WORKERS=$(nproc) scripts/crystal-spec.sh -v ${args}
-fi
+fi

scripts/entrypoint.sh

@@ -2,6 +2,7 @@
 
 set -eu
 
-source $HOME/.asdf/asdf.sh
+# TODO: After asdf support added
+# source $HOME/.asdf/asdf.sh
 
 /app/bin/build "$@"