refactor: Containerfile+Makefile improvement: build crates together, …

…then distribute the binaries (#1164)

* MASSIVE dockerfile improvement: build crates together, then distribute the binaries

* add back cargo profile

* simplify containerfile and adjust ci

* adjust lock

* Address feedback, fix local run, separate final steps per crate

* fix toolchain env var and pem files

.dockerignore

@@ -1,6 +1,5 @@
 **/target/
 **/*Containerfile*
-terraform/
 examples/
 **/.cargo/
 **/docker/

.gitignore

@@ -5,6 +5,8 @@
 **/target/
 # Ignore the cargo lockfiles not in the workspace root
 */**/Cargo.lock
+# cargo-chef testing
+recipe.json
 
 # These are backup files generated by rustfmt
 **/*.rs.bk

Containerfile

@@ -3,57 +3,95 @@
 
 # Base image for builds and cache
 ARG RUSTUP_TOOLCHAIN
-FROM lukemathwalker/cargo-chef:latest-rust-${RUSTUP_TOOLCHAIN}-buster as shuttle-build
+FROM docker.io/lukemathwalker/cargo-chef:latest-rust-${RUSTUP_TOOLCHAIN}-buster as cargo-chef
 WORKDIR /build
 
 
-# Stores source cache
-FROM shuttle-build as cache
-ARG PROD
+# Stores source cache and cargo chef recipe
+FROM cargo-chef as planner
 WORKDIR /src
 COPY . .
-RUN find ${SRC_CRATES} \( -name "*.proto" -or -name "*.rs" -or -name "*.toml" -or -name "Cargo.lock" -or -name "README.md" -or -name "*.sql" -or -name "ulid0.so" \) -type f -exec install -D \{\} /build/\{\} \;
-# This is used to carry over in the docker images any *.pem files from shuttle root directory,
-# to be used for TLS testing, as described here in the admin README.md.
-RUN if [ "$PROD" != "true" ]; then \
-    find ${SRC_CRATES} -name "*.pem" -type f -exec install -D \{\} /build/\{\} \;; \
-    fi
-
-
-# Stores cargo chef recipe
-FROM shuttle-build AS planner
-COPY --from=cache /build .
-RUN cargo chef prepare --recipe-path recipe.json
-
+# Select only the essential files for copying into next steps
+# so that changes to miscellaneous files don't trigger a new cargo-chef cook.
+# Beware that .dockerignore filters files before they get here.
+RUN find . \( \
+    -name "*.rs" -or \
+    -name "*.toml" -or \
+    -name "Cargo.lock" -or \
+    -name "*.sql" -or \
+    # Used for local TLS testing, as described in admin/README.md
+    -name "*.pem" -or \
+    -name "ulid0.so" \
+    \) -type f -exec install -D \{\} /build/\{\} \;
+WORKDIR /build
+RUN cargo chef prepare --recipe-path /recipe.json
+# TODO upstream: Reduce the cooking by allowing multiple --bin args to prepare, or like this https://github.com/LukeMathWalker/cargo-chef/issues/181
 
-# Builds crate according to cargo chef recipe
-FROM shuttle-build AS builder
-ARG folder
-COPY --from=planner /build/recipe.json recipe.json
-RUN cargo chef cook --release --recipe-path recipe.json
 
-COPY --from=cache /build .
-RUN cargo build --bin shuttle-${folder} --release
+# Builds crate according to cargo chef recipe.
+# This step is skipped if the recipe is unchanged from previous build (no dependencies changed).
+FROM cargo-chef AS builder
+ARG CARGO_PROFILE
+COPY --from=planner /recipe.json /
+# https://i.imgflip.com/2/74bvex.jpg
+RUN cargo chef cook \
+    --all-features \
+    $(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi) \
+    --recipe-path /recipe.json
+COPY --from=planner /build .
+# Building all at once to share build artifacts in the "cook" layer
+RUN cargo build \
+    $(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi) \
+    --bin shuttle-auth \
+    --bin shuttle-deployer \
+    --bin shuttle-provisioner \
+    --bin shuttle-gateway \
+    --bin shuttle-resource-recorder \
+    --bin shuttle-next -F next
 
 
-# The final image for this "shuttle-..." crate
+# Base image for running each "shuttle-..." binary
 ARG RUSTUP_TOOLCHAIN
-FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-crate
+FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-crate-base
+ARG CARGO_PROFILE
 ARG folder
+ARG crate
 ARG prepare_args
-# used as env variable in prepare script
-ARG PROD
+# Fixes some dependencies compiled with incompatible versions of rustc
 ARG RUSTUP_TOOLCHAIN
 ENV RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN}
+# Used as env variable in prepare script
+ARG PROD
 
-COPY ${folder}/prepare.sh /prepare.sh
+# Some crates need additional libs
+COPY ${folder}/*.so /usr/lib/
+ENV LD_LIBRARY_PATH=/usr/lib/
+
+COPY --from=builder /build/target/${CARGO_PROFILE}/${crate} /usr/local/bin/service
+ENTRYPOINT ["/usr/local/bin/service"]
+
+
+# Targets for each crate
+
+FROM shuttle-crate-base AS shuttle-auth
+FROM shuttle-auth AS shuttle-auth-dev
+
+FROM shuttle-crate-base AS shuttle-deployer
+ARG CARGO_PROFILE
+COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-next /usr/local/cargo/bin/
+COPY deployer/prepare.sh /prepare.sh
 RUN /prepare.sh "${prepare_args}"
+FROM shuttle-deployer AS shuttle-deployer-dev
+# Source code needed for compiling with [patch.crates-io]
+COPY --from=planner /build /usr/src/shuttle/
 
-COPY --from=cache /build /usr/src/shuttle/
+FROM shuttle-crate-base AS shuttle-gateway
+FROM shuttle-gateway AS shuttle-gateway-dev
+# For testing certificates locally
+COPY --from=planner /build/*.pem /usr/src/shuttle/
 
-# Any prepare steps that depend on the COPY from src cache.
-# In the deployer shuttle-next is installed and the panamax mirror config is added in this step.
-RUN /prepare.sh --after-src "${prepare_args}"
+FROM shuttle-crate-base AS shuttle-provisioner
+FROM shuttle-provisioner AS shuttle-provisioner-dev
 
-COPY --from=builder /build/target/release/shuttle-${folder} /usr/local/bin/service
-ENTRYPOINT ["/usr/local/bin/service"]
+FROM shuttle-crate-base AS shuttle-resource-recorder
+FROM shuttle-resource-recorder AS shuttle-resource-recorder-dev

Makefile

@@ -1,10 +1,7 @@
-SRC_CRATES=deployer common codegen cargo-shuttle proto provisioner service
-SRC=$(shell find $(SRC_CRATES) -name "*.rs" -type f -not -path "**/target/*")
+COMMIT_SHA?=$(shell git rev-parse --short HEAD)
 
-COMMIT_SHA ?= $(shell git rev-parse --short HEAD)
-
-BUILDX_CACHE?=/tmp/cache/buildx
 ifeq ($(CI),true)
+BUILDX_CACHE?=/tmp/cache/buildx
 CACHE_FLAGS=--cache-to type=local,dest=$(BUILDX_CACHE),mode=max --cache-from type=local,src=$(BUILDX_CACHE)
 endif
 
@@ -31,7 +28,6 @@ PROVISIONER_TAG?=$(TAG)
 RESOURCE_RECORDER_TAG?=$(TAG)
 
 DOCKER_BUILD?=docker buildx build
-
 ifeq ($(CI),true)
 DOCKER_BUILD+= --progress plain
 endif
@@ -47,6 +43,7 @@ POSTGRES_PASSWORD?=postgres
 MONGO_INITDB_ROOT_USERNAME?=mongodb
 MONGO_INITDB_ROOT_PASSWORD?=password
 
+
 ifeq ($(PROD),true)
 DOCKER_COMPOSE_FILES=docker-compose.yml
 STACK=shuttle-prod
@@ -56,6 +53,7 @@ CONTAINER_REGISTRY=public.ecr.aws/shuttle
 DD_ENV=production
 # make sure we only ever go to production with `--tls=enable`
 USE_TLS=enable
+CARGO_PROFILE=release
 RUST_LOG?=shuttle=debug,info
 else
 DOCKER_COMPOSE_FILES=docker-compose.yml docker-compose.dev.yml
@@ -65,10 +63,18 @@ DB_FQDN=db.unstable.shuttle.rs
 CONTAINER_REGISTRY=public.ecr.aws/shuttle-dev
 DD_ENV=unstable
 USE_TLS?=disable
+# default for local run
+CARGO_PROFILE?=debug
 RUST_LOG?=shuttle=debug,info
+DEV_SUFFIX=-dev
 DEPLOYS_API_KEY?=gateway4deployes
 endif
 
+ifeq ($(CI),true)
+# default for staging
+CARGO_PROFILE=release
+endif
+
 POSTGRES_EXTRA_PATH?=./extras/postgres
 POSTGRES_TAG?=14
 
@@ -112,13 +118,15 @@ DOCKER_COMPOSE_ENV=\
 	COMPOSE_PROFILES=$(COMPOSE_PROFILES)\
 	DOCKER_SOCK=$(DOCKER_SOCK)
 
-.PHONY: images clean src up down deploy shuttle-% postgres docker-compose.rendered.yml test bump-% deploy-examples publish publish-% --validate-version
+.PHONY: images clean src up down deploy shuttle-% shuttle-images postgres docker-compose.rendered.yml test bump-% deploy-examples publish publish-% --validate-version
 
 clean:
 	rm .shuttle-*
 	rm docker-compose.rendered.yml
 
-images: shuttle-provisioner shuttle-deployer shuttle-gateway shuttle-auth shuttle-resource-recorder postgres panamax otel
+images: shuttle-images postgres panamax otel
+
+shuttle-images: shuttle-auth shuttle-deployer shuttle-gateway shuttle-provisioner shuttle-resource-recorder
 
 postgres:
 	$(DOCKER_BUILD) \
@@ -169,12 +177,15 @@ up: $(DOCKER_COMPOSE_FILES)
 down: $(DOCKER_COMPOSE_FILES)
 	$(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) $(addprefix -f ,$(DOCKER_COMPOSE_FILES)) -p $(STACK) down
 
-shuttle-%: ${SRC} Cargo.lock
+shuttle-%:
 	$(DOCKER_BUILD) \
+		--target $(@)$(DEV_SUFFIX) \
 		--build-arg folder=$(*) \
+		--build-arg crate=$(@) \
 		--build-arg prepare_args=$(PREPARE_ARGS) \
 		--build-arg PROD=$(PROD) \
 		--build-arg RUSTUP_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) \
+		--build-arg CARGO_PROFILE=$(CARGO_PROFILE) \
 		--tag $(CONTAINER_REGISTRY)/$(*):$(COMMIT_SHA) \
 		--tag $(CONTAINER_REGISTRY)/$(*):$(TAG) \
 		--tag $(CONTAINER_REGISTRY)/$(*):latest \

deployer/prepare.sh

@@ -1,32 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 ###############################################################################
 # This file is used by our common Containerfile incase the container for this #
 # service might need some extra preparation steps for its final image         #
 ###############################################################################
 
-# Stuff that depends on local source files
-if [ "$1" = "--after-src" ]; then
-    # Install the shuttle-next runtime for shuttle-next services.
-    cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" --bin shuttle-next --features next || exit 1
-
-    while getopts "p," o; do
-    case $o in
-        "p") # if panamax is used, the '-p' parameter is passed
-            # Make future crates requests to our own mirror
-            echo '
-[source.shuttle-crates-io-mirror]
-registry = "sparse+http://panamax:8080/index/"
-[source.crates-io]
-replace-with = "shuttle-crates-io-mirror"' >> $CARGO_HOME/config.toml
-                ;;
-            *)
-                ;;
-        esac
-    done
-    exit 0
-fi
-
 # Patch crates to be on same versions
 mkdir -p $CARGO_HOME
 touch $CARGO_HOME/config.toml
@@ -72,3 +50,19 @@ VERSION="22.2" && \
 curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \
     unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \
     rm -f "protoc-$VERSION-$ARCH.zip"
+
+while getopts "p," o; do
+case $o in
+    "p") # if panamax is used, the '-p' parameter is passed
+        # Make future crates requests to our own mirror
+        # This is done after shuttle-next install in order to not sabotage it
+        echo '
+[source.shuttle-crates-io-mirror]
+registry = "sparse+http://panamax:8080/index/"
+[source.crates-io]
+replace-with = "shuttle-crates-io-mirror"' >> $CARGO_HOME/config.toml
+            ;;
+        *)
+            ;;
+    esac
+done

docker-compose.yml

@@ -77,7 +77,6 @@ services:
       - ${DOCKER_SOCK}:/var/run/docker.sock
     environment:
       - RUST_LOG=${RUST_LOG}
-      - LD_LIBRARY_PATH=/usr/src/shuttle/gateway
     command:
       - "--state=/var/lib/shuttle"
       - "start"

gateway/src/project.rs

@@ -750,8 +750,7 @@ impl ProjectCreating {
                         self.project_id.to_string()
                     ],
                     "Env": [
-                        "RUST_LOG=debug,shuttle=trace,h2=warn",
-                        "LD_LIBRARY_PATH=/usr/src/shuttle/deployer"
+                        "RUST_LOG=debug,shuttle=trace,h2=warn"
                     ]
                 })
             });