diff --git a/.gitignore b/.gitignore index 0cbb10c..6fb693e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,9 @@ -docs/ -lib/ -bin/ -.shards/ *.dwarf .DS_* +.shards/ .tool-versions -repositories -binaries +bin/ build_app +docs/ +lib/ +repositories/ diff --git a/Dockerfile b/Dockerfile index 47d8822..3704bb4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,86 +1,73 @@ ARG CRYSTAL_VERSION=1.0.0 -ARG PLACE_COMMIT="DEV" -FROM crystallang/crystal:${CRYSTAL_VERSION} as build +# Build `digest_cli` +############################################################################### +FROM crystallang/crystal:${CRYSTAL_VERSION}-alpine-build as build-digest WORKDIR /app -RUN apt-get update && \ - apt-get install -y apt-transport-https && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt install --no-install-recommends -y \ - bash \ - ca-certificates \ - curl \ - llvm-10 llvm-10-dev \ - libssh2-1 libssh2-1-dev \ - libyaml-dev \ - && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Add trusted CAs for communicating with external services -RUN update-ca-certificates - -# Create binary directories -RUN mkdir -p repositories bin/drivers -# Install watchexec -RUN curl -sLO https://github.com/watchexec/watchexec/releases/download/cli-v1.16.0/watchexec-1.16.0-x86_64-unknown-linux-gnu.deb && \ - dpkg -i watchexec-1.16.0-x86_64-unknown-linux-gnu.deb && \ - rm -rf ./*.deb - -RUN mkdir -p /app/bin/drivers - -# Install shards before adding source. -COPY shard.yml /app -COPY shard.lock /app -RUN shards install --ignore-crystal-version - -# Build digest tool before copying rest of source for better caching. -COPY src/digest_cli.cr /app/src/digest_cli.cr +RUN apk add --update --no-cache \ + bash \ + yaml-static + +# Build a missing llvm artefact, `llvm_ext.o` +COPY scripts/build_llvm_ext.sh build_llvm_ext.sh +RUN ./build_llvm_ext.sh + +COPY shard.* . +RUN shards install --production --ignore-crystal-version + +COPY src/digest_cli.cr src/digest_cli.cr + +# Build the required target RUN CRYSTAL_PATH=lib:/usr/share/crystal/src/ \ LLVM_CONFIG=$(/usr/share/crystal/src/llvm/ext/find-llvm-config) \ - shards build digest_cli -Dpreview_mt --ignore-crystal-version --no-debug --production + shards build --static --no-debug --error-trace --ignore-crystal-version --production digest_cli && \ + rm /usr/share/crystal/src/llvm/ext/llvm_ext.o + +# Build `build` +############################################################################### +FROM crystallang/crystal:${CRYSTAL_VERSION}-alpine as build + +ARG CRYSTAL_VERSION=1.0.0 +ARG PLACE_COMMIT="DEV" -# Add the rest of the source last for efficient caching -COPY scripts /app/scripts +WORKDIR /app + +COPY shard.* . +RUN shards install --production --ignore-crystal-version + +# Add source last for efficient caching COPY src /app/src +# Build the required target RUN PLACE_COMMIT=${PLACE_COMMIT} \ + PLACE_VERSION=${PLACE_VERSION} \ UNAME_AT_COMPILE_TIME=true \ - shards build --error-trace -Dpreview_mt --release --ignore-crystal-version --production build + CRYSTAL_VERSION=${CRYSTAL_VERSION}} \ + shards build --no-debug --release --error-trace --ignore-crystal-version --production build + +COPY --from=build-digest /app/bin/digest_cli /app/bin ############################################################################### ENV HOME="/app" ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -ENV CRYSTAL_VERSION=$CRYSTAL_VERSION - -# Create a non-privileged user ARG IMAGE_UID="10001" ENV UID=$IMAGE_UID ENV USER=appuser + +# Create a non-privileged user RUN adduser \ --disabled-password \ --gecos "" \ - --home "/nonexistent" \ + --home "${HOME}" \ --shell "/sbin/nologin" \ - --no-create-home \ --uid "${UID}" \ "${USER}" RUN chown appuser -R /app -# Install asdf version manager -SHELL ["/bin/bash", "-l", "-c"] -RUN git clone --depth 1 https://github.com/asdf-vm/asdf.git $HOME/.asdf --branch v0.8.0 && \ - $HOME/.asdf/bin/asdf plugin-add crystal https://github.com/asdf-community/asdf-crystal.git && \ - echo -e '\n. $HOME/.asdf/asdf.sh' >> ~/.bashrc && \ - echo -e '\n. $HOME/.asdf/asdf.sh' >> ~/.profile && \ - source ~/.bashrc - -USER appuser:appuser -RUN chown appuser -R /app - EXPOSE 3000 -HEALTHCHECK CMD wget -qO- http://localhost:3000/api/build/v1 -CMD ["/app/scripts/entrypoint.sh", "--server", "-b", "0.0.0.0", "-p", "3000"] +HEALTHCHECK CMD /app/bin/build server --curl http://localhost:3000/api/build/v1 +CMD ["/app/scripts/entrypoint.sh", server, "-b", "0.0.0.0", "-p", "3000"] diff --git a/Dockerfile.test b/Dockerfile.test index 51f81f7..4cbb935 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,58 +1,82 @@ +ARG CRYSTAL_VERSION=1.0.0 + +# Build `digest_cli` +############################################################################### + +FROM crystallang/crystal:${CRYSTAL_VERSION}-alpine-build as build-digest + ARG CRYSTAL_VERSION=1.0.0 ARG PLACE_COMMIT="DEV" -FROM crystallang/crystal:${CRYSTAL_VERSION} as build +WORKDIR /app + +# Install the latest version of LibSSH2, ping, curl, git +RUN apk add --update --no-cache \ + bash \ + yaml-static + +# Build a missing llvm artefact. +COPY scripts/build_llvm_ext.sh build_llvm_ext.sh +RUN ./build_llvm_ext.sh + +COPY shard.lock . +COPY shard.yml . + +RUN shards install --ignore-crystal-version + +COPY src/digest_cli.cr src/digest_cli.cr + +ENV CRYSTAL_VERSION=$CRYSTAL_VERSION + +RUN CRYSTAL_PATH=lib:/usr/share/crystal/src/ \ + LLVM_CONFIG=$(/usr/share/crystal/src/llvm/ext/find-llvm-config) \ + PLACE_COMMIT=${PLACE_COMMIT} \ + UNAME_AT_COMPILE_TIME=true \ + shards build --static --error-trace --ignore-crystal-version digest_cli && \ + rm /usr/share/crystal/src/llvm/ext/llvm_ext.o + +FROM crystallang/crystal:${CRYSTAL_VERSION}-alpine WORKDIR /app -RUN apt-get update && \ - apt-get install -y apt-transport-https && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt install --no-install-recommends -y \ +ENV CRYSTAL_VERSION=$CRYSTAL_VERSION + +# Install the latest version of LibSSH2, ping, curl, git +RUN apk add --update --no-cache \ bash \ ca-certificates \ - curl \ - llvm-10 llvm-10-dev \ - libssh2-1 libssh2-1-dev \ - libyaml-dev \ - && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + iputils \ + libssh2-static \ + yaml-static + +RUN apk add --update --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing watchexec # Add trusted CAs for communicating with external services RUN update-ca-certificates -# Install watchexec -RUN curl -sLO https://github.com/watchexec/watchexec/releases/download/cli-v1.16.0/watchexec-1.16.0-x86_64-unknown-linux-gnu.deb && \ - dpkg -i watchexec-1.16.0-x86_64-unknown-linux-gnu.deb && \ - rm -rf ./*.deb +# Awaiting asdf patch for static compiler builds +# +# SHELL ["/bin/bash", "-l", "-c"] +# +# # Install asdf version manager +# RUN git clone https://github.com/asdf-vm/asdf.git $HOME/.asdf --branch v0.8.0 +# RUN $HOME/.asdf/bin/asdf plugin-add crystal https://github.com/asdf-community/asdf-crystal.git +# +# RUN echo -e '\n. $HOME/.asdf/asdf.sh' >> ~/.bashrc && \ +# echo -e '\n. $HOME/.asdf/asdf.sh' >> ~/.profile && \ +# source ~/.bashrc -# Add trusted CAs for communicating with external services -RUN update-ca-certificates +RUN mkdir -p /app/bin/drivers -# Install shards before adding source. -COPY shard.yml /app -COPY shard.lock /app +COPY shard.yml . +COPY shard.lock . RUN shards install --ignore-crystal-version -# Build digest tool before copying rest of source for better caching. -COPY src/digest_cli.cr /app/src/digest_cli.cr -RUN CRYSTAL_PATH=lib:/usr/share/crystal/src/ \ - LLVM_CONFIG=$(/usr/share/crystal/src/llvm/ext/find-llvm-config) \ - shards build digest_cli -Dpreview_mt --ignore-crystal-version --no-debug --production - COPY scripts /app/scripts -COPY src /app/src -ENV HOME="/app" -ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - -SHELL ["/bin/bash", "-l", "-c"] -# Install asdf version manager -RUN git clone --depth 1 https://github.com/asdf-vm/asdf.git $HOME/.asdf --branch v0.8.0 && \ - $HOME/.asdf/bin/asdf plugin-add crystal https://github.com/asdf-community/asdf-crystal.git && \ - echo -e '\n. $HOME/.asdf/asdf.sh' >> $HOME/.bashrc && \ - echo -e '\n. $HOME/.asdf/asdf.sh' >> $HOME/.profile && \ - source ~/.bashrc +COPY --from=build-digest /app/bin/digest_cli /app/bin +# These provide certificate chain validation where communicating with external services over TLS +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt CMD /app/scripts/entrypoint-test.sh diff --git a/scripts/build_llvm_ext.sh b/scripts/build_llvm_ext.sh new file mode 100755 index 0000000..e447ae3 --- /dev/null +++ b/scripts/build_llvm_ext.sh @@ -0,0 +1,6 @@ +#! /usr/bin/env bash + +CRYSTAL_PATH=/usr/share/crystal/src +LLVM_CONFIG=$(/usr/share/crystal/src/llvm/ext/find-llvm-config) + +cc -c -g -O3 "$CRYSTAL_PATH/llvm/ext/llvm_ext.cc" -o "$CRYSTAL_PATH/llvm/ext/llvm_ext.o" $($LLVM_CONFIG --cxxflags) diff --git a/scripts/entrypoint-test.sh b/scripts/entrypoint-test.sh index bb4e833..d2e8452 100755 --- a/scripts/entrypoint-test.sh +++ b/scripts/entrypoint-test.sh @@ -2,10 +2,11 @@ set -eu -source $HOME/.asdf/asdf.sh - -asdf install crystal 1.0.0 -asdf global crystal 1.0.0 +# TODO: add once asdf patched for glibc crystal +# source $HOME/.asdf/asdf.sh +# +# asdf install crystal 1.0.0 +# asdf global crystal 1.0.0 if [ -z ${GITHUB_ACTION+x} ] then @@ -39,14 +40,14 @@ do esac done -if [[ "$multithreaded" == "true" ]]; then +if [[ "${multithreaded}" == "true" ]]; then args="-Dpreview_mt" else args="" fi -if [[ "$watch" == "true" ]]; then +if [[ "${watch}" == "true" ]]; then CRYSTAL_WORKERS=$(nproc) watchexec -e cr -c -r -w src -w spec -- scripts/crystal-spec.sh -v ${args} else CRYSTAL_WORKERS=$(nproc) scripts/crystal-spec.sh -v ${args} -fi +fi \ No newline at end of file diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index f0c72de..d68a29e 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -2,6 +2,7 @@ set -eu -source $HOME/.asdf/asdf.sh +# TODO: After asdf support added +# source $HOME/.asdf/asdf.sh /app/bin/build "$@" diff --git a/shard.lock b/shard.lock index bdf1abd..ad6dbc5 100644 --- a/shard.lock +++ b/shard.lock @@ -1,9 +1,17 @@ version: 2.0 shards: + CrystalEmail: + git: https://github.com/nephos/crystalemail.git + version: 0.2.4 + action-controller: git: https://github.com/spider-gazelle/action-controller.git version: 4.5.0 + active-model: + git: https://github.com/spider-gazelle/active-model.git + version: 3.1.1 + ameba: git: https://github.com/crystal-ameba/ameba.git version: 0.14.3 @@ -16,6 +24,10 @@ shards: git: https://github.com/taylorfinnell/awscr-signer.git version: 0.8.2 + bindata: + git: https://github.com/spider-gazelle/bindata.git + version: 1.9.0 + clip: git: https://github.com/erdnaxeli/clip.git version: 0.2.4 @@ -32,6 +44,10 @@ shards: git: https://github.com/kostya/cron_parser.git version: 0.4.0 + db: + git: https://github.com/crystal-lang/crystal-db.git + version: 0.10.1 + debug: git: https://github.com/sija/debug.cr.git version: 2.0.1 @@ -68,6 +84,14 @@ shards: git: https://github.com/sija/ipaddress.cr.git version: 0.2.1 + json_mapping: + git: https://github.com/crystal-lang/json_mapping.cr.git + version: 0.1.1 + + jwt: + git: https://github.com/crystal-community/jwt.git + version: 1.5.1 + log_helper: git: https://github.com/spider-gazelle/log_helper.git version: 1.0.3 @@ -84,6 +108,10 @@ shards: git: https://github.com/aca-labs/murmur3.git version: 0.1.1+git.commit.7cbe25c0ca8d052c9d98c377c824dcb0e038c790 + neuroplastic: + git: https://github.com/spider-gazelle/neuroplastic.git + version: 1.7.1 + open_api: git: https://github.com/elbywan/open_api.cr.git version: 1.3.0 @@ -92,6 +120,10 @@ shards: git: https://github.com/place-labs/openapi-generator.git version: 2.1.0+git.commit.d925772ae1f36c51c0665d743bba4ac6ca9466a8 + openssl_ext: + git: https://github.com/spider-gazelle/openssl_ext.git + version: 2.1.5 + placeos-compiler: git: https://github.com/placeos/compiler.git version: 4.3.1 @@ -104,6 +136,10 @@ shards: git: https://github.com/place-labs/log-backend.git version: 0.4.0 + placeos-models: + git: https://github.com/placeos/models.git + version: 5.7.0 + pool: git: https://github.com/ysbaddaden/pool.git version: 0.2.4 @@ -124,6 +160,14 @@ shards: git: https://github.com/caspiano/rendezvous-hash.git version: 0.3.1 + rethinkdb: + git: https://github.com/kingsleyh/crystal-rethinkdb.git + version: 0.2.3 + + rethinkdb-orm: + git: https://github.com/spider-gazelle/rethinkdb-orm.git + version: 4.1.0 + retriable: git: https://github.com/sija/retriable.cr.git version: 0.2.4 @@ -164,3 +208,7 @@ shards: git: https://github.com/manastech/webmock.cr.git version: 0.14.0 + yaml_mapping: + git: https://github.com/crystal-lang/yaml_mapping.cr.git + version: 0.1.1 + diff --git a/shard.yml b/shard.yml index 5ee58f9..758f3dc 100644 --- a/shard.yml +++ b/shard.yml @@ -40,6 +40,9 @@ dependencies: version: ~> 4 placeos-log-backend: github: place-labs/log-backend + placeos-models: + github: placeos/models + version: ~> 5.6 secrets-env: github: place-labs/secrets-env shards: diff --git a/src/constants.cr b/src/constants.cr index 62cb21d..ba2ca5a 100644 --- a/src/constants.cr +++ b/src/constants.cr @@ -7,6 +7,8 @@ module PlaceOS::Build BUILD_TIME = {{ system("date -u").chomp.stringify }} BUILD_COMMIT = {{ env("PLACE_COMMIT") || "DEV" }} + CRYSTAL_VERSION = {{ env("CRYSTAL_VERSION") || "latest" }} + # S3 caching ############################################################################# diff --git a/src/placeos-build/api/root.cr b/src/placeos-build/api/root.cr new file mode 100644 index 0000000..0d5a532 --- /dev/null +++ b/src/placeos-build/api/root.cr @@ -0,0 +1,37 @@ +require "../api" +require "./application" + +require "placeos-models/version" + +module PlaceOS::Build::Api + # Routes trigger builds and query the resulting artefacts. + class Root < Application + include ::OpenAPI::Generator::Controller + include ::OpenAPI::Generator::Helpers::ActionController + + base "/api/build/v1" + + get("/", :root, annotations: @[OpenAPI(<<-YAML + summary: Service healthcheck + YAML + )]) do + head code: :ok + end + + get("/version", :version, annotations: @[OpenAPI(<<-YAML + summary: Service version + YAML + )]) do + render status_code: :ok, json: Root.version + end + + class_getter version : PlaceOS::Model::Version do + PlaceOS::Model::Version.new( + service: APP_NAME, + commit: BUILD_COMMIT, + version: VERSION, + build_time: BUILD_TIME, + ) + end + end +end