From 4775de60371bb7d2844baa14549d0cc86c11f3df Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 Nov 2023 20:11:09 +0100 Subject: [PATCH 1/4] ops: merge go op-stack build into dockerfile targets, rely on docker for ccross-build caching --- .circleci/config.yml | 49 ++-------- docker-bake.hcl | 123 ++++++++++++++++---------- op-batcher/Dockerfile | 9 -- op-batcher/Dockerfile.dockerignore | 2 - op-challenger/Dockerfile | 17 ---- op-challenger/Dockerfile.dockerignore | 2 - op-heartbeat/Dockerfile | 9 -- op-heartbeat/Dockerfile.dockerignore | 2 - op-node/Dockerfile | 9 -- op-node/Dockerfile.dockerignore | 2 - op-program/Dockerfile | 9 -- op-program/Dockerfile.dockerignore | 2 - op-proposer/Dockerfile | 9 -- op-proposer/Dockerfile.dockerignore | 2 - op-wheel/Dockerfile | 8 -- op-wheel/Dockerfile.dockerignore | 2 - ops-bedrock/docker-compose.yml | 27 ++---- ops/docker/op-stack-go/Dockerfile | 88 +++++++++++++----- 18 files changed, 153 insertions(+), 218 deletions(-) delete mode 100644 op-batcher/Dockerfile delete mode 100644 op-batcher/Dockerfile.dockerignore delete mode 100644 op-challenger/Dockerfile delete mode 100644 op-challenger/Dockerfile.dockerignore delete mode 100644 op-heartbeat/Dockerfile delete mode 100644 op-heartbeat/Dockerfile.dockerignore delete mode 100644 op-node/Dockerfile delete mode 100644 op-node/Dockerfile.dockerignore delete mode 100644 op-program/Dockerfile delete mode 100644 op-program/Dockerfile.dockerignore delete mode 100644 op-proposer/Dockerfile delete mode 100644 op-proposer/Dockerfile.dockerignore delete mode 100644 op-wheel/Dockerfile delete mode 100644 op-wheel/Dockerfile.dockerignore diff --git a/.circleci/config.yml b/.circleci/config.yml index 5d056dac3f9b..baf5c9270db6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1081,12 +1081,10 @@ jobs: command: | IMAGE_BASE_PREFIX="us-docker.pkg.dev/oplabs-tools-artifacts/images" # Load from previous docker-build job - docker load < "/tmp/workspace/op-stack-go.tar" docker load < "/tmp/workspace/op-node.tar" docker load < "/tmp/workspace/op-proposer.tar" docker load < "/tmp/workspace/op-batcher.tar" # rename to the tags that the docker-compose of the devnet expects - docker tag "$IMAGE_BASE_PREFIX/op-stack-go:<>" "$IMAGE_BASE_PREFIX/op-stack-go:devnet" docker tag "$IMAGE_BASE_PREFIX/op-node:<>" "$IMAGE_BASE_PREFIX/op-node:devnet" docker tag "$IMAGE_BASE_PREFIX/op-proposer:<>" "$IMAGE_BASE_PREFIX/op-proposer:devnet" docker tag "$IMAGE_BASE_PREFIX/op-batcher:<>" "$IMAGE_BASE_PREFIX/op-batcher:devnet" @@ -1479,46 +1477,35 @@ workflows: - op-e2e-WS-tests - op-e2e-HTTP-tests - op-e2e-ext-geth-tests - - docker-build: # just to warm up the cache (other jobs run in parallel) - name: op-stack-go-docker-build - docker_name: op-stack-go - docker_tags: <>,<> - save_image_tag: <> # for devnet later - docker-build: name: op-node-docker-build docker_name: op-node docker_tags: <>,<> - requires: ['op-stack-go-docker-build'] save_image_tag: <> # for devnet later - docker-build: name: op-batcher-docker-build docker_name: op-batcher docker_tags: <>,<> - requires: ['op-stack-go-docker-build'] save_image_tag: <> # for devnet later - docker-build: name: op-program-docker-build docker_name: op-program docker_tags: <>,<> - requires: ['op-stack-go-docker-build'] save_image_tag: <> # for devnet later - docker-build: name: op-proposer-docker-build docker_name: op-proposer docker_tags: <>,<> - requires: ['op-stack-go-docker-build'] save_image_tag: <> # for devnet later - docker-build: name: op-challenger-docker-build docker_name: op-challenger docker_tags: <>,<> - requires: ['op-stack-go-docker-build'] save_image_tag: <> # for devnet later - docker-build: name: op-heartbeat-docker-build docker_name: op-heartbeat docker_tags: <>,<> - requires: ['op-stack-go-docker-build'] save_image_tag: <> # for devnet later - cannon-prestate: requires: ["op-stack-go-lint"] @@ -1554,18 +1541,6 @@ workflows: only: /^(proxyd|chain-mon|indexer|ci-builder|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/ branches: ignore: /.*/ - - docker-build: # just to warm up the cache (other jobs run in parallel) - name: op-stack-go-docker-build-release - filters: - tags: - only: /^(proxyd|chain-mon|indexer|ci-builder|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/ - branches: - ignore: /.*/ - docker_name: op-stack-go - docker_tags: <> - platforms: "linux/amd64,linux/arm64" - requires: - - hold - docker-build: name: op-heartbeat-release filters: @@ -1575,7 +1550,7 @@ workflows: ignore: /.*/ docker_name: op-heartbeat docker_tags: <> - requires: ['op-stack-go-docker-build-release'] + requires: ['hold'] platforms: "linux/amd64,linux/arm64" publish: true release: true @@ -1590,7 +1565,7 @@ workflows: ignore: /.*/ docker_name: op-node docker_tags: <> - requires: ['op-stack-go-docker-build-release'] + requires: ['hold'] platforms: "linux/amd64,linux/arm64" publish: true release: true @@ -1605,7 +1580,7 @@ workflows: ignore: /.*/ docker_name: op-batcher docker_tags: <> - requires: ['op-stack-go-docker-build-release'] + requires: ['hold'] platforms: "linux/amd64,linux/arm64" publish: true release: true @@ -1620,7 +1595,7 @@ workflows: ignore: /.*/ docker_name: op-proposer docker_tags: <> - requires: ['op-stack-go-docker-build-release'] + requires: ['hold'] platforms: "linux/amd64,linux/arm64" publish: true release: true @@ -1635,7 +1610,7 @@ workflows: ignore: /.*/ docker_name: op-challenger docker_tags: <> - requires: ['op-stack-go-docker-build-release'] + requires: ['hold'] platforms: "linux/amd64,linux/arm64" publish: true release: true @@ -1763,19 +1738,10 @@ workflows: when: equal: [ build_hourly, <> ] jobs: - - docker-build: # just to warm up the cache (other jobs run in parallel) - name: op-stack-go-docker-build-publish - docker_name: op-stack-go - docker_tags: <>,<> - platforms: "linux/amd64,linux/arm64" - context: - - oplabs-gcr - - slack - docker-build: name: op-node-docker-publish docker_name: op-node docker_tags: <>,<> - requires: [ 'op-stack-go-docker-build-publish' ] platforms: "linux/amd64,linux/arm64" publish: true context: @@ -1785,7 +1751,6 @@ workflows: name: op-batcher-docker-publish docker_name: op-batcher docker_tags: <>,<> - requires: [ 'op-stack-go-docker-build-publish' ] platforms: "linux/amd64,linux/arm64" publish: true context: @@ -1795,7 +1760,6 @@ workflows: name: op-program-docker-publish docker_name: op-program docker_tags: <>,<> - requires: [ 'op-stack-go-docker-build-publish' ] platforms: "linux/amd64,linux/arm64" publish: true context: @@ -1805,7 +1769,6 @@ workflows: name: op-proposer-docker-publish docker_name: op-proposer docker_tags: <>,<> - requires: [ 'op-stack-go-docker-build-publish' ] platforms: "linux/amd64,linux/arm64" publish: true context: @@ -1815,7 +1778,6 @@ workflows: name: op-challenger-docker-publish docker_name: op-challenger docker_tags: <>,<> - requires: [ 'op-stack-go-docker-build-publish' ] platforms: "linux/amd64,linux/arm64" publish: true context: @@ -1825,7 +1787,6 @@ workflows: name: op-heartbeat-docker-publish docker_name: op-heartbeat docker_tags: <>,<> - requires: [ 'op-stack-go-docker-build-publish' ] platforms: "linux/amd64,linux/arm64" publish: true context: diff --git a/docker-bake.hcl b/docker-bake.hcl index ae8bdd84a583..42e04d0b147c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -14,8 +14,10 @@ variable "GIT_DATE" { default = "0" } +// The default version to embed in the built images. +// During CI release builds this is set to <> variable "GIT_VERSION" { - default = "docker" // original default as set in proxyd file, not used by full go stack, yet + default = "v0.0.0" } variable "IMAGE_TAGS" { @@ -30,95 +32,126 @@ variable "PLATFORMS" { default = "linux/amd64" } -target "op-stack-go" { +// Each of the services can have a customized version, but defaults to the global specified version. +variable "OP_NODE_VERSION" { + default = "${GIT_VERSION}" +} + +variable "OP_BATCHER_VERSION" { + default = "${GIT_VERSION}" +} + +variable "OP_PROPOSER_VERSION" { + default = "${GIT_VERSION}" +} + +variable "OP_CHALLENGER_VERSION" { + default = "${GIT_VERSION}" +} + +variable OP_HEARTBEAT_VERSION { + default = "${GIT_VERSION}" +} + +variable OP_PROGRAM_VERSION { + default = "${GIT_VERSION}" +} + +variable CANNON_VERSION { + default = "${GIT_VERSION}" +} + +target "op-node" { dockerfile = "ops/docker/op-stack-go/Dockerfile" context = "." args = { GIT_COMMIT = "${GIT_COMMIT}" GIT_DATE = "${GIT_DATE}" + OP_NODE_VERSION = "${OP_NODE_VERSION}" } - platforms = split(",", PLATFORMS) - tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-stack-go:${tag}"] -} - -target "op-node" { - dockerfile = "Dockerfile" - context = "./op-node" - args = { - OP_STACK_GO_BUILDER = "op-stack-go" - } - contexts = { - op-stack-go: "target:op-stack-go" - } + target = "op-node-target" platforms = split(",", PLATFORMS) tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-node:${tag}"] } target "op-batcher" { - dockerfile = "Dockerfile" - context = "./op-batcher" + dockerfile = "ops/docker/op-stack-go/Dockerfile" + context = "." args = { - OP_STACK_GO_BUILDER = "op-stack-go" - } - contexts = { - op-stack-go: "target:op-stack-go" + GIT_COMMIT = "${GIT_COMMIT}" + GIT_DATE = "${GIT_DATE}" + OP_BATCHER_VERSION = "${OP_BATCHER_VERSION}" } + target = "op-batcher-target" platforms = split(",", PLATFORMS) tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-batcher:${tag}"] } target "op-proposer" { - dockerfile = "Dockerfile" - context = "./op-proposer" + dockerfile = "ops/docker/op-stack-go/Dockerfile" + context = "." args = { - OP_STACK_GO_BUILDER = "op-stack-go" - } - contexts = { - op-stack-go: "target:op-stack-go" + GIT_COMMIT = "${GIT_COMMIT}" + GIT_DATE = "${GIT_DATE}" + OP_PROPOSER_VERSION = "${OP_PROPOSER_VERSION}" } + target = "op-proposer-target" platforms = split(",", PLATFORMS) tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-proposer:${tag}"] } target "op-challenger" { - dockerfile = "Dockerfile" - context = "./op-challenger" + dockerfile = "ops/docker/op-stack-go/Dockerfile" + context = "." args = { - OP_STACK_GO_BUILDER = "op-stack-go" - } - contexts = { - op-stack-go: "target:op-stack-go" + GIT_COMMIT = "${GIT_COMMIT}" + GIT_DATE = "${GIT_DATE}" + OP_CHALLENGER_VERSION = "${OP_CHALLENGER_VERSION}" } + target = "op-challenger-target" platforms = split(",", PLATFORMS) tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-challenger:${tag}"] } target "op-heartbeat" { - dockerfile = "Dockerfile" - context = "./op-heartbeat" + dockerfile = "ops/docker/op-stack-go/Dockerfile" + context = "." args = { - OP_STACK_GO_BUILDER = "op-stack-go" - } - contexts = { - op-stack-go: "target:op-stack-go" + GIT_COMMIT = "${GIT_COMMIT}" + GIT_DATE = "${GIT_DATE}" + OP_HEARTBEAT_VERSION = "${OP_HEARTBEAT_VERSION}" } + target = "op-heartbeat-target" platforms = split(",", PLATFORMS) tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-heartbeat:${tag}"] } target "op-program" { - dockerfile = "Dockerfile" - context = "./op-program" + dockerfile = "ops/docker/op-stack-go/Dockerfile" + context = "." args = { - OP_STACK_GO_BUILDER = "op-stack-go" - } - contexts = { - op-stack-go: "target:op-stack-go" + GIT_COMMIT = "${GIT_COMMIT}" + GIT_DATE = "${GIT_DATE}" + OP_PROGRAM_VERSION = "${OP_PROGRAM_VERSION}" } + target = "op-program-target" platforms = split(",", PLATFORMS) tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-program:${tag}"] } +target "cannon" { + dockerfile = "ops/docker/op-stack-go/Dockerfile" + context = "." + args = { + GIT_COMMIT = "${GIT_COMMIT}" + GIT_DATE = "${GIT_DATE}" + CANNON_VERSION = "${CANNON_VERSION}" + } + target = "cannon-target" + platforms = split(",", PLATFORMS) + tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/cannon:${tag}"] +} + target "proxyd" { dockerfile = "Dockerfile" context = "./proxyd" diff --git a/op-batcher/Dockerfile b/op-batcher/Dockerfile deleted file mode 100644 index 6732ecc65678..000000000000 --- a/op-batcher/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest -FROM $OP_STACK_GO_BUILDER as builder -# See "make golang-docker" and /ops/docker/op-stack-go - -FROM alpine:3.18 - -COPY --from=builder /usr/local/bin/op-batcher /usr/local/bin/op-batcher - -CMD ["op-batcher"] diff --git a/op-batcher/Dockerfile.dockerignore b/op-batcher/Dockerfile.dockerignore deleted file mode 100644 index 4f26140750a0..000000000000 --- a/op-batcher/Dockerfile.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# ignore everything but the dockerfile, the op-stack-go base image performs the build -* diff --git a/op-challenger/Dockerfile b/op-challenger/Dockerfile deleted file mode 100644 index ad57e2b52b6f..000000000000 --- a/op-challenger/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest -FROM $OP_STACK_GO_BUILDER as builder -# See "make golang-docker" and /ops/docker/op-stack-go - -FROM alpine:3.18 - -# Make the bundled op-program the default cannon server -COPY --from=builder /usr/local/bin/op-program /usr/local/bin/op-program -ENV OP_CHALLENGER_CANNON_SERVER /usr/local/bin/op-program - -# Make the bundled cannon the default cannon executable -COPY --from=builder /usr/local/bin/cannon /usr/local/bin/cannon -ENV OP_CHALLENGER_CANNON_BIN /usr/local/bin/cannon - -COPY --from=builder /usr/local/bin/op-challenger /usr/local/bin/op-challenger - -CMD ["op-challenger"] diff --git a/op-challenger/Dockerfile.dockerignore b/op-challenger/Dockerfile.dockerignore deleted file mode 100644 index 4f26140750a0..000000000000 --- a/op-challenger/Dockerfile.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# ignore everything but the dockerfile, the op-stack-go base image performs the build -* diff --git a/op-heartbeat/Dockerfile b/op-heartbeat/Dockerfile deleted file mode 100644 index 2e3996cde666..000000000000 --- a/op-heartbeat/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest -FROM $OP_STACK_GO_BUILDER as builder -# See "make golang-docker" and /ops/docker/op-stack-go - -FROM alpine:3.18 - -COPY --from=builder /usr/local/bin/op-heartbeat /usr/local/bin/op-heartbeat - -CMD ["op-heartbeat"] diff --git a/op-heartbeat/Dockerfile.dockerignore b/op-heartbeat/Dockerfile.dockerignore deleted file mode 100644 index 4f26140750a0..000000000000 --- a/op-heartbeat/Dockerfile.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# ignore everything but the dockerfile, the op-stack-go base image performs the build -* diff --git a/op-node/Dockerfile b/op-node/Dockerfile deleted file mode 100644 index 0cd7f16d9c1d..000000000000 --- a/op-node/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest -FROM $OP_STACK_GO_BUILDER as builder -# See "make golang-docker" and /ops/docker/op-stack-go - -FROM alpine:3.18 - -COPY --from=builder /usr/local/bin/op-node /usr/local/bin/op-node - -CMD ["op-node"] diff --git a/op-node/Dockerfile.dockerignore b/op-node/Dockerfile.dockerignore deleted file mode 100644 index 4f26140750a0..000000000000 --- a/op-node/Dockerfile.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# ignore everything but the dockerfile, the op-stack-go base image performs the build -* diff --git a/op-program/Dockerfile b/op-program/Dockerfile deleted file mode 100644 index 91688b51607d..000000000000 --- a/op-program/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest -FROM $OP_STACK_GO_BUILDER as builder -# See "make golang-docker" and /ops/docker/op-stack-go - -FROM alpine:3.18 - -COPY --from=builder /usr/local/bin/op-program /usr/local/bin/op-program - -CMD ["op-program"] diff --git a/op-program/Dockerfile.dockerignore b/op-program/Dockerfile.dockerignore deleted file mode 100644 index 4f26140750a0..000000000000 --- a/op-program/Dockerfile.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# ignore everything but the dockerfile, the op-stack-go base image performs the build -* diff --git a/op-proposer/Dockerfile b/op-proposer/Dockerfile deleted file mode 100644 index 8eb4b7cc0ebb..000000000000 --- a/op-proposer/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest -FROM $OP_STACK_GO_BUILDER as builder -# See "make golang-docker" and /ops/docker/op-stack-go - -FROM alpine:3.18 - -COPY --from=builder /usr/local/bin/op-proposer /usr/local/bin/op-proposer - -CMD ["op-proposer"] diff --git a/op-proposer/Dockerfile.dockerignore b/op-proposer/Dockerfile.dockerignore deleted file mode 100644 index 4f26140750a0..000000000000 --- a/op-proposer/Dockerfile.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# ignore everything but the dockerfile, the op-stack-go base image performs the build -* diff --git a/op-wheel/Dockerfile b/op-wheel/Dockerfile deleted file mode 100644 index 30f7a8a3713a..000000000000 --- a/op-wheel/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest -FROM $OP_STACK_GO_BUILDER as builder -# See "make golang-docker" and /ops/docker/op-stack-go -FROM alpine:3.18 - -COPY --from=builder /app/op-wheel/bin/op-wheel /usr/local/bin - -CMD ["op-wheel"] diff --git a/op-wheel/Dockerfile.dockerignore b/op-wheel/Dockerfile.dockerignore deleted file mode 100644 index 4f26140750a0..000000000000 --- a/op-wheel/Dockerfile.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# ignore everything but the dockerfile, the op-stack-go base image performs the build -* diff --git a/ops-bedrock/docker-compose.yml b/ops-bedrock/docker-compose.yml index 12216ab510ff..ce49b4049eda 100644 --- a/ops-bedrock/docker-compose.yml +++ b/ops-bedrock/docker-compose.yml @@ -11,15 +11,6 @@ volumes: services: - op_stack_go_builder: # Not an actual service, but builds the prerequisite go images - build: - context: ../ - dockerfile: ops/docker/op-stack-go/Dockerfile - args: - GIT_COMMIT: "dev" - GIT_DATE: "0" - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:devnet - entrypoint: ["echo", "build complete"] l1: build: @@ -56,14 +47,12 @@ services: op-node: depends_on: - - op_stack_go_builder - l1 - l2 build: context: ../ - dockerfile: ./op-node/Dockerfile - args: - OP_STACK_GO_BUILDER: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:devnet + dockerfile: ops/docker/op-stack-go/Dockerfile + target: op-node-target image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:devnet command: > op-node @@ -103,15 +92,13 @@ services: op-proposer: depends_on: - - op_stack_go_builder - l1 - l2 - op-node build: context: ../ - dockerfile: ./op-proposer/Dockerfile - args: - OP_STACK_GO_BUILDER: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:devnet + dockerfile: ops/docker/op-stack-go/Dockerfile + target: op-proposer-target image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-proposer:devnet ports: - "6062:6060" @@ -132,15 +119,13 @@ services: op-batcher: depends_on: - - op_stack_go_builder - l1 - l2 - op-node build: context: ../ - dockerfile: ./op-batcher/Dockerfile - args: - OP_STACK_GO_BUILDER: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:devnet + dockerfile: ops/docker/op-stack-go/Dockerfile + target: op-batcher-target image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-batcher:devnet ports: - "6061:6060" diff --git a/ops/docker/op-stack-go/Dockerfile b/ops/docker/op-stack-go/Dockerfile index 87d92ca39233..f864cdce4fae 100644 --- a/ops/docker/op-stack-go/Dockerfile +++ b/ops/docker/op-stack-go/Dockerfile @@ -1,3 +1,12 @@ +# automatically set by buildkit, can be changed with --platform flag +ARG TARGETOS +ARG TARGETARCH + +# All target images use this as base image, and add the final build results. +# It will default to the target platform. +ARG TARGET_BASE_IMAGE=alpine:3.18 + +# We may be cross-building for another platform. Specify which platform we need as builder. FROM --platform=$BUILDPLATFORM golang:1.21.3-alpine3.18 as builder RUN apk add --no-cache make gcc musl-dev linux-headers git jq bash @@ -11,6 +20,7 @@ WORKDIR /app RUN echo "go mod cache: $(go env GOMODCACHE)" RUN echo "go build cache: $(go env GOCACHE)" +# warm-up the cache RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod download # NOTE: the Dockerfile.dockerignore file effectively describes all dependencies @@ -23,62 +33,92 @@ COPY . /app ARG GIT_COMMIT ARG GIT_DATE -ARG CANNON_VERSION=v0.0.0 -ARG OP_PROGRAM_VERSION=v0.0.0 - -ARG OP_HEARTBEAT_VERSION=v0.0.0 -ARG OP_WHEEL_VERSION=v0.0.0 - -ARG OP_NODE_VERSION=v0.0.0 -ARG OP_CHALLENGER_VERSION=v0.0.0 -ARG OP_BATCHER_VERSION=v0.0.0 -ARG OP_PROPOSER_VERSION=v0.0.0 - - # separate docker-builds: # - op-exporter # - op-ufm # - proxyd # - any JS/TS/smart-contract builds -ARG TARGETOS TARGETARCH - # Build the Go services, utilizing caches and share the many common packages. # The "id" defaults to the value of "target", the cache will thus be reused during this build. # "sharing" defaults to "shared", the cache will thus be available to other concurrent docker builds. - +FROM builder as cannon-builder +ARG CANNON_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd cannon && make cannon \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$CANNON_VERSION" + +FROM builder as op-program-builder +ARG OP_PROGRAM_VERSION=v0.0.0 # note: we only build the host, that's all the user needs. No Go MIPS cross-build in docker RUN --mount=type=cache,target=/root/.cache/go-build cd op-program && make op-program-host \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_PROGRAM_VERSION" +FROM builder as op-heartbeat-builder +ARG OP_HEARTBEAT_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-heartbeat && make op-heartbeat \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_HEARTBEAT_VERSION" + +FROM builder as op-wheel-builder +ARG OP_WHEEL_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-wheel && make op-wheel \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_WHEEL_VERSION" +FROM builder as op-node-builder +ARG OP_NODE_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-node && make op-node \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_NODE_VERSION" + +FROM builder as op-challenger-builder +ARG OP_CHALLENGER_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-challenger && make op-challenger \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_CHALLENGER_VERSION" + +FROM builder as op-batcher-builder +ARG OP_BATCHER_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-batcher && make op-batcher \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_BATCHER_VERSION" + +FROM builder as op-proposer-builder +ARG OP_PROPOSER_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-proposer && make op-proposer \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_PROPOSER_VERSION" -FROM alpine:3.18 +FROM $TARGET_BASE_IMAGE as cannon-target +COPY --from=cannon-builder /app/cannon/bin/cannon /usr/local/bin/ +CMD ["cannon"] + +FROM $TARGET_BASE_IMAGE as op-program-target +COPY --from=op-program-builder /app/op-program/bin/op-program /usr/local/bin/ +CMD ["op-program"] + +FROM $TARGET_BASE_IMAGE as op-heartbeat-target +COPY --from=op-heartbeat-builder /app/op-heartbeat/bin/op-heartbeat /usr/local/bin/ +CMD ["op-heartbeat"] + +FROM $TARGET_BASE_IMAGE as op-wheel-target +COPY --from=op-wheel-builder /app/op-wheel/bin/op-wheel /usr/local/bin/ +CMD ["op-wheel"] -COPY --from=builder /app/cannon/bin/cannon /usr/local/bin/ -COPY --from=builder /app/op-program/bin/op-program /usr/local/bin/ +FROM $TARGET_BASE_IMAGE as op-node-target +COPY --from=op-node-builder /app/op-node/bin/op-node /usr/local/bin/ +CMD ["op-node"] -COPY --from=builder /app/op-heartbeat/bin/op-heartbeat /usr/local/bin/ -COPY --from=builder /app/op-wheel/bin/op-wheel /usr/local/bin/ +FROM $TARGET_BASE_IMAGE as op-challenger-target +COPY --from=op-challenger-builder /app/op-challenger/bin/op-challenger /usr/local/bin/ +# Make the bundled op-program the default cannon server +COPY --from=op-program-builder /app/op-program/bin/op-program /usr/local/bin/ +ENV OP_CHALLENGER_CANNON_SERVER /usr/local/bin/op-program +# Make the bundled cannon the default cannon executable +COPY --from=cannon-builder /app/cannon/bin/cannon /usr/local/bin/ +ENV OP_CHALLENGER_CANNON_BIN /usr/local/bin/cannon +CMD ["op-challenger"] -COPY --from=builder /app/op-node/bin/op-node /usr/local/bin/ -COPY --from=builder /app/op-challenger/bin/op-challenger /usr/local/bin/ -COPY --from=builder /app/op-batcher/bin/op-batcher /usr/local/bin/ -COPY --from=builder /app/op-proposer/bin/op-proposer /usr/local/bin/ +FROM $TARGET_BASE_IMAGE as op-batcher-target +COPY --from=op-batcher-builder /app/op-batcher/bin/op-batcher /usr/local/bin/ +CMD ["op-batcher"] +FROM $TARGET_BASE_IMAGE as op-proposer-target +COPY --from=op-proposer-builder /app/op-proposer/bin/op-proposer /usr/local/bin/ +CMD ["op-proposer"] From af58f3ead8e0f553b089b8ac16c8ba5c48e553ce Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 Nov 2023 23:15:48 +0100 Subject: [PATCH 2/4] ops: make all docker image platform choices explicit --- ops/docker/op-stack-go/Dockerfile | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ops/docker/op-stack-go/Dockerfile b/ops/docker/op-stack-go/Dockerfile index f864cdce4fae..d613debc52e0 100644 --- a/ops/docker/op-stack-go/Dockerfile +++ b/ops/docker/op-stack-go/Dockerfile @@ -42,69 +42,69 @@ ARG GIT_DATE # Build the Go services, utilizing caches and share the many common packages. # The "id" defaults to the value of "target", the cache will thus be reused during this build. # "sharing" defaults to "shared", the cache will thus be available to other concurrent docker builds. -FROM builder as cannon-builder +FROM --platform=$BUILDPLATFORM builder as cannon-builder ARG CANNON_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd cannon && make cannon \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$CANNON_VERSION" -FROM builder as op-program-builder +FROM --platform=$BUILDPLATFORM builder as op-program-builder ARG OP_PROGRAM_VERSION=v0.0.0 # note: we only build the host, that's all the user needs. No Go MIPS cross-build in docker RUN --mount=type=cache,target=/root/.cache/go-build cd op-program && make op-program-host \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_PROGRAM_VERSION" -FROM builder as op-heartbeat-builder +FROM --platform=$BUILDPLATFORM builder as op-heartbeat-builder ARG OP_HEARTBEAT_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-heartbeat && make op-heartbeat \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_HEARTBEAT_VERSION" -FROM builder as op-wheel-builder +FROM --platform=$BUILDPLATFORM builder as op-wheel-builder ARG OP_WHEEL_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-wheel && make op-wheel \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_WHEEL_VERSION" -FROM builder as op-node-builder +FROM --platform=$BUILDPLATFORM builder as op-node-builder ARG OP_NODE_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-node && make op-node \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_NODE_VERSION" -FROM builder as op-challenger-builder +FROM --platform=$BUILDPLATFORM builder as op-challenger-builder ARG OP_CHALLENGER_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-challenger && make op-challenger \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_CHALLENGER_VERSION" -FROM builder as op-batcher-builder +FROM --platform=$BUILDPLATFORM builder as op-batcher-builder ARG OP_BATCHER_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-batcher && make op-batcher \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_BATCHER_VERSION" -FROM builder as op-proposer-builder +FROM --platform=$BUILDPLATFORM builder as op-proposer-builder ARG OP_PROPOSER_VERSION=v0.0.0 RUN --mount=type=cache,target=/root/.cache/go-build cd op-proposer && make op-proposer \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_PROPOSER_VERSION" -FROM $TARGET_BASE_IMAGE as cannon-target +FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE as cannon-target COPY --from=cannon-builder /app/cannon/bin/cannon /usr/local/bin/ CMD ["cannon"] -FROM $TARGET_BASE_IMAGE as op-program-target +FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE as op-program-target COPY --from=op-program-builder /app/op-program/bin/op-program /usr/local/bin/ CMD ["op-program"] -FROM $TARGET_BASE_IMAGE as op-heartbeat-target +FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE as op-heartbeat-target COPY --from=op-heartbeat-builder /app/op-heartbeat/bin/op-heartbeat /usr/local/bin/ CMD ["op-heartbeat"] -FROM $TARGET_BASE_IMAGE as op-wheel-target +FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE as op-wheel-target COPY --from=op-wheel-builder /app/op-wheel/bin/op-wheel /usr/local/bin/ CMD ["op-wheel"] -FROM $TARGET_BASE_IMAGE as op-node-target +FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE as op-node-target COPY --from=op-node-builder /app/op-node/bin/op-node /usr/local/bin/ CMD ["op-node"] -FROM $TARGET_BASE_IMAGE as op-challenger-target +FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE as op-challenger-target COPY --from=op-challenger-builder /app/op-challenger/bin/op-challenger /usr/local/bin/ # Make the bundled op-program the default cannon server COPY --from=op-program-builder /app/op-program/bin/op-program /usr/local/bin/ @@ -114,11 +114,11 @@ COPY --from=cannon-builder /app/cannon/bin/cannon /usr/local/bin/ ENV OP_CHALLENGER_CANNON_BIN /usr/local/bin/cannon CMD ["op-challenger"] -FROM $TARGET_BASE_IMAGE as op-batcher-target +FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE as op-batcher-target COPY --from=op-batcher-builder /app/op-batcher/bin/op-batcher /usr/local/bin/ CMD ["op-batcher"] -FROM $TARGET_BASE_IMAGE as op-proposer-target +FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE as op-proposer-target COPY --from=op-proposer-builder /app/op-proposer/bin/op-proposer /usr/local/bin/ CMD ["op-proposer"] From 24fd72773df52362ee7eefe21e80b01dc7a1c898 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 Nov 2023 23:41:46 +0100 Subject: [PATCH 3/4] ops: set default PLATFORMS var to local platform --- docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 42e04d0b147c..da95bfcf1557 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -29,7 +29,7 @@ variable "PLATFORMS" { // Only a specify a single platform when `--load` ing into docker. // Multi-platform is supported when outputting to disk or pushing to a registry. // Multi-platform builds can be tested locally with: --set="*.output=type=image,push=false" - default = "linux/amd64" + default = "${BAKE_LOCAL_PLATFORM}" } // Each of the services can have a customized version, but defaults to the global specified version. From cea9b0e89754c54e23f7c62f6764f4964f96701e Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 Nov 2023 10:44:15 +0100 Subject: [PATCH 4/4] ops: empty default platform value, for docker to resolve to something sensible on arm mac --- docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index da95bfcf1557..f865c0996d58 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -29,7 +29,7 @@ variable "PLATFORMS" { // Only a specify a single platform when `--load` ing into docker. // Multi-platform is supported when outputting to disk or pushing to a registry. // Multi-platform builds can be tested locally with: --set="*.output=type=image,push=false" - default = "${BAKE_LOCAL_PLATFORM}" + default = "" } // Each of the services can have a customized version, but defaults to the global specified version.