From f2a4328a4b87e1532f115293464668000052f2d1 Mon Sep 17 00:00:00 2001 From: Ciara Stacke <c.stacke@f5.com> Date: Tue, 17 Nov 2020 20:44:14 +0000 Subject: [PATCH] Use buildkit secret flag for NGINX plus images The new --secret flag for docker build allows the user to pass secret information to be used in the Dockerfile for building docker images in a safe way that will not end up stored in the final image. This commit introduces this change for transferring the cert and key for building the NGINX Plus images. --- .github/workflows/edge.yml | 102 ++++++++++-------- .github/workflows/nightly.yml | 102 ++++++++++-------- .gitignore | 1 + Makefile | 12 ++- build/DockerfileForPlus | 15 ++- build/DockerfileWithOpentracingForPlus | 15 ++- .../DockerfileWithAppProtectForPlus | 15 ++- ...ockerfileWithAppProtectForPlusForOpenShift | 22 ++-- build/openshift/DockerfileForPlus | 16 ++- 9 files changed, 170 insertions(+), 130 deletions(-) diff --git a/.github/workflows/edge.yml b/.github/workflows/edge.yml index 027aee618b..af9b952753 100644 --- a/.github/workflows/edge.yml +++ b/.github/workflows/edge.yml @@ -158,24 +158,6 @@ jobs: with: path: ${{ github.workspace }}/nginx-ingress key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }} - - name: Fetch N+ Certificates - env: - CRT: ${{ secrets.NGINX_CRT }} - KEY: ${{ secrets.NGINX_KEY }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - if: matrix.type == 'plus' - - name: Fetch N+ AppProtect Certificates - env: - CRT: ${{ secrets.NGINX_AP_CRT }} - KEY: ${{ secrets.NGINX_AP_KEY }} - RHEL_LICENSE: ${{ secrets.RHEL_LICENSE }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - echo "${RHEL_LICENSE}" | base64 --decode > rhel_license - if: matrix.type == 'plus-ap' || matrix.type == 'plus-ap-openshift' - name: Docker Buildx uses: docker/setup-buildx-action@v1 with: @@ -196,6 +178,34 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache target: ${{ matrix.target }} tags: ${{ matrix.image }}:${{ github.sha }} + if: matrix.type == 'oss' + - name: Build Plus Docker Image ${{ matrix.image }} + uses: docker/build-push-action@v2 + with: + file: ${{ matrix.file }} + context: ${{ matrix.context }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + target: ${{ matrix.target }} + tags: ${{ matrix.image }}:${{ github.sha }} + secrets: | + nginx-repo.crt=${{ secrets.NGINX_CRT }} + nginx-repo.key=${{ secrets.NGINX_KEY }} + if: matrix.type == 'plus' + - name: Build AP Docker Image ${{ matrix.image }} + uses: docker/build-push-action@v2 + with: + file: ${{ matrix.file }} + context: ${{ matrix.context }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + target: ${{ matrix.target }} + tags: ${{ matrix.image }}:${{ github.sha }} + secrets: | + nginx-repo.crt=${{ secrets.NGINX_AP_CRT }} + nginx-repo.key=${{ secrets.NGINX_AP_KEY }} + rhel_license=${{ secrets.RHEL_LICENSE }} + if: matrix.type == 'plus-ap' || matrix.type == 'plus-ap-openshift' smoke-tests: name: Smoke Tests @@ -269,22 +279,6 @@ jobs: with: path: ${{ github.workspace }}/nginx-ingress key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }} - - name: Fetch N+ Certificates - env: - CRT: ${{ secrets.NGINX_CRT }} - KEY: ${{ secrets.NGINX_KEY }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - if: matrix.type == 'plus' - - name: Fetch N+ AppProtect Certificates - env: - CRT: ${{ secrets.NGINX_AP_CRT }} - KEY: ${{ secrets.NGINX_AP_KEY }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - if: matrix.type == 'plus-ap' - name: Docker Buildx uses: docker/setup-buildx-action@v1 with: @@ -306,6 +300,35 @@ jobs: target: local tags: ${{ matrix.image }}:${{ matrix.tag }} load: true + if: matrix.type == 'oss' + - name: Build Plus Docker Image ${{ matrix.image }} + uses: docker/build-push-action@v2 + with: + file: ${{ matrix.file }} + context: '.' + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + target: local + tags: ${{ matrix.image }}:${{ matrix.tag }} + load: true + secrets: | + nginx-repo.crt=${{ secrets.NGINX_CRT }} + nginx-repo.key=${{ secrets.NGINX_KEY }} + if: matrix.type == 'plus' + - name: Build AP Docker Image ${{ matrix.image }} + uses: docker/build-push-action@v2 + with: + file: ${{ matrix.file }} + context: '.' + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + target: local + tags: ${{ matrix.image }}:${{ matrix.tag }} + load: true + secrets: | + nginx-repo.crt=${{ secrets.NGINX_AP_CRT }} + nginx-repo.key=${{ secrets.NGINX_AP_KEY }} + if: matrix.type == 'plus-ap' - name: Build Test-Runner Container uses: docker/build-push-action@v2 with: @@ -391,14 +414,6 @@ jobs: with: path: ${{ github.workspace }}/nginx-ingress key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }} - - name: Fetch N+ Certificates - env: - CRT: ${{ secrets.NGINX_CRT }} - KEY: ${{ secrets.NGINX_KEY }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - if: matrix.type == 'plus' - name: Docker Buildx uses: docker/setup-buildx-action@v1 with: @@ -420,6 +435,9 @@ jobs: target: local tags: ${{ matrix.image }}:${{ matrix.tag }} load: true + secrets: | + nginx-repo.crt=${{ secrets.NGINX_CRT }} + nginx-repo.key=${{ secrets.NGINX_KEY }} - name: Deploy Kubernetes id: k8s run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 45065ce9ca..06f20507c8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -128,24 +128,6 @@ jobs: with: path: ${{ github.workspace }}/nginx-ingress key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }} - - name: Fetch N+ Certificates - env: - CRT: ${{ secrets.NGINX_CRT }} - KEY: ${{ secrets.NGINX_KEY }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - if: matrix.type == 'plus' - - name: Fetch N+ AppProtect Certificates - env: - CRT: ${{ secrets.NGINX_AP_CRT }} - KEY: ${{ secrets.NGINX_AP_KEY }} - RHEL_LICENSE: ${{ secrets.RHEL_LICENSE }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - echo "${RHEL_LICENSE}" | base64 --decode > rhel_license - if: matrix.type == 'plus-ap' || matrix.type == 'plus-ap-openshift' - name: Docker Buildx uses: docker/setup-buildx-action@v1 with: @@ -166,6 +148,34 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache target: ${{ matrix.target }} tags: ${{ matrix.image }}:${{ github.sha }} + if: matrix.type == 'oss' + - name: Build Plus Docker Image ${{ matrix.image }} + uses: docker/build-push-action@v2 + with: + file: ${{ matrix.file }} + context: ${{ matrix.context }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + target: ${{ matrix.target }} + tags: ${{ matrix.image }}:${{ github.sha }} + secrets: | + nginx-repo.crt=${{ secrets.NGINX_CRT }} + nginx-repo.key=${{ secrets.NGINX_KEY }} + if: matrix.type == 'plus' + - name: Build AP Docker Image ${{ matrix.image }} + uses: docker/build-push-action@v2 + with: + file: ${{ matrix.file }} + context: ${{ matrix.context }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + target: ${{ matrix.target }} + tags: ${{ matrix.image }}:${{ github.sha }} + secrets: | + nginx-repo.crt=${{ secrets.NGINX_AP_CRT }} + nginx-repo.key=${{ secrets.NGINX_AP_KEY }} + rhel_license=${{ secrets.RHEL_LICENSE }} + if: matrix.type == 'plus-ap' || matrix.type == 'plus-ap-openshift' smoke-tests: name: Smoke Tests @@ -287,22 +297,6 @@ jobs: with: path: ${{ github.workspace }}/nginx-ingress key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }} - - name: Fetch N+ Certificates - env: - CRT: ${{ secrets.NGINX_CRT }} - KEY: ${{ secrets.NGINX_KEY }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - if: matrix.type == 'plus' - - name: Fetch N+ AppProtect Certificates - env: - CRT: ${{ secrets.NGINX_AP_CRT }} - KEY: ${{ secrets.NGINX_AP_KEY }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - if: matrix.type == 'plus-ap' - name: Docker Buildx uses: docker/setup-buildx-action@v1 with: @@ -324,6 +318,35 @@ jobs: target: local tags: ${{ matrix.image }}:${{ matrix.tag }} load: true + if: matrix.type == 'oss' + - name: Build Plus ${{ matrix.image }} Container + uses: docker/build-push-action@v2 + with: + file: ${{ matrix.file }} + context: '.' + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + target: local + tags: ${{ matrix.image }}:${{ matrix.tag }} + load: true + secrets: | + nginx-repo.crt=${{ secrets.NGINX_CRT }} + nginx-repo.key=${{ secrets.NGINX_KEY }} + if: matrix.type == 'plus' + - name: Build AP ${{ matrix.image }} Container + uses: docker/build-push-action@v2 + with: + file: ${{ matrix.file }} + context: '.' + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + target: local + tags: ${{ matrix.image }}:${{ matrix.tag }} + load: true + secrets: | + nginx-repo.crt=${{ secrets.NGINX_AP_CRT }} + nginx-repo.key=${{ secrets.NGINX_AP_KEY }} + if: matrix.type == 'plus-ap' - name: Build Test-Runner Container uses: docker/build-push-action@v2 with: @@ -409,14 +432,6 @@ jobs: with: path: ${{ github.workspace }}/nginx-ingress key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }} - - name: Fetch N+ Certificates - env: - CRT: ${{ secrets.NGINX_CRT }} - KEY: ${{ secrets.NGINX_KEY }} - run: | - echo "${CRT}" | base64 --decode > nginx-repo.crt - echo "${KEY}" | base64 --decode > nginx-repo.key - if: matrix.type == 'plus' - name: Docker Buildx uses: docker/setup-buildx-action@v1 with: @@ -438,6 +453,9 @@ jobs: target: local tags: ${{ matrix.image }}:${{ matrix.tag }} load: true + secrets: | + nginx-repo.crt=${{ secrets.NGINX_CRT }} + nginx-repo.key=${{ secrets.NGINX_KEY }} - name: Deploy Kubernetes id: k8s run: | diff --git a/.gitignore b/.gitignore index dff8e45bfe..6724fa3796 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ nginx-plus-ingress cmd/nginx-ingress/nginx-ingress # NGINX Plus license files +tempdir/ *.crt *.key diff --git a/Makefile b/Makefile index 2390e6e842..3c708e471e 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,16 @@ ifneq ($(BUILD_IN_CONTAINER),1) CGO_ENABLED=0 GO111MODULE=on GOFLAGS='$(GOFLAGS)' GOOS=linux go build -installsuffix cgo -ldflags "-w -X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT}" -o nginx-ingress github.com/nginxinc/kubernetes-ingress/cmd/nginx-ingress endif -container: test verify-codegen verify-crds binary certificate-and-key +prepare-license-secrets: +ifneq (,$(findstring PlusForOpenShift,$(DOCKERFILE))) + mkdir -p tempdir && base64 nginx-repo.crt > tempdir/nginx-repo.crt && base64 nginx-repo.key > tempdir/nginx-repo.key && base64 rhel_license > tempdir/rhel_license +DOCKER_BUILD_OPTIONS += --secret id=nginx-repo.crt,src=tempdir/nginx-repo.crt --secret id=nginx-repo.key,src=tempdir/nginx-repo.key --secret id=rhel_license,src=tempdir/rhel_license +else ifneq (,$(findstring Plus,$(DOCKERFILE))) + mkdir -p tempdir && base64 nginx-repo.crt > tempdir/nginx-repo.crt && base64 nginx-repo.key > tempdir/nginx-repo.key +DOCKER_BUILD_OPTIONS += --secret id=nginx-repo.crt,src=tempdir/nginx-repo.crt --secret id=nginx-repo.key,src=tempdir/nginx-repo.key +endif + +container: test verify-codegen verify-crds binary certificate-and-key prepare-license-secrets ifeq ($(BUILD_IN_CONTAINER),1) docker build $(DOCKER_BUILD_OPTIONS) --build-arg IC_VERSION=$(VERSION)-$(GIT_COMMIT) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg VERSION=$(VERSION) --build-arg GOLANG_CONTAINER=$(GOLANG_CONTAINER) --target container -f $(DOCKERFILEPATH)/$(DOCKERFILE) -t $(PREFIX):$(TAG) . else @@ -71,3 +80,4 @@ endif clean: rm -f nginx-ingress + rm -rf tempdir diff --git a/build/DockerfileForPlus b/build/DockerfileForPlus index 4eb791c777..066e8139f9 100644 --- a/build/DockerfileForPlus +++ b/build/DockerfileForPlus @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.0-experimental ARG GOLANG_CONTAINER=golang:latest FROM debian:buster-slim AS base @@ -8,14 +9,12 @@ ENV NGINX_PLUS_VERSION 22-1~buster ARG IC_VERSION # Download certificate and key from the customer portal (https://cs.nginx.com) -# and copy to the build context -COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/ - -# Make sure the certificate and key have correct permissions -RUN chmod 644 /etc/ssl/nginx/* - -# Install NGINX Plus -RUN set -x \ +# and install NGINX Plus +RUN --mount=type=secret,id=nginx-repo.crt \ + --mount=type=secret,id=nginx-repo.key \ + set -x \ + && mkdir -p /etc/ssl/nginx && base64 --decode /run/secrets/nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && base64 --decode /run/secrets/nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin \ && \ diff --git a/build/DockerfileWithOpentracingForPlus b/build/DockerfileWithOpentracingForPlus index a0394655ee..c552e7b79b 100644 --- a/build/DockerfileWithOpentracingForPlus +++ b/build/DockerfileWithOpentracingForPlus @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.0-experimental ARG GOLANG_CONTAINER=golang:latest FROM debian:buster-slim AS tracer-downloader @@ -18,14 +19,12 @@ ENV NGINX_OPENTRACING_MODULE_VERSION 22+0.9.0-1~buster ARG IC_VERSION # Download certificate and key from the customer portal (https://cs.nginx.com) -# and copy to the build context -COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/ - -# Make sure the certificate and key have correct permissions -RUN chmod 644 /etc/ssl/nginx/* - -# Install NGINX Plus -RUN set -x \ +# and install NGINX Plus +RUN --mount=type=secret,id=nginx-repo.crt \ + --mount=type=secret,id=nginx-repo.key \ + set -x \ + && mkdir -p /etc/ssl/nginx && base64 --decode /run/secrets/nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && base64 --decode /run/secrets/nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin \ && \ diff --git a/build/appprotect/DockerfileWithAppProtectForPlus b/build/appprotect/DockerfileWithAppProtectForPlus index 89191174d1..4318978cc8 100644 --- a/build/appprotect/DockerfileWithAppProtectForPlus +++ b/build/appprotect/DockerfileWithAppProtectForPlus @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.0-experimental ARG GOLANG_CONTAINER=golang:latest FROM debian:stretch-slim as base @@ -15,14 +16,12 @@ ENV NGINX_PLUS_RELEASE R22 ARG IC_VERSION # Download certificate and key from the customer portal (https://cs.nginx.com) -# and copy to the build context -COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/ - -# Make sure the certificate and key have correct permissions -RUN chmod 644 /etc/ssl/nginx/* - -# Install NGINX Plus -RUN set -x \ +# and install NGINX Plus +RUN --mount=type=secret,id=nginx-repo.crt \ + --mount=type=secret,id=nginx-repo.key \ + set -x \ + && mkdir -p /etc/ssl/nginx && base64 --decode /run/secrets/nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && base64 --decode /run/secrets/nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin wget \ && \ diff --git a/build/appprotect/DockerfileWithAppProtectForPlusForOpenShift b/build/appprotect/DockerfileWithAppProtectForPlusForOpenShift index 38dd1ecfee..ed4a95115f 100644 --- a/build/appprotect/DockerfileWithAppProtectForPlusForOpenShift +++ b/build/appprotect/DockerfileWithAppProtectForPlusForOpenShift @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.0-experimental ARG GOLANG_CONTAINER=golang:latest FROM registry.access.redhat.com/ubi7/ubi AS base @@ -19,18 +20,15 @@ ENV NGINX_PLUS_VERSION 22-1.el7.ngx ARG IC_VERSION # Download certificate and key from the customer portal (https://cs.nginx.com) -# and copy to the build context -RUN mkdir /etc/ssl/nginx/ -COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/ - -# Make sure the certificate and key have correct permissions -RUN chmod 644 /etc/ssl/nginx/* - -# Copy RHEL account data -COPY rhel_license /tmp/ - -# Install NGINX Plus -RUN groupadd --system --gid 101 nginx \ +# and install NGINX Plus +RUN --mount=type=secret,id=nginx-repo.crt \ + --mount=type=secret,id=nginx-repo.key \ + --mount=type=secret,id=rhel_license \ + set -x \ + && mkdir -p /etc/ssl/nginx && base64 --decode /run/secrets/nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && base64 --decode /run/secrets/nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + && base64 --decode /run/secrets/rhel_license > /tmp/rhel_license \ + && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && source /tmp/rhel_license \ && yum install -y wget ca-certificates \ diff --git a/build/openshift/DockerfileForPlus b/build/openshift/DockerfileForPlus index d1661792aa..ece754e402 100644 --- a/build/openshift/DockerfileForPlus +++ b/build/openshift/DockerfileForPlus @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.0-experimental ARG GOLANG_CONTAINER=golang:latest FROM registry.access.redhat.com/ubi8/ubi:8.3 AS base @@ -13,15 +14,12 @@ ENV NGINX_PLUS_VERSION 22-1.el8.ngx ARG IC_VERSION # Download certificate and key from the customer portal (https://cs.nginx.com) -# and copy to the build context -RUN mkdir /etc/ssl/nginx/ -COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/ - -# Make sure the certificate and key have correct permissions -RUN chmod 644 /etc/ssl/nginx/* - -# Install NGINX Plus -RUN set -x \ +# and install NGINX Plus +RUN --mount=type=secret,id=nginx-repo.crt \ + --mount=type=secret,id=nginx-repo.key \ + set -x \ + && mkdir -p /etc/ssl/nginx && base64 --decode /run/secrets/nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && base64 --decode /run/secrets/nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && yum install -y wget \