Use buildkit secret flag for NGINX plus images

The new --secret flag for docker build allows the user to pass secret information to be used in the Dockerfile for building docker images in a safe way that will not end up stored in the final image. This commit introduces this change for transferring the cert and key for building the NGINX Plus images.
nginx · Nov 19, 2020 · f2a4328 · f2a4328
1 parent 69f55aa
commit f2a4328
Show file tree

Hide file tree

Showing 9 changed files with 170 additions and 130 deletions.
diff --git a/.github/workflows/edge.yml b/.github/workflows/edge.yml
@@ -158,24 +158,6 @@ jobs:
         with:
           path: ${{ github.workspace }}/nginx-ingress
           key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }}
-      - name: Fetch N+ Certificates
-        env:
-          CRT: ${{ secrets.NGINX_CRT }}
-          KEY: ${{ secrets.NGINX_KEY }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-        if: matrix.type == 'plus'
-      - name: Fetch N+ AppProtect Certificates
-        env:
-          CRT: ${{ secrets.NGINX_AP_CRT }}
-          KEY: ${{ secrets.NGINX_AP_KEY }}
-          RHEL_LICENSE: ${{ secrets.RHEL_LICENSE }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-          echo "${RHEL_LICENSE}" | base64 --decode > rhel_license
-        if: matrix.type == 'plus-ap' || matrix.type == 'plus-ap-openshift'
       - name: Docker Buildx
         uses: docker/setup-buildx-action@v1
         with:
@@ -196,6 +178,34 @@ jobs:
           cache-to: type=local,dest=/tmp/.buildx-cache
           target: ${{ matrix.target }}
           tags: ${{ matrix.image }}:${{ github.sha }}
+        if: matrix.type == 'oss'
+      - name: Build Plus Docker Image ${{ matrix.image }}
+        uses: docker/build-push-action@v2
+        with:
+          file: ${{ matrix.file }}
+          context: ${{ matrix.context }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          target: ${{ matrix.target }}
+          tags: ${{ matrix.image }}:${{ github.sha }}
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_KEY }}
+        if: matrix.type == 'plus'
+      - name: Build AP Docker Image ${{ matrix.image }}
+        uses: docker/build-push-action@v2
+        with:
+          file: ${{ matrix.file }}
+          context: ${{ matrix.context }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          target: ${{ matrix.target }}
+          tags: ${{ matrix.image }}:${{ github.sha }}
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_AP_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_AP_KEY }}
+            rhel_license=${{ secrets.RHEL_LICENSE }}
+        if: matrix.type == 'plus-ap' || matrix.type == 'plus-ap-openshift'
 
   smoke-tests:
     name: Smoke Tests
@@ -269,22 +279,6 @@ jobs:
         with:
           path: ${{ github.workspace }}/nginx-ingress
           key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }}
-      - name: Fetch N+ Certificates
-        env:
-          CRT: ${{ secrets.NGINX_CRT }}
-          KEY: ${{ secrets.NGINX_KEY }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-        if: matrix.type == 'plus'
-      - name: Fetch N+ AppProtect Certificates
-        env:
-          CRT: ${{ secrets.NGINX_AP_CRT }}
-          KEY: ${{ secrets.NGINX_AP_KEY }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-        if: matrix.type == 'plus-ap'
       - name: Docker Buildx
         uses: docker/setup-buildx-action@v1
         with:
@@ -306,6 +300,35 @@ jobs:
           target: local
           tags: ${{ matrix.image }}:${{ matrix.tag }}
           load: true
+        if: matrix.type == 'oss'
+      - name: Build Plus Docker Image ${{ matrix.image }}
+        uses: docker/build-push-action@v2
+        with:
+          file: ${{ matrix.file }}
+          context: '.'
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          target: local
+          tags: ${{ matrix.image }}:${{ matrix.tag }}
+          load: true
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_KEY }}
+        if: matrix.type == 'plus'
+      - name: Build AP Docker Image ${{ matrix.image }}
+        uses: docker/build-push-action@v2
+        with:
+          file: ${{ matrix.file }}
+          context: '.'
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          target: local
+          tags: ${{ matrix.image }}:${{ matrix.tag }}
+          load: true
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_AP_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_AP_KEY }}
+        if: matrix.type == 'plus-ap'
       - name: Build Test-Runner Container
         uses: docker/build-push-action@v2
         with:
@@ -391,14 +414,6 @@ jobs:
         with:
           path: ${{ github.workspace }}/nginx-ingress
           key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }}
-      - name: Fetch N+ Certificates
-        env:
-          CRT: ${{ secrets.NGINX_CRT }}
-          KEY: ${{ secrets.NGINX_KEY }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-        if: matrix.type == 'plus'
       - name: Docker Buildx
         uses: docker/setup-buildx-action@v1
         with:
@@ -420,6 +435,9 @@ jobs:
           target: local
           tags: ${{ matrix.image }}:${{ matrix.tag }}
           load: true
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_KEY }}
       - name: Deploy Kubernetes
         id: k8s
         run: |

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -128,24 +128,6 @@ jobs:
         with:
           path: ${{ github.workspace }}/nginx-ingress
           key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }}
-      - name: Fetch N+ Certificates
-        env:
-          CRT: ${{ secrets.NGINX_CRT }}
-          KEY: ${{ secrets.NGINX_KEY }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-        if: matrix.type == 'plus'
-      - name: Fetch N+ AppProtect Certificates
-        env:
-          CRT: ${{ secrets.NGINX_AP_CRT }}
-          KEY: ${{ secrets.NGINX_AP_KEY }}
-          RHEL_LICENSE: ${{ secrets.RHEL_LICENSE }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-          echo "${RHEL_LICENSE}" | base64 --decode > rhel_license
-        if: matrix.type == 'plus-ap' || matrix.type == 'plus-ap-openshift'
       - name: Docker Buildx
         uses: docker/setup-buildx-action@v1
         with:
@@ -166,6 +148,34 @@ jobs:
           cache-to: type=local,dest=/tmp/.buildx-cache
           target: ${{ matrix.target }}
           tags: ${{ matrix.image }}:${{ github.sha }}
+        if: matrix.type == 'oss'
+      - name: Build Plus Docker Image ${{ matrix.image }}
+        uses: docker/build-push-action@v2
+        with:
+          file: ${{ matrix.file }}
+          context: ${{ matrix.context }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          target: ${{ matrix.target }}
+          tags: ${{ matrix.image }}:${{ github.sha }}
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_KEY }}
+        if: matrix.type == 'plus'
+      - name: Build AP Docker Image ${{ matrix.image }}
+        uses: docker/build-push-action@v2
+        with:
+          file: ${{ matrix.file }}
+          context: ${{ matrix.context }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          target: ${{ matrix.target }}
+          tags: ${{ matrix.image }}:${{ github.sha }}
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_AP_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_AP_KEY }}
+            rhel_license=${{ secrets.RHEL_LICENSE }}
+        if: matrix.type == 'plus-ap' || matrix.type == 'plus-ap-openshift'
 
   smoke-tests:
     name: Smoke Tests
@@ -287,22 +297,6 @@ jobs:
         with:
           path: ${{ github.workspace }}/nginx-ingress
           key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }}
-      - name: Fetch N+ Certificates
-        env:
-          CRT: ${{ secrets.NGINX_CRT }}
-          KEY: ${{ secrets.NGINX_KEY }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-        if: matrix.type == 'plus'
-      - name: Fetch N+ AppProtect Certificates
-        env:
-          CRT: ${{ secrets.NGINX_AP_CRT }}
-          KEY: ${{ secrets.NGINX_AP_KEY }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-        if: matrix.type == 'plus-ap'
       - name: Docker Buildx
         uses: docker/setup-buildx-action@v1
         with:
@@ -324,6 +318,35 @@ jobs:
           target: local
           tags: ${{ matrix.image }}:${{ matrix.tag }}
           load: true
+        if: matrix.type == 'oss'
+      - name: Build Plus ${{ matrix.image }} Container
+        uses: docker/build-push-action@v2
+        with:
+          file: ${{ matrix.file }}
+          context: '.'
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          target: local
+          tags: ${{ matrix.image }}:${{ matrix.tag }}
+          load: true
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_KEY }}
+        if: matrix.type == 'plus'
+      - name: Build AP ${{ matrix.image }} Container
+        uses: docker/build-push-action@v2
+        with:
+          file: ${{ matrix.file }}
+          context: '.'
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          target: local
+          tags: ${{ matrix.image }}:${{ matrix.tag }}
+          load: true
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_AP_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_AP_KEY }}
+        if: matrix.type == 'plus-ap'
       - name: Build Test-Runner Container
         uses: docker/build-push-action@v2
         with:
@@ -409,14 +432,6 @@ jobs:
         with:
           path: ${{ github.workspace }}/nginx-ingress
           key: nginx-ingress-${{ github.run_id }}-${{ github.run_number }}
-      - name: Fetch N+ Certificates
-        env:
-          CRT: ${{ secrets.NGINX_CRT }}
-          KEY: ${{ secrets.NGINX_KEY }}
-        run: |
-          echo "${CRT}" | base64 --decode > nginx-repo.crt
-          echo "${KEY}" | base64 --decode > nginx-repo.key
-        if: matrix.type == 'plus'
       - name: Docker Buildx
         uses: docker/setup-buildx-action@v1
         with:
@@ -438,6 +453,9 @@ jobs:
           target: local
           tags: ${{ matrix.image }}:${{ matrix.tag }}
           load: true
+          secrets: |
+            nginx-repo.crt=${{ secrets.NGINX_CRT }}
+            nginx-repo.key=${{ secrets.NGINX_KEY }}
       - name: Deploy Kubernetes
         id: k8s
         run: |

diff --git a/.gitignore b/.gitignore
@@ -30,6 +30,7 @@ nginx-plus-ingress
 cmd/nginx-ingress/nginx-ingress
 
 # NGINX Plus license files
+tempdir/
 *.crt
 *.key
 

diff --git a/Makefile b/Makefile
@@ -55,7 +55,16 @@ ifneq ($(BUILD_IN_CONTAINER),1)
 	CGO_ENABLED=0 GO111MODULE=on GOFLAGS='$(GOFLAGS)' GOOS=linux go build -installsuffix cgo -ldflags "-w -X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT}" -o nginx-ingress github.com/nginxinc/kubernetes-ingress/cmd/nginx-ingress
 endif
 
-container: test verify-codegen verify-crds binary certificate-and-key
+prepare-license-secrets:
+ifneq (,$(findstring PlusForOpenShift,$(DOCKERFILE)))
+	mkdir -p tempdir && base64 nginx-repo.crt > tempdir/nginx-repo.crt && base64 nginx-repo.key > tempdir/nginx-repo.key && base64 rhel_license > tempdir/rhel_license
+DOCKER_BUILD_OPTIONS += --secret id=nginx-repo.crt,src=tempdir/nginx-repo.crt --secret id=nginx-repo.key,src=tempdir/nginx-repo.key --secret id=rhel_license,src=tempdir/rhel_license
+else ifneq (,$(findstring Plus,$(DOCKERFILE)))
+	mkdir -p tempdir && base64 nginx-repo.crt > tempdir/nginx-repo.crt && base64 nginx-repo.key > tempdir/nginx-repo.key
+DOCKER_BUILD_OPTIONS += --secret id=nginx-repo.crt,src=tempdir/nginx-repo.crt --secret id=nginx-repo.key,src=tempdir/nginx-repo.key 
+endif
+
+container: test verify-codegen verify-crds binary certificate-and-key prepare-license-secrets
 ifeq ($(BUILD_IN_CONTAINER),1)
 	docker build $(DOCKER_BUILD_OPTIONS) --build-arg IC_VERSION=$(VERSION)-$(GIT_COMMIT) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg VERSION=$(VERSION) --build-arg GOLANG_CONTAINER=$(GOLANG_CONTAINER) --target container -f $(DOCKERFILEPATH)/$(DOCKERFILE) -t $(PREFIX):$(TAG) .
 else
@@ -71,3 +80,4 @@ endif
 
 clean:
 	rm -f nginx-ingress
+	rm -rf tempdir
diff --git a/build/DockerfileForPlus b/build/DockerfileForPlus
@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1.0-experimental
 ARG GOLANG_CONTAINER=golang:latest
 
 FROM debian:buster-slim AS base
@@ -8,14 +9,12 @@ ENV NGINX_PLUS_VERSION 22-1~buster
 ARG IC_VERSION
 
 # Download certificate and key from the customer portal (https://cs.nginx.com)
-# and copy to the build context
-COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/
-
-# Make sure the certificate and key have correct permissions
-RUN chmod 644 /etc/ssl/nginx/*
-
-# Install NGINX Plus
-RUN set -x \
+# and install NGINX Plus
+RUN --mount=type=secret,id=nginx-repo.crt \
+	--mount=type=secret,id=nginx-repo.key \
+	set -x \
+	&& mkdir -p /etc/ssl/nginx && base64 --decode /run/secrets/nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt  \
+	&& base64 --decode /run/secrets/nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \
 	&& apt-get update \
 	&& apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin \
 	&& \

diff --git a/build/DockerfileWithOpentracingForPlus b/build/DockerfileWithOpentracingForPlus
@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1.0-experimental
 ARG GOLANG_CONTAINER=golang:latest
 
 FROM debian:buster-slim AS tracer-downloader
@@ -18,14 +19,12 @@ ENV NGINX_OPENTRACING_MODULE_VERSION 22+0.9.0-1~buster
 ARG IC_VERSION
 
 # Download certificate and key from the customer portal (https://cs.nginx.com)
-# and copy to the build context
-COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/
-
-# Make sure the certificate and key have correct permissions
-RUN chmod 644 /etc/ssl/nginx/*
-
-# Install NGINX Plus
-RUN set -x \
+# and install NGINX Plus
+RUN --mount=type=secret,id=nginx-repo.crt \
+	--mount=type=secret,id=nginx-repo.key \
+	set -x \
+	&& mkdir -p /etc/ssl/nginx && base64 --decode /run/secrets/nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt  \
+	&& base64 --decode /run/secrets/nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \
 	&& apt-get update \
 	&& apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin \
 	&& \

diff --git a/build/appprotect/DockerfileWithAppProtectForPlus b/build/appprotect/DockerfileWithAppProtectForPlus
@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1.0-experimental
 ARG GOLANG_CONTAINER=golang:latest
 
 FROM debian:stretch-slim as base
@@ -15,14 +16,12 @@ ENV NGINX_PLUS_RELEASE R22
 ARG IC_VERSION
 
 # Download certificate and key from the customer portal (https://cs.nginx.com)
-# and copy to the build context
-COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/
-
-# Make sure the certificate and key have correct permissions
-RUN chmod 644 /etc/ssl/nginx/*
-
-# Install NGINX Plus
-RUN set -x \
+# and install NGINX Plus
+RUN --mount=type=secret,id=nginx-repo.crt \
+	--mount=type=secret,id=nginx-repo.key \
+	set -x \
+	&& mkdir -p /etc/ssl/nginx && base64 --decode /run/secrets/nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt  \
+	&& base64 --decode /run/secrets/nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \
 	&& apt-get update \
 	&& apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin wget \
 	&& \
-Original file line number
+Diff line change
@@ Expand Up / @@ -30,6 +30,7 @@ nginx-plus-ingress @@
     cmd/nginx-ingress/nginx-ingress
     # NGINX Plus license files
+    tempdir/
     *.crt
     *.key
@@ Expand Down @@