always use buildkit to build your docker images, even in concourse :)

springernature · Sep 5, 2023 · 961d8db · 961d8db
1 parent 909e879
commit 961d8db
Show file tree

Hide file tree

Showing 15 changed files with 613 additions and 414 deletions.
diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml
@@ -436,13 +436,15 @@ jobs:
       image_resource:
         name: ""
         source:
-          repository: concourse/oci-build-task
+          password: ((halfpipe-gcr.private_key))
+          repository: eu.gcr.io/halfpipe-io/halfpipe-buildx
+          tag: latest
+          username: _json_key
         type: registry-image
       inputs:
       - name: git
+      - name: tagList
       - name: docker_build
-      outputs:
-      - name: image
       params:
         BUILD_ARG_A: a
         BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password))
@@ -458,9 +460,8 @@ jobs:
         args:
         - -c
         - |-
-          mkdir ~/.docker
           echo $DOCKER_CONFIG_JSON > ~/.docker/config.json
-          build
+          docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT
         path: /bin/sh
     privileged: true
     task: build
@@ -473,25 +474,53 @@ jobs:
         type: docker-image
       inputs:
       - name: git
-      - name: image
       - name: docker_build
+      params:
+        DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config))
       platform: linux
       run:
         args:
         - -c
         - |-
           [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true
           [ -f .trivyignore ] && cat .trivyignore; echo || true
-          trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true
+          trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true
         dir: docker_build/e2e/concourse/artifacts
         path: /bin/sh
     task: trivy
     timeout: 1h
-  - no_get: true
-    params:
-      additional_tags: tagList/tagList
-      image: image/image.tar
-    put: halfpipe-fly
+  - config:
+      image_resource:
+        name: ""
+        source:
+          password: ((halfpipe-gcr.private_key))
+          repository: eu.gcr.io/halfpipe-io/halfpipe-buildx
+          tag: latest
+          username: _json_key
+        type: registry-image
+      inputs:
+      - name: git
+      - name: tagList
+      params:
+        BUILD_ARG_A: a
+        BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password))
+        BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url))
+        BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username))
+        BUILD_ARG_B: b
+        BUILD_ARG_RUNNING_IN_CI: "true"
+        CONTEXT: docker_build/e2e/concourse/artifacts
+        DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config))
+        DOCKERFILE: docker_build/e2e/concourse/artifacts/Dockerfile
+      platform: linux
+      run:
+        args:
+        - -c
+        - |-
+          echo $DOCKER_CONFIG_JSON > ~/.docker/config.json
+          for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done
+        path: /bin/sh
+    privileged: true
+    task: publish-final-image
     timeout: 1h
   serial: true
 - build_log_retention:
@@ -747,13 +776,6 @@ resources:
     folder: halfpipe-team/halfpipe-e2e-artifacts
     json_key: ((halfpipe-artifacts.private_key))
   type: gcp-resource
-- check_every: 24h0m0s
-  name: halfpipe-fly
-  source:
-    password: verysecret
-    repository: springerplatformengineering/halfpipe-fly
-    username: rob
-  type: registry-image
 - check_every: 24h0m0s
   name: cf-snpaas-pe-staging
   source:

diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml
@@ -87,12 +87,14 @@ jobs:
       image_resource:
         name: ""
         source:
-          repository: concourse/oci-build-task
+          password: ((halfpipe-gcr.private_key))
+          repository: eu.gcr.io/halfpipe-io/halfpipe-buildx
+          tag: latest
+          username: _json_key
         type: registry-image
       inputs:
       - name: git
-      outputs:
-      - name: image
+      - name: tagList
       params:
         BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password))
         BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url))
@@ -106,9 +108,8 @@ jobs:
         args:
         - -c
         - |-
-          mkdir ~/.docker
           echo $DOCKER_CONFIG_JSON > ~/.docker/config.json
-          build
+          docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --push --provenance=false $CONTEXT
         path: /bin/sh
     privileged: true
     task: build
@@ -121,24 +122,50 @@ jobs:
         type: docker-image
       inputs:
       - name: git
-      - name: image
+      params:
+        DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config))
       platform: linux
       run:
         args:
         - -c
         - |-
           [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true
           [ -f .trivyignore ] && cat .trivyignore; echo || true
-          trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true
+          trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat ../../../.git/ref) || true
         dir: git/e2e/concourse/deploy-katee
         path: /bin/sh
     task: trivy
     timeout: 1h
-  - no_get: true
-    params:
-      additional_tags: tagList/tagList
-      image: image/image.tar
-    put: someimage
+  - config:
+      image_resource:
+        name: ""
+        source:
+          password: ((halfpipe-gcr.private_key))
+          repository: eu.gcr.io/halfpipe-io/halfpipe-buildx
+          tag: latest
+          username: _json_key
+        type: registry-image
+      inputs:
+      - name: git
+      - name: tagList
+      params:
+        BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password))
+        BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url))
+        BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username))
+        BUILD_ARG_RUNNING_IN_CI: "true"
+        CONTEXT: git/e2e/concourse/deploy-katee
+        DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config))
+        DOCKERFILE: git/e2e/concourse/deploy-katee/Dockerfile
+      platform: linux
+      run:
+        args:
+        - -c
+        - |-
+          echo $DOCKER_CONFIG_JSON > ~/.docker/config.json
+          for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --tag eu.gcr.io/halfpipe-io/halfpipe-team/someImage:$tag; done
+        path: /bin/sh
+    privileged: true
+    task: publish-final-image
     timeout: 1h
   serial: true
 - build_log_retention:
@@ -379,10 +406,3 @@ resources:
     json_key: ((halfpipe-semver.private_key))
     key: halfpipe-team-pipeline-name
   type: semver
-- check_every: 24h0m0s
-  name: someimage
-  source:
-    password: ((halfpipe-gcr.private_key))
-    repository: eu.gcr.io/halfpipe-io/halfpipe-team/someImage
-    username: _json_key
-  type: registry-image
diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml
@@ -34,12 +34,14 @@ jobs:
       image_resource:
         name: ""
         source:
-          repository: concourse/oci-build-task
+          password: ((halfpipe-gcr.private_key))
+          repository: eu.gcr.io/halfpipe-io/halfpipe-buildx
+          tag: latest
+          username: _json_key
         type: registry-image
       inputs:
       - name: git
-      outputs:
-      - name: image
+      - name: tagList
       params:
         BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password))
         BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url))
@@ -53,9 +55,8 @@ jobs:
         args:
         - -c
         - |-
-          mkdir ~/.docker
           echo $DOCKER_CONFIG_JSON > ~/.docker/config.json
-          build
+          docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT
         path: /bin/sh
     privileged: true
     task: build
@@ -68,24 +69,50 @@ jobs:
         type: docker-image
       inputs:
       - name: git
-      - name: image
+      params:
+        DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config))
       platform: linux
       run:
         args:
         - -c
         - |-
           [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true
           [ -f .trivyignore ] && cat .trivyignore; echo || true
-          trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true
+          trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true
         dir: git/e2e/concourse/docker-push-paths
         path: /bin/sh
     task: trivy
     timeout: 1h
-  - no_get: true
-    params:
-      additional_tags: tagList/tagList
-      image: image/image.tar
-    put: halfpipe-fly
+  - config:
+      image_resource:
+        name: ""
+        source:
+          password: ((halfpipe-gcr.private_key))
+          repository: eu.gcr.io/halfpipe-io/halfpipe-buildx
+          tag: latest
+          username: _json_key
+        type: registry-image
+      inputs:
+      - name: git
+      - name: tagList
+      params:
+        BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password))
+        BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url))
+        BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username))
+        BUILD_ARG_RUNNING_IN_CI: "true"
+        CONTEXT: git/e2e/concourse/docker-push-paths/some/build/dir
+        DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config))
+        DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile
+      platform: linux
+      run:
+        args:
+        - -c
+        - |-
+          echo $DOCKER_CONFIG_JSON > ~/.docker/config.json
+          for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done
+        path: /bin/sh
+    privileged: true
+    task: publish-final-image
     timeout: 1h
   serial: true
 - build_log_retention:
@@ -124,12 +151,14 @@ jobs:
       image_resource:
         name: ""
         source:
-          repository: concourse/oci-build-task
+          password: ((halfpipe-gcr.private_key))
+          repository: eu.gcr.io/halfpipe-io/halfpipe-buildx
+          tag: latest
+          username: _json_key
         type: registry-image
       inputs:
       - name: git
-      outputs:
-      - name: image
+      - name: tagList
       params:
         BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password))
         BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url))
@@ -143,9 +172,8 @@ jobs:
         args:
         - -c
         - |-
-          mkdir ~/.docker
           echo $DOCKER_CONFIG_JSON > ~/.docker/config.json
-          build
+          docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --push --provenance=false $CONTEXT
         path: /bin/sh
     privileged: true
     task: build
@@ -158,24 +186,50 @@ jobs:
         type: docker-image
       inputs:
       - name: git
-      - name: image
+      params:
+        DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config))
       platform: linux
       run:
         args:
         - -c
         - |-
           [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true
           [ -f .trivyignore ] && cat .trivyignore; echo || true
-          trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true
+          trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat ../../../.git/ref) || true
         dir: git/e2e/concourse/docker-push-paths
         path: /bin/sh
     task: trivy
     timeout: 1h
-  - no_get: true
-    params:
-      additional_tags: tagList/tagList
-      image: image/image.tar
-    put: halfpipe
+  - config:
+      image_resource:
+        name: ""
+        source:
+          password: ((halfpipe-gcr.private_key))
+          repository: eu.gcr.io/halfpipe-io/halfpipe-buildx
+          tag: latest
+          username: _json_key
+        type: registry-image
+      inputs:
+      - name: git
+      - name: tagList
+      params:
+        BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password))
+        BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url))
+        BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username))
+        BUILD_ARG_RUNNING_IN_CI: "true"
+        CONTEXT: git/e2e/concourse/docker-push-paths
+        DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config))
+        DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile
+      platform: linux
+      run:
+        args:
+        - -c
+        - |-
+          echo $DOCKER_CONFIG_JSON > ~/.docker/config.json
+          for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe:$tag; done
+        path: /bin/sh
+    privileged: true
+    task: publish-final-image
     timeout: 1h
   serial: true
 resources:
@@ -188,17 +242,3 @@ resources:
     private_key: ((halfpipe-github.private_key))
     uri: [email protected]:springernature/halfpipe.git
   type: git
-- check_every: 24h0m0s
-  name: halfpipe-fly
-  source:
-    password: verysecret
-    repository: springerplatformengineering/halfpipe-fly
-    username: rob
-  type: registry-image
-- check_every: 24h0m0s
-  name: halfpipe
-  source:
-    password: verysecret
-    repository: springerplatformengineering/halfpipe
-    username: rob
-  type: registry-image