From 961d8dbac04d6d2c0b9b98c1f6b8a162752b0820 Mon Sep 17 00:00:00 2001 From: Michiel Stigter Date: Tue, 5 Sep 2023 15:08:47 +0200 Subject: [PATCH 01/40] always use buildkit to build your docker images, even in concourse :) --- e2e/concourse/artifacts/pipelineExpected.yml | 60 ++++-- .../deploy-katee/pipelineExpected.yml | 58 ++++-- .../docker-push-paths/pipelineExpected.yml | 116 +++++++---- .../pipelineExpected.yml | 60 ++++-- .../pipelineExpected.yml | 60 ++++-- .../pipelineExpected.yml | 120 +++++++---- .../pipelineExpected.yml | 120 +++++++---- .../docker-push/pipelineExpected.yml | 128 ++++++++---- .../timer-trigger/pipelineExpected.yml | 60 ++++-- linters/docker-push.go | 7 + linters/docker-push_test.go | 6 +- linters/errors.go | 10 +- renderers/concourse/docker_push.go | 191 +++++++----------- renderers/concourse/pipeline.go | 18 -- renderers/concourse/pipeline_resources.go | 13 -- 15 files changed, 613 insertions(+), 414 deletions(-) diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index dea44ff7..a10e6ef0 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -436,13 +436,15 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git + - name: tagList - name: docker_build - outputs: - - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -458,9 +460,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -473,8 +474,9 @@ jobs: type: docker-image inputs: - name: git - - name: image - name: docker_build + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -482,16 +484,43 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: docker_build/e2e/concourse/artifacts path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: docker_build/e2e/concourse/artifacts + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: docker_build/e2e/concourse/artifacts/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -747,13 +776,6 @@ resources: folder: halfpipe-team/halfpipe-e2e-artifacts json_key: ((halfpipe-artifacts.private_key)) type: gcp-resource -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image - check_every: 24h0m0s name: cf-snpaas-pe-staging source: diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml index b4d6bb32..641329d2 100644 --- a/e2e/concourse/deploy-katee/pipelineExpected.yml +++ b/e2e/concourse/deploy-katee/pipelineExpected.yml @@ -87,12 +87,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) @@ -106,9 +108,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -121,7 +122,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -129,16 +131,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/deploy-katee path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: someimage + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/deploy-katee + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/deploy-katee/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --tag eu.gcr.io/halfpipe-io/halfpipe-team/someImage:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -379,10 +406,3 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-pipeline-name type: semver -- check_every: 24h0m0s - name: someimage - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-team/someImage - username: _json_key - type: registry-image diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml index bcd4794f..db911bb0 100644 --- a/e2e/concourse/docker-push-paths/pipelineExpected.yml +++ b/e2e/concourse/docker-push-paths/pipelineExpected.yml @@ -34,12 +34,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) @@ -53,9 +55,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -68,7 +69,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -76,16 +78,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-paths path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-paths/some/build/dir + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -124,12 +151,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) @@ -143,9 +172,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -158,7 +186,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -166,16 +195,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-paths path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-paths + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resources: @@ -188,17 +242,3 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image -- check_every: 24h0m0s - name: halfpipe - source: - password: verysecret - repository: springerplatformengineering/halfpipe - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml index 202fc7d0..140308d7 100644 --- a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml @@ -41,12 +41,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -62,9 +64,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -77,7 +78,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -85,16 +87,43 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-with-docker-trigger + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resources: @@ -111,10 +140,3 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml index 281e223a..ade3a00e 100644 --- a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml @@ -39,12 +39,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -60,9 +62,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -75,7 +76,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -83,16 +85,43 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-with-pipeline-trigger + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resource_types: @@ -113,13 +142,6 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image - name: halfpipe-example-docker-push.docker-push source: concourse_url: ((concourse.url)) diff --git a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml index c9d96dbc..f8a89d8d 100644 --- a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml @@ -235,13 +235,15 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git + - name: tagList - name: docker_build - outputs: - - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -257,9 +259,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -272,8 +273,9 @@ jobs: type: docker-image inputs: - name: git - - name: image - name: docker_build + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -281,16 +283,43 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat ../../../.git/ref) || true dir: docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image1 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --tag springerplatformengineering/image1:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -384,13 +413,15 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git + - name: tagList - name: docker_build - outputs: - - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -406,9 +437,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -421,8 +451,9 @@ jobs: type: docker-image inputs: - name: git - - name: image - name: docker_build + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -430,16 +461,43 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat ../../../.git/ref) || true dir: docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image2 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --tag springerplatformengineering/image2:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resource_types: @@ -476,17 +534,3 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-docker-push-with-update-pipeline type: semver -- check_every: 24h0m0s - name: image1 - source: - password: verysecret - repository: springerplatformengineering/image1 - username: rob - type: registry-image -- check_every: 24h0m0s - name: image2 - source: - password: verysecret - repository: springerplatformengineering/image2 - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml index b0f327bd..f2138490 100644 --- a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml @@ -87,12 +87,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -108,9 +110,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -123,7 +124,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -131,16 +133,43 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image1 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --tag springerplatformengineering/image1:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -186,12 +215,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -207,9 +238,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -222,7 +252,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -230,16 +261,43 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image2 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --tag springerplatformengineering/image2:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resources: @@ -260,17 +318,3 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-docker-push-with-update-pipeline type: semver -- check_every: 24h0m0s - name: image1 - source: - password: verysecret - repository: springerplatformengineering/image1 - username: rob - type: registry-image -- check_every: 24h0m0s - name: image2 - source: - password: verysecret - repository: springerplatformengineering/image2 - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index c6c2f4ef..460f05fd 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -34,12 +34,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -55,9 +57,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -70,7 +71,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -78,16 +80,43 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly.thisismy-tag + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) thisIsMy_Tag; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe_fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -128,12 +157,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) @@ -147,9 +178,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -163,7 +193,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -171,17 +202,42 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h - attempts: 2 - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly.thisismy-tag2 + config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) thisIsMy_Tag2; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe_fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -230,8 +286,6 @@ jobs: inputs: - name: git - name: tagList - outputs: - - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) @@ -260,7 +314,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -268,7 +323,8 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) || true + trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true + dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h @@ -285,8 +341,6 @@ jobs: inputs: - name: git - name: tagList - outputs: - - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) @@ -317,17 +371,3 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly.thisismy-tag - source: - password: verysecret - repository: springerplatformengineering/halfpipe_fly:thisIsMy_Tag - username: rob - type: registry-image -- check_every: 24h0m0s - name: halfpipe-fly.thisismy-tag2 - source: - password: verysecret - repository: springerplatformengineering/halfpipe_fly:thisIsMy_Tag2 - username: rob - type: registry-image diff --git a/e2e/concourse/timer-trigger/pipelineExpected.yml b/e2e/concourse/timer-trigger/pipelineExpected.yml index 167c3e62..94870239 100644 --- a/e2e/concourse/timer-trigger/pipelineExpected.yml +++ b/e2e/concourse/timer-trigger/pipelineExpected.yml @@ -39,12 +39,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -60,9 +62,8 @@ jobs: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -75,7 +76,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -83,16 +85,43 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/timer-trigger path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/timer-trigger + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/timer-trigger/Dockerfile + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resource_types: @@ -119,10 +148,3 @@ resources: fire_immediately: true location: UTC type: halfpipe-cron-resource -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image diff --git a/linters/docker-push.go b/linters/docker-push.go index 8b188df8..c9a45361 100644 --- a/linters/docker-push.go +++ b/linters/docker-push.go @@ -2,6 +2,7 @@ package linters import ( "fmt" + "golang.org/x/exp/slices" "os" "regexp" "strings" @@ -61,5 +62,11 @@ func LintDockerPushTask(docker manifest.DockerPush, manifest manifest.Manifest, errs = append(errs, ErrDockerPushTag.AsWarning()) } + for _, platform := range docker.Platforms { + if !slices.Contains([]string{"linux/amd64", "linux/arm64"}, platform) { + errs = append(errs, ErrDockerPlatformUnknown) + } + } + return errs } diff --git a/linters/docker-push_test.go b/linters/docker-push_test.go index 30b298ab..032c610b 100644 --- a/linters/docker-push_test.go +++ b/linters/docker-push_test.go @@ -355,7 +355,7 @@ func TestMultiplePlatforms(t *testing.T) { assert.Empty(t, errors) }) - t.Run("only linux/amd64 for concourse is fine", func(t *testing.T) { + t.Run("errors when unknown platform in docker push", func(t *testing.T) { fs := afero.Afero{Fs: afero.NewMemMapFs()} fs.WriteFile("Dockerfile", []byte("FROM ubuntu"), 0777) @@ -364,12 +364,12 @@ func TestMultiplePlatforms(t *testing.T) { Username: "asd", Password: "asdf", DockerfilePath: "Dockerfile", - Platforms: []string{"linux/amd64"}, + Platforms: []string{"linux/ad64"}, } m := manifest.Manifest{Platform: "actions"} errors := LintDockerPushTask(task, m, fs) - assert.Empty(t, errors) + assertContainsError(t, errors, ErrDockerPlatformUnknown) }) } diff --git a/linters/errors.go b/linters/errors.go index 8092d33e..0493c04b 100644 --- a/linters/errors.go +++ b/linters/errors.go @@ -33,10 +33,12 @@ var ( ErrCFFromArtifact = newError("this file must be saved as an artifact in a previous task") ErrCFPrePromoteArtifact = newError("cannot have pre promote tasks with CF manifest restored from artifact") - ErrUnsupportedRegistry = newError("image must be from halfpipe registry. Please see ") - ErrDockerPushTag = newError("the field 'tag' is no longer used and is safe to delete") - ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") - ErrMultipleTriggers = newError("cannot have multiple triggers of this type") + ErrUnsupportedRegistry = newError("image must be from halfpipe registry. Please see ") + ErrDockerPushTag = newError("the field 'tag' is no longer used and is safe to delete") + + ErrDockerPlatformUnknown = newError("only linux/amd64 and/or linux/arm64 are supported") + ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") + ErrMultipleTriggers = newError("cannot have multiple triggers of this type") ErrVelaVariableMissing = newError("vela manifest variable is not specified in halfpipe manifest") ErrVelaNamespace = newError("vela namespace must start with 'katee-'") diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index f15ef681..54e522f0 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -16,16 +16,10 @@ var tagListFile = path.Join(tagList_Dir, "tagList") func (c Concourse) dockerPushJob(task manifest.DockerPush, basePath string, man manifest.Manifest) atc.JobConfig { var steps []atc.Step - resourceName := manifest.DockerTrigger{Image: task.Image}.GetTriggerName() - - fullBasePath := path.Join(gitDir, basePath) - if task.RestoreArtifacts { - fullBasePath = path.Join(dockerBuildTmpDir, basePath) - } steps = append(steps, restoreArtifacts(task)...) steps = append(steps, createTagList(task, man.FeatureToggles.UpdatePipeline())...) - steps = append(steps, buildAndPush(task, resourceName, fullBasePath)...) + steps = append(steps, buildAndPush(task, basePath)...) return atc.JobConfig{ Name: task.GetName(), @@ -104,16 +98,12 @@ func createTagList(task manifest.DockerPush, updatePipeline bool) []atc.Step { return append([]atc.Step{}, stepWithAttemptsAndTimeout(createTagList, task.GetAttempts(), task.Timeout)) } -func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { +func trivyTask(task manifest.DockerPush, fullBasePath string, basePath string) atc.StepConfig { var imageFile string var gitRef string - if !multiPlatform(task) { - imageFile = fmt.Sprintf("--input %s", path.Join(relativePathToRepoRoot(gitDir, fullBasePath), "image/image.tar")) - } else { - imageFile = shared.CachePath(task, "") - gitRef = fmt.Sprintf(":$(cat %s)", path.Join(gitDir, ".git", "ref")) - fullBasePath = "" - } + imageFile = shared.CachePath(task, "") + + gitRef = fmt.Sprintf(":$(cat %s)", pathToGitRef(gitDir, basePath)) //exitCode := 1 //if task.IgnoreVulnerabilities { @@ -142,9 +132,11 @@ func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { }, "\n")}, Dir: fullBasePath, }, + Params: atc.TaskEnv{ + "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", + }, Inputs: []atc.TaskInputConfig{ {Name: gitDir}, - {Name: "image"}, }, }, } @@ -156,11 +148,16 @@ func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { return step } -func buildAndPush(task manifest.DockerPush, resourceName string, fullBasePath string) []atc.Step { +func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { var steps []atc.Step image, tag := shared.SplitTag(task.Image) dockerImageWithCachePath := shared.CachePath(task, "") + fullBasePath := path.Join(gitDir, basePath) + if task.RestoreArtifacts { + fullBasePath = path.Join(dockerBuildTmpDir, basePath) + } + params := atc.TaskEnv{ "CONTEXT": path.Join(fullBasePath, task.BuildPath), "DOCKERFILE": path.Join(fullBasePath, task.DockerfilePath), @@ -173,68 +170,35 @@ func buildAndPush(task manifest.DockerPush, resourceName string, fullBasePath st var buildStep *atc.TaskStep - if !multiPlatform(task) { - buildStep = &atc.TaskStep{ - Name: "build", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": "concourse/oci-build-task", - }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{ - "-c", - fmt.Sprintf("%s\n%s\n%s", "mkdir ~/.docker", "echo $DOCKER_CONFIG_JSON > ~/.docker/config.json", "build"), - }, - }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - }, - Outputs: []atc.TaskOutputConfig{ - {Name: "image"}, - }, - }, - } - } else { - gitRef := fmt.Sprintf("$(cat %s)", path.Join(gitDir, ".git", "ref")) + platforms := strings.Join(task.Platforms, ",") - buildStep = &atc.TaskStep{ - Name: "build", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": config.DockerRegistry + "halfpipe-buildx", - "tag": "latest", - "password": "((halfpipe-gcr.private_key))", - "username": "_json_key", - }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{"-c", strings.Join([]string{ - `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - fmt.Sprintf(`docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t %s:%s --push --provenance=false $CONTEXT`, dockerImageWithCachePath, gitRef)}, "\n"), - }, - }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - {Name: tagList_Dir}, + buildStep = &atc.TaskStep{ + Name: "build", + Privileged: true, + Config: &atc.TaskConfig{ + Platform: "linux", + ImageResource: &atc.ImageResource{ + Type: "registry-image", + Source: atc.Source{ + "repository": config.DockerRegistry + "halfpipe-buildx", + "tag": "latest", + "password": "((halfpipe-gcr.private_key))", + "username": "_json_key", }, - Outputs: []atc.TaskOutputConfig{ - {Name: "image"}, + }, + Params: params, + Run: atc.TaskRunConfig{ + Path: "/bin/sh", + Args: []string{"-c", strings.Join([]string{ + `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, + fmt.Sprintf(`docker buildx build -f $DOCKERFILE --platform %s -t %s:$(cat git/.git/ref) --push --provenance=false $CONTEXT`, platforms, dockerImageWithCachePath)}, "\n"), }, }, - } + Inputs: []atc.TaskInputConfig{ + {Name: gitDir}, + {Name: tagList_Dir}, + }, + }, } if task.ReadsFromArtifacts() { @@ -242,59 +206,40 @@ func buildAndPush(task manifest.DockerPush, resourceName string, fullBasePath st } steps = append(steps, stepWithAttemptsAndTimeout(buildStep, task.GetAttempts(), task.GetTimeout())) - steps = append(steps, stepWithAttemptsAndTimeout(trivyTask(task, fullBasePath), task.GetAttempts(), task.GetTimeout())) + steps = append(steps, stepWithAttemptsAndTimeout(trivyTask(task, fullBasePath, basePath), task.GetAttempts(), task.GetTimeout())) - if !multiPlatform(task) { - putStep := &atc.PutStep{ - Name: resourceName, - Params: atc.Params{ - "image": "image/image.tar", - "additional_tags": tagListFile, - }, - NoGet: true, - } - steps = append(steps, stepWithAttemptsAndTimeout(putStep, task.GetAttempts(), task.GetTimeout())) - } else { - gitRef := fmt.Sprintf("$(cat %s)", path.Join(gitDir, ".git", "ref")) - publishCommand := fmt.Sprintf(`for tag in $(cat %s) %s; do docker buildx imagetools create %s:%s --tag %s:$tag; done`, tagListFile, tag, dockerImageWithCachePath, gitRef, image) + publishCommand := fmt.Sprintf(`for tag in $(cat %s) %s; do docker buildx imagetools create %s:$(cat git/.git/ref) --tag %s:$tag; done`, tagListFile, tag, dockerImageWithCachePath, image) - pushStep := &atc.TaskStep{ - Name: "publish-final-image", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": config.DockerRegistry + "halfpipe-buildx", - "tag": "latest", - "password": "((halfpipe-gcr.private_key))", - "username": "_json_key", - }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{"-c", strings.Join([]string{ - `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - publishCommand, - }, "\n"), - }, - }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - {Name: tagList_Dir}, + pushStep := &atc.TaskStep{ + Name: "publish-final-image", + Privileged: true, + Config: &atc.TaskConfig{ + Platform: "linux", + ImageResource: &atc.ImageResource{ + Type: "registry-image", + Source: atc.Source{ + "repository": config.DockerRegistry + "halfpipe-buildx", + "tag": "latest", + "password": "((halfpipe-gcr.private_key))", + "username": "_json_key", }, - Outputs: []atc.TaskOutputConfig{ - {Name: "image"}, + }, + Params: params, + Run: atc.TaskRunConfig{ + Path: "/bin/sh", + Args: []string{"-c", strings.Join([]string{ + `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, + publishCommand, + }, "\n"), }, }, - } - steps = append(steps, stepWithAttemptsAndTimeout(pushStep, task.GetAttempts(), task.GetTimeout())) + Inputs: []atc.TaskInputConfig{ + {Name: gitDir}, + {Name: tagList_Dir}, + }, + }, } - return steps -} + steps = append(steps, stepWithAttemptsAndTimeout(pushStep, task.GetAttempts(), task.GetTimeout())) -func multiPlatform(task manifest.DockerPush) bool { - return !(len(task.Platforms) == 1 && task.Platforms[0] == "linux/amd64") + return steps } diff --git a/renderers/concourse/pipeline.go b/renderers/concourse/pipeline.go index aa2a8260..fe54b4b3 100644 --- a/renderers/concourse/pipeline.go +++ b/renderers/concourse/pipeline.go @@ -184,22 +184,6 @@ func (c Concourse) initialPlan(man manifest.Manifest, task manifest.Task, previo return steps } -func (c Concourse) dockerPushResources(tasks manifest.TaskList) (resourceConfigs atc.ResourceConfigs) { - for _, task := range tasks { - switch task := task.(type) { - case manifest.DockerPush: - if len(task.Platforms) == 1 { - resourceConfigs = append(resourceConfigs, c.dockerPushResource(task)) - } - case manifest.Parallel: - resourceConfigs = append(resourceConfigs, c.dockerPushResources(task.Tasks)...) - case manifest.Sequence: - resourceConfigs = append(resourceConfigs, c.dockerPushResources(task.Tasks)...) - } - } - - return resourceConfigs -} func (c Concourse) pipelineResources(triggers manifest.TriggerList) (resourceTypes atc.ResourceTypes, resourceConfigs atc.ResourceConfigs) { for _, trigger := range triggers { @@ -268,8 +252,6 @@ func (c Concourse) resourceConfigs(man manifest.Manifest) (resourceTypes atc.Res resourceConfigs = append(resourceConfigs, c.versionResource(man)) } - resourceConfigs = append(resourceConfigs, c.dockerPushResources(man.Tasks)...) - cfResourceTypes, cfResources := c.cfPushResources(man) resourceTypes = append(resourceTypes, cfResourceTypes...) resourceConfigs = append(resourceConfigs, cfResources...) diff --git a/renderers/concourse/pipeline_resources.go b/renderers/concourse/pipeline_resources.go index 69dca598..86ec60a1 100644 --- a/renderers/concourse/pipeline_resources.go +++ b/renderers/concourse/pipeline_resources.go @@ -230,19 +230,6 @@ func (c Concourse) deployCFResource(deployCF manifest.DeployCF, resourceName str } } -func (c Concourse) dockerPushResource(docker manifest.DockerPush) atc.ResourceConfig { - return atc.ResourceConfig{ - Name: manifest.DockerTrigger{Image: docker.Image}.GetTriggerName(), - Type: "registry-image", - Source: atc.Source{ - "username": docker.Username, - "password": docker.Password, - "repository": docker.Image, - }, - CheckEvery: &longResourceCheckInterval, - } -} - func (c Concourse) dockerTriggerResource(trigger manifest.DockerTrigger) atc.ResourceConfig { config := atc.ResourceConfig{ Name: trigger.GetTriggerName(), From ef67107cca0d87da9f33cb3b2b5c24c8735ac140 Mon Sep 17 00:00:00 2001 From: Simon Johansson Date: Wed, 6 Sep 2023 12:19:43 +0200 Subject: [PATCH 02/40] Make sure the cache works with buildx. --- e2e/actions/docker-push/workflowExpected.yml | 6 ++-- e2e/concourse/docker-push/.halfpipe.io | 1 + .../docker-push/pipelineExpected.yml | 2 +- renderers/actions/docker_push.go | 5 ++-- renderers/concourse/docker_push.go | 29 ++++++++++++------- renderers/shared/docker_push.go | 3 ++ 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/e2e/actions/docker-push/workflowExpected.yml b/e2e/actions/docker-push/workflowExpected.yml index cea79dbb..10f26533 100644 --- a/e2e/actions/docker-push/workflowExpected.yml +++ b/e2e/actions/docker-push/workflowExpected.yml @@ -293,14 +293,16 @@ jobs: RUNNING_IN_CI=true VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} - cache-from: type=registry,ref=eu.gcr.io/halfpipe-io/cache/someImage + cache-from: type=registry,ref=eu.gcr.io/halfpipe-io/cache/someImage:buildcache cache-to: type=inline context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64,linux/arm64 provenance: false push: true - tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} + tags: |- + eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} + eu.gcr.io/halfpipe-io/cache/someImage:buildcache - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy with: diff --git a/e2e/concourse/docker-push/.halfpipe.io b/e2e/concourse/docker-push/.halfpipe.io index 03657f2f..bfd0dced 100644 --- a/e2e/concourse/docker-push/.halfpipe.io +++ b/e2e/concourse/docker-push/.halfpipe.io @@ -28,6 +28,7 @@ tasks: retries: 1 ignore_vulnerabilities: true scan_timeout: 30 + use_cache: true platforms: - "linux/amd64" - "linux/arm64" diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index 460f05fd..b3e0066e 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -300,7 +300,7 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache --push --provenance=false --cache-from=type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache --cache-to=type=inline $CONTEXT path: /bin/sh privileged: true task: build diff --git a/renderers/actions/docker_push.go b/renderers/actions/docker_push.go index 523abc23..31d16068 100644 --- a/renderers/actions/docker_push.go +++ b/renderers/actions/docker_push.go @@ -54,7 +54,7 @@ func buildImage(a *Actions, task manifest.DockerPush, buildArgs map[string]strin "context": path.Join(a.workingDir, task.BuildPath), "file": path.Join(a.workingDir, task.DockerfilePath), "push": true, - "tags": shared.CachePath(task, ":${{ env.GIT_REVISION }}"), + "tags": shared.CachePath(task, "${{ env.GIT_REVISION }}"), "build-args": MultiLine{buildArgs}, "platforms": strings.Join(task.Platforms, ","), "provenance": false, @@ -62,7 +62,8 @@ func buildImage(a *Actions, task manifest.DockerPush, buildArgs map[string]strin } if task.UseCache { - step.With["cache-from"] = fmt.Sprintf("type=registry,ref=%s", shared.CachePath(task, "")) + step.With["tags"] = fmt.Sprintf("%s\n%s", step.With["tags"], shared.CachePath(task, "buildcache")) + step.With["cache-from"] = fmt.Sprintf("type=registry,ref=%s", shared.CachePath(task, "buildcache")) step.With["cache-to"] = "type=inline" } diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index 54e522f0..a12ec020 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -99,16 +99,8 @@ func createTagList(task manifest.DockerPush, updatePipeline bool) []atc.Step { } func trivyTask(task manifest.DockerPush, fullBasePath string, basePath string) atc.StepConfig { - var imageFile string - var gitRef string - imageFile = shared.CachePath(task, "") - - gitRef = fmt.Sprintf(":$(cat %s)", pathToGitRef(gitDir, basePath)) - - //exitCode := 1 - //if task.IgnoreVulnerabilities { - // exitCode = 0 - //} + imageFile := shared.CachePath(task, "") + gitRef := fmt.Sprintf(":$(cat %s)", pathToGitRef(gitDir, basePath)) // temporary: always exit 0 until we have communicated the ignoreVulnerabilites opt-in exitCode := 0 @@ -152,6 +144,7 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { var steps []atc.Step image, tag := shared.SplitTag(task.Image) dockerImageWithCachePath := shared.CachePath(task, "") + buildCachePath := shared.CachePath(task, "buildcache") fullBasePath := path.Join(gitDir, basePath) if task.RestoreArtifacts { @@ -172,6 +165,20 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { platforms := strings.Join(task.Platforms, ",") + tags := []string{ + fmt.Sprintf("-t %s:$(cat git/.git/ref)", dockerImageWithCachePath), + } + if task.UseCache { + tags = append(tags, fmt.Sprintf("-t %s", buildCachePath)) + } + + buildCommand := fmt.Sprintf(`docker buildx build -f $DOCKERFILE --platform %s %s --push --provenance=false`, platforms, strings.Join(tags, " ")) + if task.UseCache { + buildCommand = fmt.Sprintf(`%s --cache-from=type=registry,ref=%s`, buildCommand, buildCachePath) + buildCommand = fmt.Sprintf(`%s --cache-to=type=inline`, buildCommand) + } + buildCommand = fmt.Sprintf(`%s $CONTEXT`, buildCommand) + buildStep = &atc.TaskStep{ Name: "build", Privileged: true, @@ -191,7 +198,7 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { Path: "/bin/sh", Args: []string{"-c", strings.Join([]string{ `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - fmt.Sprintf(`docker buildx build -f $DOCKERFILE --platform %s -t %s:$(cat git/.git/ref) --push --provenance=false $CONTEXT`, platforms, dockerImageWithCachePath)}, "\n"), + buildCommand}, "\n"), }, }, Inputs: []atc.TaskInputConfig{ diff --git a/renderers/shared/docker_push.go b/renderers/shared/docker_push.go index 4e10c34b..9f9b545e 100644 --- a/renderers/shared/docker_push.go +++ b/renderers/shared/docker_push.go @@ -9,6 +9,9 @@ import ( func CachePath(task manifest.DockerPush, tag string) string { image, _ := SplitTag(task.Image) + if tag != "" && !strings.HasPrefix(tag, ":") { + tag = fmt.Sprintf(":%s", tag) + } if strings.HasPrefix(task.Image, config.DockerRegistry) { r := strings.Replace(image, config.DockerRegistry, fmt.Sprintf("%scache/", config.DockerRegistry), 1) From 1d7abd33b94680e4bd6f2e1773ba6b56e9ae447a Mon Sep 17 00:00:00 2001 From: Simon Johansson Date: Wed, 6 Sep 2023 12:26:28 +0200 Subject: [PATCH 03/40] Dont use env vars for docker file path and context --- e2e/concourse/artifacts/pipelineExpected.yml | 6 +----- .../deploy-katee/pipelineExpected.yml | 6 +----- .../docker-push-paths/pipelineExpected.yml | 12 ++---------- .../pipelineExpected.yml | 6 +----- .../pipelineExpected.yml | 6 +----- .../pipelineExpected.yml | 12 ++---------- .../pipelineExpected.yml | 12 ++---------- e2e/concourse/docker-push/pipelineExpected.yml | 18 +++--------------- .../timer-trigger/pipelineExpected.yml | 6 +----- renderers/concourse/docker_push.go | 8 ++++---- 10 files changed, 18 insertions(+), 74 deletions(-) diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index a10e6ef0..ed0d040e 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -452,16 +452,14 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/artifacts/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f docker_build/e2e/concourse/artifacts/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false docker_build/e2e/concourse/artifacts path: /bin/sh privileged: true task: build @@ -508,9 +506,7 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/artifacts/Dockerfile platform: linux run: args: diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml index 641329d2..9ad7e4f7 100644 --- a/e2e/concourse/deploy-katee/pipelineExpected.yml +++ b/e2e/concourse/deploy-katee/pipelineExpected.yml @@ -100,16 +100,14 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/deploy-katee DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/deploy-katee/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/deploy-katee/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/deploy-katee path: /bin/sh privileged: true task: build @@ -153,9 +151,7 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/deploy-katee DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/deploy-katee/Dockerfile platform: linux run: args: diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml index db911bb0..bf4f7180 100644 --- a/e2e/concourse/docker-push-paths/pipelineExpected.yml +++ b/e2e/concourse/docker-push-paths/pipelineExpected.yml @@ -47,16 +47,14 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-paths/some/build/dir DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-paths/some/build/dir path: /bin/sh privileged: true task: build @@ -100,9 +98,7 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-paths/some/build/dir DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile platform: linux run: args: @@ -164,16 +160,14 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-paths DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-paths path: /bin/sh privileged: true task: build @@ -217,9 +211,7 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-paths DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml index 140308d7..111fb237 100644 --- a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml @@ -56,16 +56,14 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-docker-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh privileged: true task: build @@ -111,9 +109,7 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-docker-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml index ade3a00e..629b29d1 100644 --- a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml @@ -54,16 +54,14 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-pipeline-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh privileged: true task: build @@ -109,9 +107,7 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-pipeline-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml index f8a89d8d..9e694e85 100644 --- a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml @@ -251,16 +251,14 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --push --provenance=false docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true task: build @@ -307,9 +305,7 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile platform: linux run: args: @@ -429,16 +425,14 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --push --provenance=false docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true task: build @@ -485,9 +479,7 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml index f2138490..2f83b891 100644 --- a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml @@ -102,16 +102,14 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true task: build @@ -157,9 +155,7 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile platform: linux run: args: @@ -230,16 +226,14 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true task: build @@ -285,9 +279,7 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile platform: linux run: args: diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index b3e0066e..758052cb 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -49,16 +49,14 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/docker-push/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -104,9 +102,7 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: @@ -170,16 +166,14 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/docker-push/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -225,9 +219,7 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: @@ -291,16 +283,14 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache --push --provenance=false --cache-from=type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache --cache-to=type=inline $CONTEXT + docker buildx build -f git/e2e/concourse/docker-push/Dockerfile --platform linux/amd64,linux/arm64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache --push --provenance=false --cache-from=type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache --cache-to=type=inline git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -346,9 +336,7 @@ jobs: BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: diff --git a/e2e/concourse/timer-trigger/pipelineExpected.yml b/e2e/concourse/timer-trigger/pipelineExpected.yml index 94870239..500fb3fa 100644 --- a/e2e/concourse/timer-trigger/pipelineExpected.yml +++ b/e2e/concourse/timer-trigger/pipelineExpected.yml @@ -54,16 +54,14 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/timer-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/timer-trigger/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + docker buildx build -f git/e2e/concourse/timer-trigger/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/timer-trigger path: /bin/sh privileged: true task: build @@ -109,9 +107,7 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/timer-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/timer-trigger/Dockerfile platform: linux run: args: diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index a12ec020..065bf9cc 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -151,9 +151,9 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { fullBasePath = path.Join(dockerBuildTmpDir, basePath) } + dockerFilePath := path.Join(fullBasePath, task.DockerfilePath) + dockerContext := path.Join(fullBasePath, task.BuildPath) params := atc.TaskEnv{ - "CONTEXT": path.Join(fullBasePath, task.BuildPath), - "DOCKERFILE": path.Join(fullBasePath, task.DockerfilePath), "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", } @@ -172,12 +172,12 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { tags = append(tags, fmt.Sprintf("-t %s", buildCachePath)) } - buildCommand := fmt.Sprintf(`docker buildx build -f $DOCKERFILE --platform %s %s --push --provenance=false`, platforms, strings.Join(tags, " ")) + buildCommand := fmt.Sprintf(`docker buildx build -f %s --platform %s %s --push --provenance=false`, dockerFilePath, platforms, strings.Join(tags, " ")) if task.UseCache { buildCommand = fmt.Sprintf(`%s --cache-from=type=registry,ref=%s`, buildCommand, buildCachePath) buildCommand = fmt.Sprintf(`%s --cache-to=type=inline`, buildCommand) } - buildCommand = fmt.Sprintf(`%s $CONTEXT`, buildCommand) + buildCommand = fmt.Sprintf(`%s %s`, buildCommand, dockerContext) buildStep = &atc.TaskStep{ Name: "build", From 5f4f21c73f13edb431eaa523123e477305477cc8 Mon Sep 17 00:00:00 2001 From: Simon Johansson Date: Wed, 6 Sep 2023 12:53:55 +0200 Subject: [PATCH 04/40] Construct docker build string from an array for easier readability --- renderers/concourse/docker_push.go | 31 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index 065bf9cc..9a01b735 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -144,15 +144,12 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { var steps []atc.Step image, tag := shared.SplitTag(task.Image) dockerImageWithCachePath := shared.CachePath(task, "") - buildCachePath := shared.CachePath(task, "buildcache") fullBasePath := path.Join(gitDir, basePath) if task.RestoreArtifacts { fullBasePath = path.Join(dockerBuildTmpDir, basePath) } - dockerFilePath := path.Join(fullBasePath, task.DockerfilePath) - dockerContext := path.Join(fullBasePath, task.BuildPath) params := atc.TaskEnv{ "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", } @@ -163,22 +160,23 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { var buildStep *atc.TaskStep - platforms := strings.Join(task.Platforms, ",") - - tags := []string{ - fmt.Sprintf("-t %s:$(cat git/.git/ref)", dockerImageWithCachePath), - } - if task.UseCache { - tags = append(tags, fmt.Sprintf("-t %s", buildCachePath)) + buildCommand := []string{ + "docker buildx build", + fmt.Sprintf("-f %s", path.Join(fullBasePath, task.DockerfilePath)), + "--push", + "--provenance false", + fmt.Sprintf("--platform %s", strings.Join(task.Platforms, ",")), + fmt.Sprintf("--tag %s", shared.CachePath(task, "$(cat git/.git/ref)")), } - - buildCommand := fmt.Sprintf(`docker buildx build -f %s --platform %s %s --push --provenance=false`, dockerFilePath, platforms, strings.Join(tags, " ")) if task.UseCache { - buildCommand = fmt.Sprintf(`%s --cache-from=type=registry,ref=%s`, buildCommand, buildCachePath) - buildCommand = fmt.Sprintf(`%s --cache-to=type=inline`, buildCommand) + buildCommand = append(buildCommand, fmt.Sprintf("--tag %s", shared.CachePath(task, "buildcache"))) + buildCommand = append(buildCommand, fmt.Sprintf("--cache-from type=registry,ref=%s", shared.CachePath(task, "buildcache"))) + buildCommand = append(buildCommand, "--cache-to type=inline") } - buildCommand = fmt.Sprintf(`%s %s`, buildCommand, dockerContext) + buildCommand = append(buildCommand, path.Join(fullBasePath, task.BuildPath)) + buildCommandStr := strings.Join(buildCommand, ` \ + `) buildStep = &atc.TaskStep{ Name: "build", Privileged: true, @@ -198,7 +196,8 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { Path: "/bin/sh", Args: []string{"-c", strings.Join([]string{ `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - buildCommand}, "\n"), + fmt.Sprintf(`echo $ %s`, buildCommandStr), + buildCommandStr}, "\n"), }, }, Inputs: []atc.TaskInputConfig{ From 35f25416d23af5462ff6b31a323a8a315322019e Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 6 Sep 2023 14:28:47 +0100 Subject: [PATCH 05/40] fix simon's mess --- e2e/concourse/artifacts/pipelineExpected.yml | 15 +++++- .../deploy-katee/pipelineExpected.yml | 15 +++++- .../docker-push-paths/pipelineExpected.yml | 30 ++++++++++- .../pipelineExpected.yml | 15 +++++- .../pipelineExpected.yml | 15 +++++- .../pipelineExpected.yml | 30 ++++++++++- .../pipelineExpected.yml | 30 ++++++++++- .../docker-push/pipelineExpected.yml | 51 +++++++++++++++++-- .../timer-trigger/pipelineExpected.yml | 15 +++++- 9 files changed, 202 insertions(+), 14 deletions(-) diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index ed0d040e..00001462 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -459,7 +459,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f docker_build/e2e/concourse/artifacts/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false docker_build/e2e/concourse/artifacts + echo $ docker buildx build \ + -f docker_build/e2e/concourse/artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + docker_build/e2e/concourse/artifacts + docker buildx build \ + -f docker_build/e2e/concourse/artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + docker_build/e2e/concourse/artifacts path: /bin/sh privileged: true task: build diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml index 9ad7e4f7..97d7d48c 100644 --- a/e2e/concourse/deploy-katee/pipelineExpected.yml +++ b/e2e/concourse/deploy-katee/pipelineExpected.yml @@ -107,7 +107,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/deploy-katee/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/deploy-katee + echo $ docker buildx build \ + -f git/e2e/concourse/deploy-katee/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ + git/e2e/concourse/deploy-katee + docker buildx build \ + -f git/e2e/concourse/deploy-katee/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ + git/e2e/concourse/deploy-katee path: /bin/sh privileged: true task: build diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml index bf4f7180..246527ac 100644 --- a/e2e/concourse/docker-push-paths/pipelineExpected.yml +++ b/e2e/concourse/docker-push-paths/pipelineExpected.yml @@ -54,7 +54,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-paths/some/build/dir + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-paths/some/build/dir + docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-paths/some/build/dir path: /bin/sh privileged: true task: build @@ -167,7 +180,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-paths + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-paths + docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-paths path: /bin/sh privileged: true task: build diff --git a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml index 111fb237..bc40614a 100644 --- a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml @@ -63,7 +63,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-with-docker-trigger + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-docker-trigger + docker buildx build \ + -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh privileged: true task: build diff --git a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml index 629b29d1..fdec0935 100644 --- a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml @@ -61,7 +61,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-with-pipeline-trigger + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-pipeline-trigger + docker buildx build \ + -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh privileged: true task: build diff --git a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml index 9e694e85..c83e5f0d 100644 --- a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml @@ -258,7 +258,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --push --provenance=false docker_build/e2e/concourse/docker-push-with-restore-artifacts + echo $ docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts + docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true task: build @@ -432,7 +445,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --push --provenance=false docker_build/e2e/concourse/docker-push-with-restore-artifacts + echo $ docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts + docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true task: build diff --git a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml index 2f83b891..a59af73b 100644 --- a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml @@ -109,7 +109,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-with-update-pipeline + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-update-pipeline + docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true task: build @@ -233,7 +246,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push-with-update-pipeline + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-update-pipeline + docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true task: build diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index 758052cb..cdc8c7a0 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -56,7 +56,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/docker-push/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push + docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -173,7 +186,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/docker-push/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/docker-push + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push + docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -290,7 +316,26 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/docker-push/Dockerfile --platform linux/amd64,linux/arm64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache --push --provenance=false --cache-from=type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache --cache-to=type=inline git/e2e/concourse/docker-push + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64,linux/arm64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-to type=inline \ + git/e2e/concourse/docker-push + docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64,linux/arm64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-to type=inline \ + git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build diff --git a/e2e/concourse/timer-trigger/pipelineExpected.yml b/e2e/concourse/timer-trigger/pipelineExpected.yml index 500fb3fa..dc91d383 100644 --- a/e2e/concourse/timer-trigger/pipelineExpected.yml +++ b/e2e/concourse/timer-trigger/pipelineExpected.yml @@ -61,7 +61,20 @@ jobs: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f git/e2e/concourse/timer-trigger/Dockerfile --platform linux/amd64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --push --provenance=false git/e2e/concourse/timer-trigger + echo $ docker buildx build \ + -f git/e2e/concourse/timer-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/timer-trigger + docker buildx build \ + -f git/e2e/concourse/timer-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/timer-trigger path: /bin/sh privileged: true task: build From 33621f5f01a1e394fffb4b78e9ce1d3667755a03 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 6 Sep 2023 15:15:24 +0100 Subject: [PATCH 06/40] update-pipeline for actions now runs a script in the ARC image instead of using docker - much faster --- .../workflowExpected.yml | 5 +---- .../feature-update-pipeline/workflowExpected.yml | 5 +---- renderers/actions/update.go | 11 +---------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml b/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml index 8405de0c..32f10560 100644 --- a/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml +++ b/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml @@ -35,10 +35,7 @@ jobs: submodules: recursive - name: Sync workflow with halfpipe manifest id: sync - uses: docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update - with: - args: -c "cd e2e/actions/feature-update-pipeline-and-tag; update-actions-workflow" - entrypoint: /bin/bash + run: halfpipe-update-workflow env: HALFPIPE_FILE_PATH: .halfpipe.io.yml - name: Commit and push changes to workflow diff --git a/e2e/actions/feature-update-pipeline/workflowExpected.yml b/e2e/actions/feature-update-pipeline/workflowExpected.yml index f26f9131..060404c9 100644 --- a/e2e/actions/feature-update-pipeline/workflowExpected.yml +++ b/e2e/actions/feature-update-pipeline/workflowExpected.yml @@ -35,10 +35,7 @@ jobs: submodules: recursive - name: Sync workflow with halfpipe manifest id: sync - uses: docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update - with: - args: -c "cd e2e/actions/feature-update-pipeline; update-actions-workflow" - entrypoint: /bin/bash + run: halfpipe-update-workflow env: HALFPIPE_FILE_PATH: .halfpipe.io - name: Commit and push changes to workflow diff --git a/renderers/actions/update.go b/renderers/actions/update.go index 2f0117d2..b36dd334 100644 --- a/renderers/actions/update.go +++ b/renderers/actions/update.go @@ -6,19 +6,10 @@ import ( ) func (a *Actions) updateSteps(task manifest.Update, man manifest.Manifest) Steps { - cdPrefix := "" - if a.workingDir != "" { - cdPrefix = fmt.Sprintf("cd %s; ", a.workingDir) - } - update := Step{ Name: "Sync workflow with halfpipe manifest", ID: "sync", - Uses: "docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update", - With: With{ - "args": fmt.Sprintf(`-c "%supdate-actions-workflow"`, cdPrefix), - "entrypoint": "/bin/bash", - }, + Run: "halfpipe-update-workflow", Env: Env{ "HALFPIPE_FILE_PATH": a.halfpipeFilePath, }, From 1d007259f879f0711afb5b84763ecb0394be3e0b Mon Sep 17 00:00:00 2001 From: Michiel Stigter Date: Thu, 7 Sep 2023 12:42:15 +0200 Subject: [PATCH 07/40] revert to before introducing buildx builds --- e2e/actions/docker-push/workflowExpected.yml | 6 +- .../workflowExpected.yml | 5 +- .../workflowExpected.yml | 5 +- e2e/concourse/artifacts/pipelineExpected.yml | 73 ++---- .../deploy-katee/pipelineExpected.yml | 71 ++---- .../docker-push-paths/pipelineExpected.yml | 142 ++++-------- .../pipelineExpected.yml | 73 ++---- .../pipelineExpected.yml | 73 ++---- .../pipelineExpected.yml | 146 ++++-------- .../pipelineExpected.yml | 146 ++++-------- e2e/concourse/docker-push/.halfpipe.io | 1 - .../docker-push/pipelineExpected.yml | 179 +++++---------- .../timer-trigger/pipelineExpected.yml | 73 ++---- linters/docker-push.go | 7 - linters/docker-push_test.go | 6 +- linters/errors.go | 10 +- renderers/actions/docker_push.go | 5 +- renderers/actions/update.go | 11 +- renderers/concourse/docker_push.go | 215 +++++++++++------- renderers/concourse/pipeline.go | 18 ++ renderers/concourse/pipeline_resources.go | 13 ++ renderers/shared/docker_push.go | 3 - 22 files changed, 476 insertions(+), 805 deletions(-) diff --git a/e2e/actions/docker-push/workflowExpected.yml b/e2e/actions/docker-push/workflowExpected.yml index 10f26533..cea79dbb 100644 --- a/e2e/actions/docker-push/workflowExpected.yml +++ b/e2e/actions/docker-push/workflowExpected.yml @@ -293,16 +293,14 @@ jobs: RUNNING_IN_CI=true VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} - cache-from: type=registry,ref=eu.gcr.io/halfpipe-io/cache/someImage:buildcache + cache-from: type=registry,ref=eu.gcr.io/halfpipe-io/cache/someImage cache-to: type=inline context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64,linux/arm64 provenance: false push: true - tags: |- - eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - eu.gcr.io/halfpipe-io/cache/someImage:buildcache + tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy with: diff --git a/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml b/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml index 32f10560..8405de0c 100644 --- a/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml +++ b/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml @@ -35,7 +35,10 @@ jobs: submodules: recursive - name: Sync workflow with halfpipe manifest id: sync - run: halfpipe-update-workflow + uses: docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update + with: + args: -c "cd e2e/actions/feature-update-pipeline-and-tag; update-actions-workflow" + entrypoint: /bin/bash env: HALFPIPE_FILE_PATH: .halfpipe.io.yml - name: Commit and push changes to workflow diff --git a/e2e/actions/feature-update-pipeline/workflowExpected.yml b/e2e/actions/feature-update-pipeline/workflowExpected.yml index 060404c9..f26f9131 100644 --- a/e2e/actions/feature-update-pipeline/workflowExpected.yml +++ b/e2e/actions/feature-update-pipeline/workflowExpected.yml @@ -35,7 +35,10 @@ jobs: submodules: recursive - name: Sync workflow with halfpipe manifest id: sync - run: halfpipe-update-workflow + uses: docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update + with: + args: -c "cd e2e/actions/feature-update-pipeline; update-actions-workflow" + entrypoint: /bin/bash env: HALFPIPE_FILE_PATH: .halfpipe.io - name: Commit and push changes to workflow diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index 00001462..dea44ff7 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -436,15 +436,13 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList - name: docker_build + outputs: + - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -452,27 +450,17 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: docker_build/e2e/concourse/artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: docker_build/e2e/concourse/artifacts/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f docker_build/e2e/concourse/artifacts/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - docker_build/e2e/concourse/artifacts - docker buildx build \ - -f docker_build/e2e/concourse/artifacts/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - docker_build/e2e/concourse/artifacts + build path: /bin/sh privileged: true task: build @@ -485,9 +473,8 @@ jobs: type: docker-image inputs: - name: git + - name: image - name: docker_build - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -495,41 +482,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: docker_build/e2e/concourse/artifacts path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: halfpipe-fly timeout: 1h serial: true - build_log_retention: @@ -785,6 +747,13 @@ resources: folder: halfpipe-team/halfpipe-e2e-artifacts json_key: ((halfpipe-artifacts.private_key)) type: gcp-resource +- check_every: 24h0m0s + name: halfpipe-fly + source: + password: verysecret + repository: springerplatformengineering/halfpipe-fly + username: rob + type: registry-image - check_every: 24h0m0s name: cf-snpaas-pe-staging source: diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml index 97d7d48c..b4d6bb32 100644 --- a/e2e/concourse/deploy-katee/pipelineExpected.yml +++ b/e2e/concourse/deploy-katee/pipelineExpected.yml @@ -87,40 +87,28 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/deploy-katee DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/deploy-katee/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/deploy-katee/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ - git/e2e/concourse/deploy-katee - docker buildx build \ - -f git/e2e/concourse/deploy-katee/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ - git/e2e/concourse/deploy-katee + build path: /bin/sh privileged: true task: build @@ -133,8 +121,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -142,39 +129,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/deploy-katee path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --tag eu.gcr.io/halfpipe-io/halfpipe-team/someImage:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: someimage timeout: 1h serial: true - build_log_retention: @@ -415,3 +379,10 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-pipeline-name type: semver +- check_every: 24h0m0s + name: someimage + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-team/someImage + username: _json_key + type: registry-image diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml index 246527ac..bcd4794f 100644 --- a/e2e/concourse/docker-push-paths/pipelineExpected.yml +++ b/e2e/concourse/docker-push-paths/pipelineExpected.yml @@ -34,40 +34,28 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-paths/some/build/dir DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-paths/some/build/dir - docker buildx build \ - -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-paths/some/build/dir + build path: /bin/sh privileged: true task: build @@ -80,8 +68,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -89,39 +76,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/docker-push-paths path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: halfpipe-fly timeout: 1h serial: true - build_log_retention: @@ -160,40 +124,28 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-paths DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-paths - docker buildx build \ - -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-paths + build path: /bin/sh privileged: true task: build @@ -206,8 +158,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -215,39 +166,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/docker-push-paths path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: halfpipe timeout: 1h serial: true resources: @@ -260,3 +188,17 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git +- check_every: 24h0m0s + name: halfpipe-fly + source: + password: verysecret + repository: springerplatformengineering/halfpipe-fly + username: rob + type: registry-image +- check_every: 24h0m0s + name: halfpipe + source: + password: verysecret + repository: springerplatformengineering/halfpipe + username: rob + type: registry-image diff --git a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml index bc40614a..202fc7d0 100644 --- a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml @@ -41,14 +41,12 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -56,27 +54,17 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-with-docker-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-with-docker-trigger - docker buildx build \ - -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-with-docker-trigger + build path: /bin/sh privileged: true task: build @@ -89,8 +77,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -98,41 +85,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: halfpipe-fly timeout: 1h serial: true resources: @@ -149,3 +111,10 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git +- check_every: 24h0m0s + name: halfpipe-fly + source: + password: verysecret + repository: springerplatformengineering/halfpipe-fly + username: rob + type: registry-image diff --git a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml index fdec0935..281e223a 100644 --- a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml @@ -39,14 +39,12 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -54,27 +52,17 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-with-pipeline-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-with-pipeline-trigger - docker buildx build \ - -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-with-pipeline-trigger + build path: /bin/sh privileged: true task: build @@ -87,8 +75,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -96,41 +83,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: halfpipe-fly timeout: 1h serial: true resource_types: @@ -151,6 +113,13 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git +- check_every: 24h0m0s + name: halfpipe-fly + source: + password: verysecret + repository: springerplatformengineering/halfpipe-fly + username: rob + type: registry-image - name: halfpipe-example-docker-push.docker-push source: concourse_url: ((concourse.url)) diff --git a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml index c83e5f0d..c9d96dbc 100644 --- a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml @@ -235,15 +235,13 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList - name: docker_build + outputs: + - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -251,27 +249,17 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ - docker_build/e2e/concourse/docker-push-with-restore-artifacts - docker buildx build \ - -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ - docker_build/e2e/concourse/docker-push-with-restore-artifacts + build path: /bin/sh privileged: true task: build @@ -284,9 +272,8 @@ jobs: type: docker-image inputs: - name: git + - name: image - name: docker_build - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -294,41 +281,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --tag springerplatformengineering/image1:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: image1 timeout: 1h serial: true - build_log_retention: @@ -422,15 +384,13 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList - name: docker_build + outputs: + - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -438,27 +398,17 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ - docker_build/e2e/concourse/docker-push-with-restore-artifacts - docker buildx build \ - -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ - docker_build/e2e/concourse/docker-push-with-restore-artifacts + build path: /bin/sh privileged: true task: build @@ -471,9 +421,8 @@ jobs: type: docker-image inputs: - name: git + - name: image - name: docker_build - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -481,41 +430,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --tag springerplatformengineering/image2:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: image2 timeout: 1h serial: true resource_types: @@ -552,3 +476,17 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-docker-push-with-update-pipeline type: semver +- check_every: 24h0m0s + name: image1 + source: + password: verysecret + repository: springerplatformengineering/image1 + username: rob + type: registry-image +- check_every: 24h0m0s + name: image2 + source: + password: verysecret + repository: springerplatformengineering/image2 + username: rob + type: registry-image diff --git a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml index a59af73b..b0f327bd 100644 --- a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml @@ -87,14 +87,12 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -102,27 +100,17 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-with-update-pipeline - docker buildx build \ - -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-with-update-pipeline + build path: /bin/sh privileged: true task: build @@ -135,8 +123,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -144,41 +131,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --tag springerplatformengineering/image1:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: image1 timeout: 1h serial: true - build_log_retention: @@ -224,14 +186,12 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -239,27 +199,17 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-with-update-pipeline - docker buildx build \ - -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push-with-update-pipeline + build path: /bin/sh privileged: true task: build @@ -272,8 +222,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -281,41 +230,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --tag springerplatformengineering/image2:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: image2 timeout: 1h serial: true resources: @@ -336,3 +260,17 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-docker-push-with-update-pipeline type: semver +- check_every: 24h0m0s + name: image1 + source: + password: verysecret + repository: springerplatformengineering/image1 + username: rob + type: registry-image +- check_every: 24h0m0s + name: image2 + source: + password: verysecret + repository: springerplatformengineering/image2 + username: rob + type: registry-image diff --git a/e2e/concourse/docker-push/.halfpipe.io b/e2e/concourse/docker-push/.halfpipe.io index bfd0dced..03657f2f 100644 --- a/e2e/concourse/docker-push/.halfpipe.io +++ b/e2e/concourse/docker-push/.halfpipe.io @@ -28,7 +28,6 @@ tasks: retries: 1 ignore_vulnerabilities: true scan_timeout: 30 - use_cache: true platforms: - "linux/amd64" - "linux/arm64" diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index cdc8c7a0..c6c2f4ef 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -34,14 +34,12 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -49,27 +47,17 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/docker-push/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push - docker buildx build \ - -f git/e2e/concourse/docker-push/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push + build path: /bin/sh privileged: true task: build @@ -82,8 +70,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -91,41 +78,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) thisIsMy_Tag; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe_fly:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: halfpipe-fly.thisismy-tag timeout: 1h serial: true - build_log_retention: @@ -166,40 +128,28 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/docker-push/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push - docker buildx build \ - -f git/e2e/concourse/docker-push/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ - git/e2e/concourse/docker-push + build path: /bin/sh privileged: true task: build @@ -213,8 +163,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -222,40 +171,17 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true + trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h - attempts: 2 - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) thisIsMy_Tag2; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe_fly:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: halfpipe-fly.thisismy-tag2 timeout: 1h serial: true - build_log_retention: @@ -304,38 +230,23 @@ jobs: inputs: - name: git - name: tagList + outputs: + - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/docker-push/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64,linux/arm64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ - --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ - --cache-to type=inline \ - git/e2e/concourse/docker-push - docker buildx build \ - -f git/e2e/concourse/docker-push/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64,linux/arm64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ - --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ - --cache-to type=inline \ - git/e2e/concourse/docker-push + docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT path: /bin/sh privileged: true task: build @@ -349,8 +260,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -358,8 +268,7 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true - dir: git/e2e/concourse/docker-push + trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) || true path: /bin/sh task: trivy timeout: 1h @@ -376,12 +285,16 @@ jobs: inputs: - name: git - name: tagList + outputs: + - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: @@ -404,3 +317,17 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git +- check_every: 24h0m0s + name: halfpipe-fly.thisismy-tag + source: + password: verysecret + repository: springerplatformengineering/halfpipe_fly:thisIsMy_Tag + username: rob + type: registry-image +- check_every: 24h0m0s + name: halfpipe-fly.thisismy-tag2 + source: + password: verysecret + repository: springerplatformengineering/halfpipe_fly:thisIsMy_Tag2 + username: rob + type: registry-image diff --git a/e2e/concourse/timer-trigger/pipelineExpected.yml b/e2e/concourse/timer-trigger/pipelineExpected.yml index dc91d383..167c3e62 100644 --- a/e2e/concourse/timer-trigger/pipelineExpected.yml +++ b/e2e/concourse/timer-trigger/pipelineExpected.yml @@ -39,14 +39,12 @@ jobs: image_resource: name: "" source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key + repository: concourse/oci-build-task type: registry-image inputs: - name: git - - name: tagList + outputs: + - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -54,27 +52,17 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" + CONTEXT: git/e2e/concourse/timer-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + DOCKERFILE: git/e2e/concourse/timer-trigger/Dockerfile platform: linux run: args: - -c - |- + mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - echo $ docker buildx build \ - -f git/e2e/concourse/timer-trigger/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - git/e2e/concourse/timer-trigger - docker buildx build \ - -f git/e2e/concourse/timer-trigger/Dockerfile \ - --push \ - --provenance false \ - --platform linux/amd64 \ - --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ - git/e2e/concourse/timer-trigger + build path: /bin/sh privileged: true task: build @@ -87,8 +75,7 @@ jobs: type: docker-image inputs: - name: git - params: - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + - name: image platform: linux run: args: @@ -96,41 +83,16 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true dir: git/e2e/concourse/timer-trigger path: /bin/sh task: trivy timeout: 1h - - config: - image_resource: - name: "" - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-buildx - tag: latest - username: _json_key - type: registry-image - inputs: - - name: git - - name: tagList - params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - platform: linux - run: - args: - - -c - - |- - echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done - path: /bin/sh - privileged: true - task: publish-final-image + - no_get: true + params: + additional_tags: tagList/tagList + image: image/image.tar + put: halfpipe-fly timeout: 1h serial: true resource_types: @@ -157,3 +119,10 @@ resources: fire_immediately: true location: UTC type: halfpipe-cron-resource +- check_every: 24h0m0s + name: halfpipe-fly + source: + password: verysecret + repository: springerplatformengineering/halfpipe-fly + username: rob + type: registry-image diff --git a/linters/docker-push.go b/linters/docker-push.go index c9a45361..8b188df8 100644 --- a/linters/docker-push.go +++ b/linters/docker-push.go @@ -2,7 +2,6 @@ package linters import ( "fmt" - "golang.org/x/exp/slices" "os" "regexp" "strings" @@ -62,11 +61,5 @@ func LintDockerPushTask(docker manifest.DockerPush, manifest manifest.Manifest, errs = append(errs, ErrDockerPushTag.AsWarning()) } - for _, platform := range docker.Platforms { - if !slices.Contains([]string{"linux/amd64", "linux/arm64"}, platform) { - errs = append(errs, ErrDockerPlatformUnknown) - } - } - return errs } diff --git a/linters/docker-push_test.go b/linters/docker-push_test.go index 032c610b..30b298ab 100644 --- a/linters/docker-push_test.go +++ b/linters/docker-push_test.go @@ -355,7 +355,7 @@ func TestMultiplePlatforms(t *testing.T) { assert.Empty(t, errors) }) - t.Run("errors when unknown platform in docker push", func(t *testing.T) { + t.Run("only linux/amd64 for concourse is fine", func(t *testing.T) { fs := afero.Afero{Fs: afero.NewMemMapFs()} fs.WriteFile("Dockerfile", []byte("FROM ubuntu"), 0777) @@ -364,12 +364,12 @@ func TestMultiplePlatforms(t *testing.T) { Username: "asd", Password: "asdf", DockerfilePath: "Dockerfile", - Platforms: []string{"linux/ad64"}, + Platforms: []string{"linux/amd64"}, } m := manifest.Manifest{Platform: "actions"} errors := LintDockerPushTask(task, m, fs) - assertContainsError(t, errors, ErrDockerPlatformUnknown) + assert.Empty(t, errors) }) } diff --git a/linters/errors.go b/linters/errors.go index 0493c04b..8092d33e 100644 --- a/linters/errors.go +++ b/linters/errors.go @@ -33,12 +33,10 @@ var ( ErrCFFromArtifact = newError("this file must be saved as an artifact in a previous task") ErrCFPrePromoteArtifact = newError("cannot have pre promote tasks with CF manifest restored from artifact") - ErrUnsupportedRegistry = newError("image must be from halfpipe registry. Please see ") - ErrDockerPushTag = newError("the field 'tag' is no longer used and is safe to delete") - - ErrDockerPlatformUnknown = newError("only linux/amd64 and/or linux/arm64 are supported") - ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") - ErrMultipleTriggers = newError("cannot have multiple triggers of this type") + ErrUnsupportedRegistry = newError("image must be from halfpipe registry. Please see ") + ErrDockerPushTag = newError("the field 'tag' is no longer used and is safe to delete") + ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") + ErrMultipleTriggers = newError("cannot have multiple triggers of this type") ErrVelaVariableMissing = newError("vela manifest variable is not specified in halfpipe manifest") ErrVelaNamespace = newError("vela namespace must start with 'katee-'") diff --git a/renderers/actions/docker_push.go b/renderers/actions/docker_push.go index 31d16068..523abc23 100644 --- a/renderers/actions/docker_push.go +++ b/renderers/actions/docker_push.go @@ -54,7 +54,7 @@ func buildImage(a *Actions, task manifest.DockerPush, buildArgs map[string]strin "context": path.Join(a.workingDir, task.BuildPath), "file": path.Join(a.workingDir, task.DockerfilePath), "push": true, - "tags": shared.CachePath(task, "${{ env.GIT_REVISION }}"), + "tags": shared.CachePath(task, ":${{ env.GIT_REVISION }}"), "build-args": MultiLine{buildArgs}, "platforms": strings.Join(task.Platforms, ","), "provenance": false, @@ -62,8 +62,7 @@ func buildImage(a *Actions, task manifest.DockerPush, buildArgs map[string]strin } if task.UseCache { - step.With["tags"] = fmt.Sprintf("%s\n%s", step.With["tags"], shared.CachePath(task, "buildcache")) - step.With["cache-from"] = fmt.Sprintf("type=registry,ref=%s", shared.CachePath(task, "buildcache")) + step.With["cache-from"] = fmt.Sprintf("type=registry,ref=%s", shared.CachePath(task, "")) step.With["cache-to"] = "type=inline" } diff --git a/renderers/actions/update.go b/renderers/actions/update.go index b36dd334..2f0117d2 100644 --- a/renderers/actions/update.go +++ b/renderers/actions/update.go @@ -6,10 +6,19 @@ import ( ) func (a *Actions) updateSteps(task manifest.Update, man manifest.Manifest) Steps { + cdPrefix := "" + if a.workingDir != "" { + cdPrefix = fmt.Sprintf("cd %s; ", a.workingDir) + } + update := Step{ Name: "Sync workflow with halfpipe manifest", ID: "sync", - Run: "halfpipe-update-workflow", + Uses: "docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update", + With: With{ + "args": fmt.Sprintf(`-c "%supdate-actions-workflow"`, cdPrefix), + "entrypoint": "/bin/bash", + }, Env: Env{ "HALFPIPE_FILE_PATH": a.halfpipeFilePath, }, diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index 9a01b735..f15ef681 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -16,10 +16,16 @@ var tagListFile = path.Join(tagList_Dir, "tagList") func (c Concourse) dockerPushJob(task manifest.DockerPush, basePath string, man manifest.Manifest) atc.JobConfig { var steps []atc.Step + resourceName := manifest.DockerTrigger{Image: task.Image}.GetTriggerName() + + fullBasePath := path.Join(gitDir, basePath) + if task.RestoreArtifacts { + fullBasePath = path.Join(dockerBuildTmpDir, basePath) + } steps = append(steps, restoreArtifacts(task)...) steps = append(steps, createTagList(task, man.FeatureToggles.UpdatePipeline())...) - steps = append(steps, buildAndPush(task, basePath)...) + steps = append(steps, buildAndPush(task, resourceName, fullBasePath)...) return atc.JobConfig{ Name: task.GetName(), @@ -98,9 +104,21 @@ func createTagList(task manifest.DockerPush, updatePipeline bool) []atc.Step { return append([]atc.Step{}, stepWithAttemptsAndTimeout(createTagList, task.GetAttempts(), task.Timeout)) } -func trivyTask(task manifest.DockerPush, fullBasePath string, basePath string) atc.StepConfig { - imageFile := shared.CachePath(task, "") - gitRef := fmt.Sprintf(":$(cat %s)", pathToGitRef(gitDir, basePath)) +func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { + var imageFile string + var gitRef string + if !multiPlatform(task) { + imageFile = fmt.Sprintf("--input %s", path.Join(relativePathToRepoRoot(gitDir, fullBasePath), "image/image.tar")) + } else { + imageFile = shared.CachePath(task, "") + gitRef = fmt.Sprintf(":$(cat %s)", path.Join(gitDir, ".git", "ref")) + fullBasePath = "" + } + + //exitCode := 1 + //if task.IgnoreVulnerabilities { + // exitCode = 0 + //} // temporary: always exit 0 until we have communicated the ignoreVulnerabilites opt-in exitCode := 0 @@ -124,11 +142,9 @@ func trivyTask(task manifest.DockerPush, fullBasePath string, basePath string) a }, "\n")}, Dir: fullBasePath, }, - Params: atc.TaskEnv{ - "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", - }, Inputs: []atc.TaskInputConfig{ {Name: gitDir}, + {Name: "image"}, }, }, } @@ -140,17 +156,14 @@ func trivyTask(task manifest.DockerPush, fullBasePath string, basePath string) a return step } -func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { +func buildAndPush(task manifest.DockerPush, resourceName string, fullBasePath string) []atc.Step { var steps []atc.Step image, tag := shared.SplitTag(task.Image) dockerImageWithCachePath := shared.CachePath(task, "") - fullBasePath := path.Join(gitDir, basePath) - if task.RestoreArtifacts { - fullBasePath = path.Join(dockerBuildTmpDir, basePath) - } - params := atc.TaskEnv{ + "CONTEXT": path.Join(fullBasePath, task.BuildPath), + "DOCKERFILE": path.Join(fullBasePath, task.DockerfilePath), "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", } @@ -160,51 +173,68 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { var buildStep *atc.TaskStep - buildCommand := []string{ - "docker buildx build", - fmt.Sprintf("-f %s", path.Join(fullBasePath, task.DockerfilePath)), - "--push", - "--provenance false", - fmt.Sprintf("--platform %s", strings.Join(task.Platforms, ",")), - fmt.Sprintf("--tag %s", shared.CachePath(task, "$(cat git/.git/ref)")), - } - if task.UseCache { - buildCommand = append(buildCommand, fmt.Sprintf("--tag %s", shared.CachePath(task, "buildcache"))) - buildCommand = append(buildCommand, fmt.Sprintf("--cache-from type=registry,ref=%s", shared.CachePath(task, "buildcache"))) - buildCommand = append(buildCommand, "--cache-to type=inline") - } - buildCommand = append(buildCommand, path.Join(fullBasePath, task.BuildPath)) - - buildCommandStr := strings.Join(buildCommand, ` \ - `) - buildStep = &atc.TaskStep{ - Name: "build", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": config.DockerRegistry + "halfpipe-buildx", - "tag": "latest", - "password": "((halfpipe-gcr.private_key))", - "username": "_json_key", + if !multiPlatform(task) { + buildStep = &atc.TaskStep{ + Name: "build", + Privileged: true, + Config: &atc.TaskConfig{ + Platform: "linux", + ImageResource: &atc.ImageResource{ + Type: "registry-image", + Source: atc.Source{ + "repository": "concourse/oci-build-task", + }, }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{"-c", strings.Join([]string{ - `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - fmt.Sprintf(`echo $ %s`, buildCommandStr), - buildCommandStr}, "\n"), + Params: params, + Run: atc.TaskRunConfig{ + Path: "/bin/sh", + Args: []string{ + "-c", + fmt.Sprintf("%s\n%s\n%s", "mkdir ~/.docker", "echo $DOCKER_CONFIG_JSON > ~/.docker/config.json", "build"), + }, + }, + Inputs: []atc.TaskInputConfig{ + {Name: gitDir}, + }, + Outputs: []atc.TaskOutputConfig{ + {Name: "image"}, }, }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - {Name: tagList_Dir}, + } + } else { + gitRef := fmt.Sprintf("$(cat %s)", path.Join(gitDir, ".git", "ref")) + + buildStep = &atc.TaskStep{ + Name: "build", + Privileged: true, + Config: &atc.TaskConfig{ + Platform: "linux", + ImageResource: &atc.ImageResource{ + Type: "registry-image", + Source: atc.Source{ + "repository": config.DockerRegistry + "halfpipe-buildx", + "tag": "latest", + "password": "((halfpipe-gcr.private_key))", + "username": "_json_key", + }, + }, + Params: params, + Run: atc.TaskRunConfig{ + Path: "/bin/sh", + Args: []string{"-c", strings.Join([]string{ + `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, + fmt.Sprintf(`docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t %s:%s --push --provenance=false $CONTEXT`, dockerImageWithCachePath, gitRef)}, "\n"), + }, + }, + Inputs: []atc.TaskInputConfig{ + {Name: gitDir}, + {Name: tagList_Dir}, + }, + Outputs: []atc.TaskOutputConfig{ + {Name: "image"}, + }, }, - }, + } } if task.ReadsFromArtifacts() { @@ -212,40 +242,59 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { } steps = append(steps, stepWithAttemptsAndTimeout(buildStep, task.GetAttempts(), task.GetTimeout())) - steps = append(steps, stepWithAttemptsAndTimeout(trivyTask(task, fullBasePath, basePath), task.GetAttempts(), task.GetTimeout())) + steps = append(steps, stepWithAttemptsAndTimeout(trivyTask(task, fullBasePath), task.GetAttempts(), task.GetTimeout())) - publishCommand := fmt.Sprintf(`for tag in $(cat %s) %s; do docker buildx imagetools create %s:$(cat git/.git/ref) --tag %s:$tag; done`, tagListFile, tag, dockerImageWithCachePath, image) + if !multiPlatform(task) { + putStep := &atc.PutStep{ + Name: resourceName, + Params: atc.Params{ + "image": "image/image.tar", + "additional_tags": tagListFile, + }, + NoGet: true, + } + steps = append(steps, stepWithAttemptsAndTimeout(putStep, task.GetAttempts(), task.GetTimeout())) + } else { + gitRef := fmt.Sprintf("$(cat %s)", path.Join(gitDir, ".git", "ref")) + publishCommand := fmt.Sprintf(`for tag in $(cat %s) %s; do docker buildx imagetools create %s:%s --tag %s:$tag; done`, tagListFile, tag, dockerImageWithCachePath, gitRef, image) - pushStep := &atc.TaskStep{ - Name: "publish-final-image", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": config.DockerRegistry + "halfpipe-buildx", - "tag": "latest", - "password": "((halfpipe-gcr.private_key))", - "username": "_json_key", + pushStep := &atc.TaskStep{ + Name: "publish-final-image", + Privileged: true, + Config: &atc.TaskConfig{ + Platform: "linux", + ImageResource: &atc.ImageResource{ + Type: "registry-image", + Source: atc.Source{ + "repository": config.DockerRegistry + "halfpipe-buildx", + "tag": "latest", + "password": "((halfpipe-gcr.private_key))", + "username": "_json_key", + }, }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{"-c", strings.Join([]string{ - `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - publishCommand, - }, "\n"), + Params: params, + Run: atc.TaskRunConfig{ + Path: "/bin/sh", + Args: []string{"-c", strings.Join([]string{ + `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, + publishCommand, + }, "\n"), + }, + }, + Inputs: []atc.TaskInputConfig{ + {Name: gitDir}, + {Name: tagList_Dir}, + }, + Outputs: []atc.TaskOutputConfig{ + {Name: "image"}, }, }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - {Name: tagList_Dir}, - }, - }, + } + steps = append(steps, stepWithAttemptsAndTimeout(pushStep, task.GetAttempts(), task.GetTimeout())) } - steps = append(steps, stepWithAttemptsAndTimeout(pushStep, task.GetAttempts(), task.GetTimeout())) - return steps } + +func multiPlatform(task manifest.DockerPush) bool { + return !(len(task.Platforms) == 1 && task.Platforms[0] == "linux/amd64") +} diff --git a/renderers/concourse/pipeline.go b/renderers/concourse/pipeline.go index fe54b4b3..aa2a8260 100644 --- a/renderers/concourse/pipeline.go +++ b/renderers/concourse/pipeline.go @@ -184,6 +184,22 @@ func (c Concourse) initialPlan(man manifest.Manifest, task manifest.Task, previo return steps } +func (c Concourse) dockerPushResources(tasks manifest.TaskList) (resourceConfigs atc.ResourceConfigs) { + for _, task := range tasks { + switch task := task.(type) { + case manifest.DockerPush: + if len(task.Platforms) == 1 { + resourceConfigs = append(resourceConfigs, c.dockerPushResource(task)) + } + case manifest.Parallel: + resourceConfigs = append(resourceConfigs, c.dockerPushResources(task.Tasks)...) + case manifest.Sequence: + resourceConfigs = append(resourceConfigs, c.dockerPushResources(task.Tasks)...) + } + } + + return resourceConfigs +} func (c Concourse) pipelineResources(triggers manifest.TriggerList) (resourceTypes atc.ResourceTypes, resourceConfigs atc.ResourceConfigs) { for _, trigger := range triggers { @@ -252,6 +268,8 @@ func (c Concourse) resourceConfigs(man manifest.Manifest) (resourceTypes atc.Res resourceConfigs = append(resourceConfigs, c.versionResource(man)) } + resourceConfigs = append(resourceConfigs, c.dockerPushResources(man.Tasks)...) + cfResourceTypes, cfResources := c.cfPushResources(man) resourceTypes = append(resourceTypes, cfResourceTypes...) resourceConfigs = append(resourceConfigs, cfResources...) diff --git a/renderers/concourse/pipeline_resources.go b/renderers/concourse/pipeline_resources.go index 86ec60a1..69dca598 100644 --- a/renderers/concourse/pipeline_resources.go +++ b/renderers/concourse/pipeline_resources.go @@ -230,6 +230,19 @@ func (c Concourse) deployCFResource(deployCF manifest.DeployCF, resourceName str } } +func (c Concourse) dockerPushResource(docker manifest.DockerPush) atc.ResourceConfig { + return atc.ResourceConfig{ + Name: manifest.DockerTrigger{Image: docker.Image}.GetTriggerName(), + Type: "registry-image", + Source: atc.Source{ + "username": docker.Username, + "password": docker.Password, + "repository": docker.Image, + }, + CheckEvery: &longResourceCheckInterval, + } +} + func (c Concourse) dockerTriggerResource(trigger manifest.DockerTrigger) atc.ResourceConfig { config := atc.ResourceConfig{ Name: trigger.GetTriggerName(), diff --git a/renderers/shared/docker_push.go b/renderers/shared/docker_push.go index 9f9b545e..4e10c34b 100644 --- a/renderers/shared/docker_push.go +++ b/renderers/shared/docker_push.go @@ -9,9 +9,6 @@ import ( func CachePath(task manifest.DockerPush, tag string) string { image, _ := SplitTag(task.Image) - if tag != "" && !strings.HasPrefix(tag, ":") { - tag = fmt.Sprintf(":%s", tag) - } if strings.HasPrefix(task.Image, config.DockerRegistry) { r := strings.Replace(image, config.DockerRegistry, fmt.Sprintf("%scache/", config.DockerRegistry), 1) From b0e6ac53063716e437389af1ae73fa9f658825a2 Mon Sep 17 00:00:00 2001 From: Michiel Stigter Date: Thu, 7 Sep 2023 14:13:40 +0200 Subject: [PATCH 08/40] Revert "revert to before introducing buildx builds" This reverts commit 1d007259f879f0711afb5b84763ecb0394be3e0b. --- e2e/actions/docker-push/workflowExpected.yml | 6 +- .../workflowExpected.yml | 5 +- .../workflowExpected.yml | 5 +- e2e/concourse/artifacts/pipelineExpected.yml | 73 ++++-- .../deploy-katee/pipelineExpected.yml | 71 ++++-- .../docker-push-paths/pipelineExpected.yml | 142 ++++++++---- .../pipelineExpected.yml | 73 ++++-- .../pipelineExpected.yml | 73 ++++-- .../pipelineExpected.yml | 146 ++++++++---- .../pipelineExpected.yml | 146 ++++++++---- e2e/concourse/docker-push/.halfpipe.io | 1 + .../docker-push/pipelineExpected.yml | 179 ++++++++++----- .../timer-trigger/pipelineExpected.yml | 73 ++++-- linters/docker-push.go | 7 + linters/docker-push_test.go | 6 +- linters/errors.go | 10 +- renderers/actions/docker_push.go | 5 +- renderers/actions/update.go | 11 +- renderers/concourse/docker_push.go | 215 +++++++----------- renderers/concourse/pipeline.go | 18 -- renderers/concourse/pipeline_resources.go | 13 -- renderers/shared/docker_push.go | 3 + 22 files changed, 805 insertions(+), 476 deletions(-) diff --git a/e2e/actions/docker-push/workflowExpected.yml b/e2e/actions/docker-push/workflowExpected.yml index cea79dbb..10f26533 100644 --- a/e2e/actions/docker-push/workflowExpected.yml +++ b/e2e/actions/docker-push/workflowExpected.yml @@ -293,14 +293,16 @@ jobs: RUNNING_IN_CI=true VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} - cache-from: type=registry,ref=eu.gcr.io/halfpipe-io/cache/someImage + cache-from: type=registry,ref=eu.gcr.io/halfpipe-io/cache/someImage:buildcache cache-to: type=inline context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64,linux/arm64 provenance: false push: true - tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} + tags: |- + eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} + eu.gcr.io/halfpipe-io/cache/someImage:buildcache - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy with: diff --git a/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml b/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml index 8405de0c..32f10560 100644 --- a/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml +++ b/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml @@ -35,10 +35,7 @@ jobs: submodules: recursive - name: Sync workflow with halfpipe manifest id: sync - uses: docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update - with: - args: -c "cd e2e/actions/feature-update-pipeline-and-tag; update-actions-workflow" - entrypoint: /bin/bash + run: halfpipe-update-workflow env: HALFPIPE_FILE_PATH: .halfpipe.io.yml - name: Commit and push changes to workflow diff --git a/e2e/actions/feature-update-pipeline/workflowExpected.yml b/e2e/actions/feature-update-pipeline/workflowExpected.yml index f26f9131..060404c9 100644 --- a/e2e/actions/feature-update-pipeline/workflowExpected.yml +++ b/e2e/actions/feature-update-pipeline/workflowExpected.yml @@ -35,10 +35,7 @@ jobs: submodules: recursive - name: Sync workflow with halfpipe manifest id: sync - uses: docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update - with: - args: -c "cd e2e/actions/feature-update-pipeline; update-actions-workflow" - entrypoint: /bin/bash + run: halfpipe-update-workflow env: HALFPIPE_FILE_PATH: .halfpipe.io - name: Commit and push changes to workflow diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index dea44ff7..00001462 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -436,13 +436,15 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git + - name: tagList - name: docker_build - outputs: - - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -450,17 +452,27 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/artifacts/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f docker_build/e2e/concourse/artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + docker_build/e2e/concourse/artifacts + docker buildx build \ + -f docker_build/e2e/concourse/artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + docker_build/e2e/concourse/artifacts path: /bin/sh privileged: true task: build @@ -473,8 +485,9 @@ jobs: type: docker-image inputs: - name: git - - name: image - name: docker_build + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -482,16 +495,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: docker_build/e2e/concourse/artifacts path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -747,13 +785,6 @@ resources: folder: halfpipe-team/halfpipe-e2e-artifacts json_key: ((halfpipe-artifacts.private_key)) type: gcp-resource -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image - check_every: 24h0m0s name: cf-snpaas-pe-staging source: diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml index b4d6bb32..97d7d48c 100644 --- a/e2e/concourse/deploy-katee/pipelineExpected.yml +++ b/e2e/concourse/deploy-katee/pipelineExpected.yml @@ -87,28 +87,40 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/deploy-katee DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/deploy-katee/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/deploy-katee/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ + git/e2e/concourse/deploy-katee + docker buildx build \ + -f git/e2e/concourse/deploy-katee/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ + git/e2e/concourse/deploy-katee path: /bin/sh privileged: true task: build @@ -121,7 +133,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -129,16 +142,39 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/deploy-katee path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: someimage + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --tag eu.gcr.io/halfpipe-io/halfpipe-team/someImage:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -379,10 +415,3 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-pipeline-name type: semver -- check_every: 24h0m0s - name: someimage - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-team/someImage - username: _json_key - type: registry-image diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml index bcd4794f..246527ac 100644 --- a/e2e/concourse/docker-push-paths/pipelineExpected.yml +++ b/e2e/concourse/docker-push-paths/pipelineExpected.yml @@ -34,28 +34,40 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-paths/some/build/dir DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-paths/some/build/dir + docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-paths/some/build/dir path: /bin/sh privileged: true task: build @@ -68,7 +80,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -76,16 +89,39 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-paths path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -124,28 +160,40 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-paths DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-paths + docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-paths path: /bin/sh privileged: true task: build @@ -158,7 +206,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -166,16 +215,39 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-paths path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resources: @@ -188,17 +260,3 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image -- check_every: 24h0m0s - name: halfpipe - source: - password: verysecret - repository: springerplatformengineering/halfpipe - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml index 202fc7d0..bc40614a 100644 --- a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml @@ -41,12 +41,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -54,17 +56,27 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-docker-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-docker-trigger + docker buildx build \ + -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh privileged: true task: build @@ -77,7 +89,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -85,16 +98,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resources: @@ -111,10 +149,3 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml index 281e223a..fdec0935 100644 --- a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml @@ -39,12 +39,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -52,17 +54,27 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-pipeline-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-pipeline-trigger + docker buildx build \ + -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh privileged: true task: build @@ -75,7 +87,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -83,16 +96,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resource_types: @@ -113,13 +151,6 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image - name: halfpipe-example-docker-push.docker-push source: concourse_url: ((concourse.url)) diff --git a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml index c9d96dbc..c83e5f0d 100644 --- a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml @@ -235,13 +235,15 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git + - name: tagList - name: docker_build - outputs: - - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -249,17 +251,27 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts + docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true task: build @@ -272,8 +284,9 @@ jobs: type: docker-image inputs: - name: git - - name: image - name: docker_build + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -281,16 +294,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat ../../../.git/ref) || true dir: docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image1 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --tag springerplatformengineering/image1:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -384,13 +422,15 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git + - name: tagList - name: docker_build - outputs: - - name: image params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -398,17 +438,27 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts + docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true task: build @@ -421,8 +471,9 @@ jobs: type: docker-image inputs: - name: git - - name: image - name: docker_build + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -430,16 +481,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat ../../../.git/ref) || true dir: docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image2 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --tag springerplatformengineering/image2:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resource_types: @@ -476,17 +552,3 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-docker-push-with-update-pipeline type: semver -- check_every: 24h0m0s - name: image1 - source: - password: verysecret - repository: springerplatformengineering/image1 - username: rob - type: registry-image -- check_every: 24h0m0s - name: image2 - source: - password: verysecret - repository: springerplatformengineering/image2 - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml index b0f327bd..a59af73b 100644 --- a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml @@ -87,12 +87,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -100,17 +102,27 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-update-pipeline + docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true task: build @@ -123,7 +135,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -131,16 +144,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image1 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --tag springerplatformengineering/image1:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -186,12 +224,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -199,17 +239,27 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-update-pipeline + docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true task: build @@ -222,7 +272,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -230,16 +281,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image2 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --tag springerplatformengineering/image2:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resources: @@ -260,17 +336,3 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-docker-push-with-update-pipeline type: semver -- check_every: 24h0m0s - name: image1 - source: - password: verysecret - repository: springerplatformengineering/image1 - username: rob - type: registry-image -- check_every: 24h0m0s - name: image2 - source: - password: verysecret - repository: springerplatformengineering/image2 - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push/.halfpipe.io b/e2e/concourse/docker-push/.halfpipe.io index 03657f2f..bfd0dced 100644 --- a/e2e/concourse/docker-push/.halfpipe.io +++ b/e2e/concourse/docker-push/.halfpipe.io @@ -28,6 +28,7 @@ tasks: retries: 1 ignore_vulnerabilities: true scan_timeout: 30 + use_cache: true platforms: - "linux/amd64" - "linux/arm64" diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index c6c2f4ef..cdc8c7a0 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -34,12 +34,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -47,17 +49,27 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push + docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -70,7 +82,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -78,16 +91,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly.thisismy-tag + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) thisIsMy_Tag; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe_fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -128,28 +166,40 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push + docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -163,7 +213,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -171,17 +222,40 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h - attempts: 2 - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly.thisismy-tag2 + config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) thisIsMy_Tag2; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe_fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -230,23 +304,38 @@ jobs: inputs: - name: git - name: tagList - outputs: - - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64,linux/arm64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-to type=inline \ + git/e2e/concourse/docker-push + docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64,linux/arm64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-to type=inline \ + git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -260,7 +349,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -268,7 +358,8 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) || true + trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true + dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h @@ -285,16 +376,12 @@ jobs: inputs: - name: git - name: tagList - outputs: - - name: image params: BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: @@ -317,17 +404,3 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly.thisismy-tag - source: - password: verysecret - repository: springerplatformengineering/halfpipe_fly:thisIsMy_Tag - username: rob - type: registry-image -- check_every: 24h0m0s - name: halfpipe-fly.thisismy-tag2 - source: - password: verysecret - repository: springerplatformengineering/halfpipe_fly:thisIsMy_Tag2 - username: rob - type: registry-image diff --git a/e2e/concourse/timer-trigger/pipelineExpected.yml b/e2e/concourse/timer-trigger/pipelineExpected.yml index 167c3e62..dc91d383 100644 --- a/e2e/concourse/timer-trigger/pipelineExpected.yml +++ b/e2e/concourse/timer-trigger/pipelineExpected.yml @@ -39,12 +39,14 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: BUILD_ARG_A: a BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) @@ -52,17 +54,27 @@ jobs: BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) BUILD_ARG_B: b BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/timer-trigger DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/timer-trigger/Dockerfile platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/timer-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/timer-trigger + docker buildx build \ + -f git/e2e/concourse/timer-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + git/e2e/concourse/timer-trigger path: /bin/sh privileged: true task: build @@ -75,7 +87,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -83,16 +96,41 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/timer-trigger path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + BUILD_ARG_A: a + BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) + BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) + BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) + BUILD_ARG_B: b + BUILD_ARG_RUNNING_IN_CI: "true" + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resource_types: @@ -119,10 +157,3 @@ resources: fire_immediately: true location: UTC type: halfpipe-cron-resource -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image diff --git a/linters/docker-push.go b/linters/docker-push.go index 8b188df8..c9a45361 100644 --- a/linters/docker-push.go +++ b/linters/docker-push.go @@ -2,6 +2,7 @@ package linters import ( "fmt" + "golang.org/x/exp/slices" "os" "regexp" "strings" @@ -61,5 +62,11 @@ func LintDockerPushTask(docker manifest.DockerPush, manifest manifest.Manifest, errs = append(errs, ErrDockerPushTag.AsWarning()) } + for _, platform := range docker.Platforms { + if !slices.Contains([]string{"linux/amd64", "linux/arm64"}, platform) { + errs = append(errs, ErrDockerPlatformUnknown) + } + } + return errs } diff --git a/linters/docker-push_test.go b/linters/docker-push_test.go index 30b298ab..032c610b 100644 --- a/linters/docker-push_test.go +++ b/linters/docker-push_test.go @@ -355,7 +355,7 @@ func TestMultiplePlatforms(t *testing.T) { assert.Empty(t, errors) }) - t.Run("only linux/amd64 for concourse is fine", func(t *testing.T) { + t.Run("errors when unknown platform in docker push", func(t *testing.T) { fs := afero.Afero{Fs: afero.NewMemMapFs()} fs.WriteFile("Dockerfile", []byte("FROM ubuntu"), 0777) @@ -364,12 +364,12 @@ func TestMultiplePlatforms(t *testing.T) { Username: "asd", Password: "asdf", DockerfilePath: "Dockerfile", - Platforms: []string{"linux/amd64"}, + Platforms: []string{"linux/ad64"}, } m := manifest.Manifest{Platform: "actions"} errors := LintDockerPushTask(task, m, fs) - assert.Empty(t, errors) + assertContainsError(t, errors, ErrDockerPlatformUnknown) }) } diff --git a/linters/errors.go b/linters/errors.go index 8092d33e..0493c04b 100644 --- a/linters/errors.go +++ b/linters/errors.go @@ -33,10 +33,12 @@ var ( ErrCFFromArtifact = newError("this file must be saved as an artifact in a previous task") ErrCFPrePromoteArtifact = newError("cannot have pre promote tasks with CF manifest restored from artifact") - ErrUnsupportedRegistry = newError("image must be from halfpipe registry. Please see ") - ErrDockerPushTag = newError("the field 'tag' is no longer used and is safe to delete") - ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") - ErrMultipleTriggers = newError("cannot have multiple triggers of this type") + ErrUnsupportedRegistry = newError("image must be from halfpipe registry. Please see ") + ErrDockerPushTag = newError("the field 'tag' is no longer used and is safe to delete") + + ErrDockerPlatformUnknown = newError("only linux/amd64 and/or linux/arm64 are supported") + ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") + ErrMultipleTriggers = newError("cannot have multiple triggers of this type") ErrVelaVariableMissing = newError("vela manifest variable is not specified in halfpipe manifest") ErrVelaNamespace = newError("vela namespace must start with 'katee-'") diff --git a/renderers/actions/docker_push.go b/renderers/actions/docker_push.go index 523abc23..31d16068 100644 --- a/renderers/actions/docker_push.go +++ b/renderers/actions/docker_push.go @@ -54,7 +54,7 @@ func buildImage(a *Actions, task manifest.DockerPush, buildArgs map[string]strin "context": path.Join(a.workingDir, task.BuildPath), "file": path.Join(a.workingDir, task.DockerfilePath), "push": true, - "tags": shared.CachePath(task, ":${{ env.GIT_REVISION }}"), + "tags": shared.CachePath(task, "${{ env.GIT_REVISION }}"), "build-args": MultiLine{buildArgs}, "platforms": strings.Join(task.Platforms, ","), "provenance": false, @@ -62,7 +62,8 @@ func buildImage(a *Actions, task manifest.DockerPush, buildArgs map[string]strin } if task.UseCache { - step.With["cache-from"] = fmt.Sprintf("type=registry,ref=%s", shared.CachePath(task, "")) + step.With["tags"] = fmt.Sprintf("%s\n%s", step.With["tags"], shared.CachePath(task, "buildcache")) + step.With["cache-from"] = fmt.Sprintf("type=registry,ref=%s", shared.CachePath(task, "buildcache")) step.With["cache-to"] = "type=inline" } diff --git a/renderers/actions/update.go b/renderers/actions/update.go index 2f0117d2..b36dd334 100644 --- a/renderers/actions/update.go +++ b/renderers/actions/update.go @@ -6,19 +6,10 @@ import ( ) func (a *Actions) updateSteps(task manifest.Update, man manifest.Manifest) Steps { - cdPrefix := "" - if a.workingDir != "" { - cdPrefix = fmt.Sprintf("cd %s; ", a.workingDir) - } - update := Step{ Name: "Sync workflow with halfpipe manifest", ID: "sync", - Uses: "docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update", - With: With{ - "args": fmt.Sprintf(`-c "%supdate-actions-workflow"`, cdPrefix), - "entrypoint": "/bin/bash", - }, + Run: "halfpipe-update-workflow", Env: Env{ "HALFPIPE_FILE_PATH": a.halfpipeFilePath, }, diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index f15ef681..9a01b735 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -16,16 +16,10 @@ var tagListFile = path.Join(tagList_Dir, "tagList") func (c Concourse) dockerPushJob(task manifest.DockerPush, basePath string, man manifest.Manifest) atc.JobConfig { var steps []atc.Step - resourceName := manifest.DockerTrigger{Image: task.Image}.GetTriggerName() - - fullBasePath := path.Join(gitDir, basePath) - if task.RestoreArtifacts { - fullBasePath = path.Join(dockerBuildTmpDir, basePath) - } steps = append(steps, restoreArtifacts(task)...) steps = append(steps, createTagList(task, man.FeatureToggles.UpdatePipeline())...) - steps = append(steps, buildAndPush(task, resourceName, fullBasePath)...) + steps = append(steps, buildAndPush(task, basePath)...) return atc.JobConfig{ Name: task.GetName(), @@ -104,21 +98,9 @@ func createTagList(task manifest.DockerPush, updatePipeline bool) []atc.Step { return append([]atc.Step{}, stepWithAttemptsAndTimeout(createTagList, task.GetAttempts(), task.Timeout)) } -func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { - var imageFile string - var gitRef string - if !multiPlatform(task) { - imageFile = fmt.Sprintf("--input %s", path.Join(relativePathToRepoRoot(gitDir, fullBasePath), "image/image.tar")) - } else { - imageFile = shared.CachePath(task, "") - gitRef = fmt.Sprintf(":$(cat %s)", path.Join(gitDir, ".git", "ref")) - fullBasePath = "" - } - - //exitCode := 1 - //if task.IgnoreVulnerabilities { - // exitCode = 0 - //} +func trivyTask(task manifest.DockerPush, fullBasePath string, basePath string) atc.StepConfig { + imageFile := shared.CachePath(task, "") + gitRef := fmt.Sprintf(":$(cat %s)", pathToGitRef(gitDir, basePath)) // temporary: always exit 0 until we have communicated the ignoreVulnerabilites opt-in exitCode := 0 @@ -142,9 +124,11 @@ func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { }, "\n")}, Dir: fullBasePath, }, + Params: atc.TaskEnv{ + "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", + }, Inputs: []atc.TaskInputConfig{ {Name: gitDir}, - {Name: "image"}, }, }, } @@ -156,14 +140,17 @@ func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { return step } -func buildAndPush(task manifest.DockerPush, resourceName string, fullBasePath string) []atc.Step { +func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { var steps []atc.Step image, tag := shared.SplitTag(task.Image) dockerImageWithCachePath := shared.CachePath(task, "") + fullBasePath := path.Join(gitDir, basePath) + if task.RestoreArtifacts { + fullBasePath = path.Join(dockerBuildTmpDir, basePath) + } + params := atc.TaskEnv{ - "CONTEXT": path.Join(fullBasePath, task.BuildPath), - "DOCKERFILE": path.Join(fullBasePath, task.DockerfilePath), "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", } @@ -173,68 +160,51 @@ func buildAndPush(task manifest.DockerPush, resourceName string, fullBasePath st var buildStep *atc.TaskStep - if !multiPlatform(task) { - buildStep = &atc.TaskStep{ - Name: "build", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": "concourse/oci-build-task", - }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{ - "-c", - fmt.Sprintf("%s\n%s\n%s", "mkdir ~/.docker", "echo $DOCKER_CONFIG_JSON > ~/.docker/config.json", "build"), - }, - }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - }, - Outputs: []atc.TaskOutputConfig{ - {Name: "image"}, - }, - }, - } - } else { - gitRef := fmt.Sprintf("$(cat %s)", path.Join(gitDir, ".git", "ref")) + buildCommand := []string{ + "docker buildx build", + fmt.Sprintf("-f %s", path.Join(fullBasePath, task.DockerfilePath)), + "--push", + "--provenance false", + fmt.Sprintf("--platform %s", strings.Join(task.Platforms, ",")), + fmt.Sprintf("--tag %s", shared.CachePath(task, "$(cat git/.git/ref)")), + } + if task.UseCache { + buildCommand = append(buildCommand, fmt.Sprintf("--tag %s", shared.CachePath(task, "buildcache"))) + buildCommand = append(buildCommand, fmt.Sprintf("--cache-from type=registry,ref=%s", shared.CachePath(task, "buildcache"))) + buildCommand = append(buildCommand, "--cache-to type=inline") + } + buildCommand = append(buildCommand, path.Join(fullBasePath, task.BuildPath)) - buildStep = &atc.TaskStep{ - Name: "build", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": config.DockerRegistry + "halfpipe-buildx", - "tag": "latest", - "password": "((halfpipe-gcr.private_key))", - "username": "_json_key", - }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{"-c", strings.Join([]string{ - `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - fmt.Sprintf(`docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t %s:%s --push --provenance=false $CONTEXT`, dockerImageWithCachePath, gitRef)}, "\n"), - }, - }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - {Name: tagList_Dir}, + buildCommandStr := strings.Join(buildCommand, ` \ + `) + buildStep = &atc.TaskStep{ + Name: "build", + Privileged: true, + Config: &atc.TaskConfig{ + Platform: "linux", + ImageResource: &atc.ImageResource{ + Type: "registry-image", + Source: atc.Source{ + "repository": config.DockerRegistry + "halfpipe-buildx", + "tag": "latest", + "password": "((halfpipe-gcr.private_key))", + "username": "_json_key", }, - Outputs: []atc.TaskOutputConfig{ - {Name: "image"}, + }, + Params: params, + Run: atc.TaskRunConfig{ + Path: "/bin/sh", + Args: []string{"-c", strings.Join([]string{ + `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, + fmt.Sprintf(`echo $ %s`, buildCommandStr), + buildCommandStr}, "\n"), }, }, - } + Inputs: []atc.TaskInputConfig{ + {Name: gitDir}, + {Name: tagList_Dir}, + }, + }, } if task.ReadsFromArtifacts() { @@ -242,59 +212,40 @@ func buildAndPush(task manifest.DockerPush, resourceName string, fullBasePath st } steps = append(steps, stepWithAttemptsAndTimeout(buildStep, task.GetAttempts(), task.GetTimeout())) - steps = append(steps, stepWithAttemptsAndTimeout(trivyTask(task, fullBasePath), task.GetAttempts(), task.GetTimeout())) + steps = append(steps, stepWithAttemptsAndTimeout(trivyTask(task, fullBasePath, basePath), task.GetAttempts(), task.GetTimeout())) - if !multiPlatform(task) { - putStep := &atc.PutStep{ - Name: resourceName, - Params: atc.Params{ - "image": "image/image.tar", - "additional_tags": tagListFile, - }, - NoGet: true, - } - steps = append(steps, stepWithAttemptsAndTimeout(putStep, task.GetAttempts(), task.GetTimeout())) - } else { - gitRef := fmt.Sprintf("$(cat %s)", path.Join(gitDir, ".git", "ref")) - publishCommand := fmt.Sprintf(`for tag in $(cat %s) %s; do docker buildx imagetools create %s:%s --tag %s:$tag; done`, tagListFile, tag, dockerImageWithCachePath, gitRef, image) + publishCommand := fmt.Sprintf(`for tag in $(cat %s) %s; do docker buildx imagetools create %s:$(cat git/.git/ref) --tag %s:$tag; done`, tagListFile, tag, dockerImageWithCachePath, image) - pushStep := &atc.TaskStep{ - Name: "publish-final-image", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": config.DockerRegistry + "halfpipe-buildx", - "tag": "latest", - "password": "((halfpipe-gcr.private_key))", - "username": "_json_key", - }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{"-c", strings.Join([]string{ - `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - publishCommand, - }, "\n"), - }, - }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - {Name: tagList_Dir}, + pushStep := &atc.TaskStep{ + Name: "publish-final-image", + Privileged: true, + Config: &atc.TaskConfig{ + Platform: "linux", + ImageResource: &atc.ImageResource{ + Type: "registry-image", + Source: atc.Source{ + "repository": config.DockerRegistry + "halfpipe-buildx", + "tag": "latest", + "password": "((halfpipe-gcr.private_key))", + "username": "_json_key", }, - Outputs: []atc.TaskOutputConfig{ - {Name: "image"}, + }, + Params: params, + Run: atc.TaskRunConfig{ + Path: "/bin/sh", + Args: []string{"-c", strings.Join([]string{ + `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, + publishCommand, + }, "\n"), }, }, - } - steps = append(steps, stepWithAttemptsAndTimeout(pushStep, task.GetAttempts(), task.GetTimeout())) + Inputs: []atc.TaskInputConfig{ + {Name: gitDir}, + {Name: tagList_Dir}, + }, + }, } - return steps -} + steps = append(steps, stepWithAttemptsAndTimeout(pushStep, task.GetAttempts(), task.GetTimeout())) -func multiPlatform(task manifest.DockerPush) bool { - return !(len(task.Platforms) == 1 && task.Platforms[0] == "linux/amd64") + return steps } diff --git a/renderers/concourse/pipeline.go b/renderers/concourse/pipeline.go index aa2a8260..fe54b4b3 100644 --- a/renderers/concourse/pipeline.go +++ b/renderers/concourse/pipeline.go @@ -184,22 +184,6 @@ func (c Concourse) initialPlan(man manifest.Manifest, task manifest.Task, previo return steps } -func (c Concourse) dockerPushResources(tasks manifest.TaskList) (resourceConfigs atc.ResourceConfigs) { - for _, task := range tasks { - switch task := task.(type) { - case manifest.DockerPush: - if len(task.Platforms) == 1 { - resourceConfigs = append(resourceConfigs, c.dockerPushResource(task)) - } - case manifest.Parallel: - resourceConfigs = append(resourceConfigs, c.dockerPushResources(task.Tasks)...) - case manifest.Sequence: - resourceConfigs = append(resourceConfigs, c.dockerPushResources(task.Tasks)...) - } - } - - return resourceConfigs -} func (c Concourse) pipelineResources(triggers manifest.TriggerList) (resourceTypes atc.ResourceTypes, resourceConfigs atc.ResourceConfigs) { for _, trigger := range triggers { @@ -268,8 +252,6 @@ func (c Concourse) resourceConfigs(man manifest.Manifest) (resourceTypes atc.Res resourceConfigs = append(resourceConfigs, c.versionResource(man)) } - resourceConfigs = append(resourceConfigs, c.dockerPushResources(man.Tasks)...) - cfResourceTypes, cfResources := c.cfPushResources(man) resourceTypes = append(resourceTypes, cfResourceTypes...) resourceConfigs = append(resourceConfigs, cfResources...) diff --git a/renderers/concourse/pipeline_resources.go b/renderers/concourse/pipeline_resources.go index 69dca598..86ec60a1 100644 --- a/renderers/concourse/pipeline_resources.go +++ b/renderers/concourse/pipeline_resources.go @@ -230,19 +230,6 @@ func (c Concourse) deployCFResource(deployCF manifest.DeployCF, resourceName str } } -func (c Concourse) dockerPushResource(docker manifest.DockerPush) atc.ResourceConfig { - return atc.ResourceConfig{ - Name: manifest.DockerTrigger{Image: docker.Image}.GetTriggerName(), - Type: "registry-image", - Source: atc.Source{ - "username": docker.Username, - "password": docker.Password, - "repository": docker.Image, - }, - CheckEvery: &longResourceCheckInterval, - } -} - func (c Concourse) dockerTriggerResource(trigger manifest.DockerTrigger) atc.ResourceConfig { config := atc.ResourceConfig{ Name: trigger.GetTriggerName(), diff --git a/renderers/shared/docker_push.go b/renderers/shared/docker_push.go index 4e10c34b..9f9b545e 100644 --- a/renderers/shared/docker_push.go +++ b/renderers/shared/docker_push.go @@ -9,6 +9,9 @@ import ( func CachePath(task manifest.DockerPush, tag string) string { image, _ := SplitTag(task.Image) + if tag != "" && !strings.HasPrefix(tag, ":") { + tag = fmt.Sprintf(":%s", tag) + } if strings.HasPrefix(task.Image, config.DockerRegistry) { r := strings.Replace(image, config.DockerRegistry, fmt.Sprintf("%scache/", config.DockerRegistry), 1) From 724b78c3228ad458023ec709ea8a8fc1fa03f396 Mon Sep 17 00:00:00 2001 From: Michiel Stigter Date: Thu, 7 Sep 2023 14:55:39 +0200 Subject: [PATCH 09/40] add build-args to buildx-docker push task/ --- e2e/concourse/artifacts/pipelineExpected.yml | 36 +++++--- .../deploy-katee/pipelineExpected.yml | 24 ++++-- .../docker-push-paths/pipelineExpected.yml | 48 +++++++---- .../pipelineExpected.yml | 36 +++++--- .../pipelineExpected.yml | 36 +++++--- .../pipelineExpected.yml | 72 ++++++++++------ .../pipelineExpected.yml | 72 ++++++++++------ e2e/concourse/docker-push/.halfpipe.io | 2 + .../docker-push/pipelineExpected.yml | 84 ++++++++++++------- .../timer-trigger/pipelineExpected.yml | 36 +++++--- renderers/concourse/docker_push.go | 16 +++- 11 files changed, 310 insertions(+), 152 deletions(-) diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index 00001462..8dcee0cd 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -446,13 +446,13 @@ jobs: - name: tagList - name: docker_build params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -465,6 +465,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ docker_build/e2e/concourse/artifacts docker buildx build \ -f docker_build/e2e/concourse/artifacts/Dockerfile \ @@ -472,6 +478,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ docker_build/e2e/concourse/artifacts path: /bin/sh privileged: true @@ -513,13 +525,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml index 97d7d48c..38e9f619 100644 --- a/e2e/concourse/deploy-katee/pipelineExpected.yml +++ b/e2e/concourse/deploy-katee/pipelineExpected.yml @@ -96,11 +96,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -113,6 +113,10 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/deploy-katee docker buildx build \ -f git/e2e/concourse/deploy-katee/Dockerfile \ @@ -120,6 +124,10 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/deploy-katee path: /bin/sh privileged: true @@ -160,11 +168,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml index 246527ac..bf9775b8 100644 --- a/e2e/concourse/docker-push-paths/pipelineExpected.yml +++ b/e2e/concourse/docker-push-paths/pipelineExpected.yml @@ -43,11 +43,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -60,6 +60,10 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-paths/some/build/dir docker buildx build \ -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ @@ -67,6 +71,10 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-paths/some/build/dir path: /bin/sh privileged: true @@ -107,11 +115,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -169,11 +177,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -186,6 +194,10 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-paths docker buildx build \ -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ @@ -193,6 +205,10 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-paths path: /bin/sh privileged: true @@ -233,11 +249,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml index bc40614a..9e7795b4 100644 --- a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml @@ -50,13 +50,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -69,6 +69,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-with-docker-trigger docker buildx build \ -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ @@ -76,6 +82,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh privileged: true @@ -116,13 +128,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml index fdec0935..ff3bdafa 100644 --- a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml @@ -48,13 +48,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -67,6 +67,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-with-pipeline-trigger docker buildx build \ -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ @@ -74,6 +80,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh privileged: true @@ -114,13 +126,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml index c83e5f0d..611c4277 100644 --- a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml @@ -245,13 +245,13 @@ jobs: - name: tagList - name: docker_build params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -264,6 +264,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ docker_build/e2e/concourse/docker-push-with-restore-artifacts docker buildx build \ -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ @@ -271,6 +277,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true @@ -312,13 +324,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -432,13 +444,13 @@ jobs: - name: tagList - name: docker_build params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -451,6 +463,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ docker_build/e2e/concourse/docker-push-with-restore-artifacts docker buildx build \ -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ @@ -458,6 +476,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true @@ -499,13 +523,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml index a59af73b..803fc065 100644 --- a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml @@ -96,13 +96,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -115,6 +115,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-with-update-pipeline docker buildx build \ -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ @@ -122,6 +128,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true @@ -162,13 +174,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -233,13 +245,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -252,6 +264,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-with-update-pipeline docker buildx build \ -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ @@ -259,6 +277,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true @@ -299,13 +323,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push/.halfpipe.io b/e2e/concourse/docker-push/.halfpipe.io index bfd0dced..5fb8c548 100644 --- a/e2e/concourse/docker-push/.halfpipe.io +++ b/e2e/concourse/docker-push/.halfpipe.io @@ -15,6 +15,8 @@ tasks: vars: A: a B: b + + - type: docker-push username: rob password: verysecret diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index cdc8c7a0..f66dce9c 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -43,13 +43,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -62,6 +62,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push docker buildx build \ -f git/e2e/concourse/docker-push/Dockerfile \ @@ -69,6 +75,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push path: /bin/sh privileged: true @@ -109,13 +121,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -175,11 +187,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -192,6 +204,10 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push docker buildx build \ -f git/e2e/concourse/docker-push/Dockerfile \ @@ -199,6 +215,10 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/docker-push path: /bin/sh privileged: true @@ -241,11 +261,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -305,11 +325,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -322,6 +342,10 @@ jobs: --provenance false \ --platform linux/amd64,linux/arm64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ --cache-to type=inline \ @@ -332,6 +356,10 @@ jobs: --provenance false \ --platform linux/amd64,linux/arm64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ --cache-to type=inline \ @@ -377,11 +405,11 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/timer-trigger/pipelineExpected.yml b/e2e/concourse/timer-trigger/pipelineExpected.yml index dc91d383..e0073985 100644 --- a/e2e/concourse/timer-trigger/pipelineExpected.yml +++ b/e2e/concourse/timer-trigger/pipelineExpected.yml @@ -48,13 +48,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: @@ -67,6 +67,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/timer-trigger docker buildx build \ -f git/e2e/concourse/timer-trigger/Dockerfile \ @@ -74,6 +80,12 @@ jobs: --provenance false \ --platform linux/amd64 \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ git/e2e/concourse/timer-trigger path: /bin/sh privileged: true @@ -114,13 +126,13 @@ jobs: - name: git - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index 9a01b735..608e9a6b 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -6,6 +6,7 @@ import ( "github.com/springernature/halfpipe/config" "github.com/springernature/halfpipe/manifest" "github.com/springernature/halfpipe/renderers/shared" + "golang.org/x/exp/slices" "path" "strings" ) @@ -154,10 +155,6 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", } - for k, v := range convertVars(task.Vars) { - params[fmt.Sprintf("BUILD_ARG_%s", k)] = fmt.Sprintf("%s", v) - } - var buildStep *atc.TaskStep buildCommand := []string{ @@ -168,11 +165,22 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { fmt.Sprintf("--platform %s", strings.Join(task.Platforms, ",")), fmt.Sprintf("--tag %s", shared.CachePath(task, "$(cat git/.git/ref)")), } + + buildArgs := []string{} + for k, v := range convertVars(task.Vars) { + params[k] = v.(string) + buildArgs = append(buildArgs, fmt.Sprintf("--build-arg %s", k)) + } + + slices.Sort(buildArgs) + + buildCommand = append(buildCommand, buildArgs...) if task.UseCache { buildCommand = append(buildCommand, fmt.Sprintf("--tag %s", shared.CachePath(task, "buildcache"))) buildCommand = append(buildCommand, fmt.Sprintf("--cache-from type=registry,ref=%s", shared.CachePath(task, "buildcache"))) buildCommand = append(buildCommand, "--cache-to type=inline") } + buildCommand = append(buildCommand, path.Join(fullBasePath, task.BuildPath)) buildCommandStr := strings.Join(buildCommand, ` \ From 9aa837070f2138927a16c090da45fca11c47487b Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Fri, 8 Sep 2023 11:36:54 +0100 Subject: [PATCH 10/40] add artifactory secrets to docker builds --- defaults/tasks_env_defaulter.go | 2 ++ defaults/tasks_env_defaulter_test.go | 8 +++++++- e2e/concourse/artifacts/pipelineExpected.yml | 6 ++++++ .../deploy-katee/pipelineExpected.yml | 6 ++++++ .../docker-push-paths/pipelineExpected.yml | 12 ++++++++++++ .../pipelineExpected.yml | 6 ++++++ .../pipelineExpected.yml | 6 ++++++ .../pipelineExpected.yml | 12 ++++++++++++ .../pipelineExpected.yml | 12 ++++++++++++ e2e/concourse/docker-push/.halfpipe.io | 2 -- e2e/concourse/docker-push/pipelineExpected.yml | 18 ++++++++++++++++++ .../timer-trigger/pipelineExpected.yml | 6 ++++++ manifest/docker_push.go | 1 + renderers/concourse/docker_push.go | 11 +++++++++-- 14 files changed, 103 insertions(+), 5 deletions(-) diff --git a/defaults/tasks_env_defaulter.go b/defaults/tasks_env_defaulter.go index 9380921a..9609d274 100644 --- a/defaults/tasks_env_defaulter.go +++ b/defaults/tasks_env_defaulter.go @@ -43,6 +43,8 @@ func (t tasksEnvVarsDefaulter) Apply(original manifest.TaskList, defaults Defaul tt = task case manifest.DockerPush: task.Vars = t.addDefaultsToVars(task.Vars, defaults) + task.Secrets = t.addDefaultsToVars(task.Secrets, defaults) + delete(task.Secrets, "RUNNING_IN_CI") tt = task case manifest.DeployCF: task.PrePromote = t.Apply(task.PrePromote, defaults) diff --git a/defaults/tasks_env_defaulter_test.go b/defaults/tasks_env_defaulter_test.go index ae4583cb..e216a8aa 100644 --- a/defaults/tasks_env_defaulter_test.go +++ b/defaults/tasks_env_defaulter_test.go @@ -14,6 +14,12 @@ func TestSetsCorrectEnvVarsToEmptyVars(t *testing.T) { "RUNNING_IN_CI": "true", } + expectedSecrets := map[string]string{ + "ARTIFACTORY_URL": Concourse.Artifactory.URL, + "ARTIFACTORY_USERNAME": Concourse.Artifactory.Username, + "ARTIFACTORY_PASSWORD": Concourse.Artifactory.Password, + } + input := manifest.TaskList{ manifest.Update{}, manifest.Run{}, @@ -63,7 +69,7 @@ func TestSetsCorrectEnvVarsToEmptyVars(t *testing.T) { }, }, }, - manifest.DockerPush{Vars: expectedVars}, + manifest.DockerPush{Vars: expectedVars, Secrets: expectedSecrets}, manifest.Parallel{ Tasks: manifest.TaskList{ manifest.Sequence{ diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index 8dcee0cd..9509cee1 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -471,6 +471,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ docker_build/e2e/concourse/artifacts docker buildx build \ -f docker_build/e2e/concourse/artifacts/Dockerfile \ @@ -484,6 +487,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ docker_build/e2e/concourse/artifacts path: /bin/sh privileged: true diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml index 38e9f619..2f5178b4 100644 --- a/e2e/concourse/deploy-katee/pipelineExpected.yml +++ b/e2e/concourse/deploy-katee/pipelineExpected.yml @@ -117,6 +117,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/deploy-katee docker buildx build \ -f git/e2e/concourse/deploy-katee/Dockerfile \ @@ -128,6 +131,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/deploy-katee path: /bin/sh privileged: true diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml index bf9775b8..886dee73 100644 --- a/e2e/concourse/docker-push-paths/pipelineExpected.yml +++ b/e2e/concourse/docker-push-paths/pipelineExpected.yml @@ -64,6 +64,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-paths/some/build/dir docker buildx build \ -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ @@ -75,6 +78,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-paths/some/build/dir path: /bin/sh privileged: true @@ -198,6 +204,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-paths docker buildx build \ -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ @@ -209,6 +218,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-paths path: /bin/sh privileged: true diff --git a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml index 9e7795b4..75d575bd 100644 --- a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml @@ -75,6 +75,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-with-docker-trigger docker buildx build \ -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ @@ -88,6 +91,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh privileged: true diff --git a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml index ff3bdafa..6e11de14 100644 --- a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml @@ -73,6 +73,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-with-pipeline-trigger docker buildx build \ -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ @@ -86,6 +89,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh privileged: true diff --git a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml index 611c4277..040393c9 100644 --- a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml @@ -270,6 +270,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ docker_build/e2e/concourse/docker-push-with-restore-artifacts docker buildx build \ -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ @@ -283,6 +286,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true @@ -469,6 +475,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ docker_build/e2e/concourse/docker-push-with-restore-artifacts docker buildx build \ -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ @@ -482,6 +491,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true diff --git a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml index 803fc065..e73a1afb 100644 --- a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml @@ -121,6 +121,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-with-update-pipeline docker buildx build \ -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ @@ -134,6 +137,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true @@ -270,6 +276,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-with-update-pipeline docker buildx build \ -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ @@ -283,6 +292,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true diff --git a/e2e/concourse/docker-push/.halfpipe.io b/e2e/concourse/docker-push/.halfpipe.io index 5fb8c548..4ee1cf6d 100644 --- a/e2e/concourse/docker-push/.halfpipe.io +++ b/e2e/concourse/docker-push/.halfpipe.io @@ -16,7 +16,6 @@ tasks: A: a B: b - - type: docker-push username: rob password: verysecret @@ -34,4 +33,3 @@ tasks: platforms: - "linux/amd64" - "linux/arm64" - diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index f66dce9c..655e98da 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -68,6 +68,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push docker buildx build \ -f git/e2e/concourse/docker-push/Dockerfile \ @@ -81,6 +84,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push path: /bin/sh privileged: true @@ -208,6 +214,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push docker buildx build \ -f git/e2e/concourse/docker-push/Dockerfile \ @@ -219,6 +228,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/docker-push path: /bin/sh privileged: true @@ -346,6 +358,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ --cache-to type=inline \ @@ -360,6 +375,9 @@ jobs: --build-arg ARTIFACTORY_URL \ --build-arg ARTIFACTORY_USERNAME \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ --cache-to type=inline \ diff --git a/e2e/concourse/timer-trigger/pipelineExpected.yml b/e2e/concourse/timer-trigger/pipelineExpected.yml index e0073985..75e86b26 100644 --- a/e2e/concourse/timer-trigger/pipelineExpected.yml +++ b/e2e/concourse/timer-trigger/pipelineExpected.yml @@ -73,6 +73,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/timer-trigger docker buildx build \ -f git/e2e/concourse/timer-trigger/Dockerfile \ @@ -86,6 +89,9 @@ jobs: --build-arg ARTIFACTORY_USERNAME \ --build-arg B \ --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ git/e2e/concourse/timer-trigger path: /bin/sh privileged: true diff --git a/manifest/docker_push.go b/manifest/docker_push.go index d9274138..c6af13e0 100644 --- a/manifest/docker_push.go +++ b/manifest/docker_push.go @@ -10,6 +10,7 @@ type DockerPush struct { IgnoreVulnerabilities bool `json:"ignore_vulnerabilities,omitempty" yaml:"ignore_vulnerabilities,omitempty"` ScanTimeout int `json:"scan_timeout,omitempty" yaml:"scan_timeout,omitempty"` Vars Vars `yaml:"vars,omitempty" secretAllowed:"true"` + Secrets Vars `yaml:"secrets,omitempty" secretAllowed:"true"` RestoreArtifacts bool `json:"restore_artifacts" yaml:"restore_artifacts,omitempty"` Retries int `yaml:"retries,omitempty"` NotifyOnSuccess bool `json:"notify_on_success,omitempty" yaml:"notify_on_success,omitempty"` diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index 608e9a6b..86629d9f 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -171,10 +171,17 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { params[k] = v.(string) buildArgs = append(buildArgs, fmt.Sprintf("--build-arg %s", k)) } - slices.Sort(buildArgs) - buildCommand = append(buildCommand, buildArgs...) + + secrets := []string{} + for k, v := range convertVars(task.Secrets) { + params[k] = v.(string) + secrets = append(secrets, fmt.Sprintf("--secret id=%s", k)) + } + slices.Sort(secrets) + buildCommand = append(buildCommand, secrets...) + if task.UseCache { buildCommand = append(buildCommand, fmt.Sprintf("--tag %s", shared.CachePath(task, "buildcache"))) buildCommand = append(buildCommand, fmt.Sprintf("--cache-from type=registry,ref=%s", shared.CachePath(task, "buildcache"))) From 4b232142cd2e3bf4395fecea73f2c85715833f6e Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Fri, 8 Sep 2023 11:48:01 +0100 Subject: [PATCH 11/40] support new `secrets` field in docker-push (concourse) --- e2e/concourse/artifacts/pipelineExpected.yml | 6 ------ .../deploy-katee/pipelineExpected.yml | 4 ---- .../docker-push-paths/pipelineExpected.yml | 8 -------- .../pipelineExpected.yml | 6 ------ .../pipelineExpected.yml | 6 ------ .../pipelineExpected.yml | 12 ----------- .../pipelineExpected.yml | 12 ----------- e2e/concourse/docker-push/.halfpipe.io | 3 +++ .../docker-push/pipelineExpected.yml | 20 ++++++------------- .../timer-trigger/pipelineExpected.yml | 6 ------ renderers/concourse/docker_push.go | 5 ++++- 11 files changed, 13 insertions(+), 75 deletions(-) diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index 9509cee1..49e43d77 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -531,13 +531,7 @@ jobs: - name: git - name: tagList params: - A: a - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) - B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml index 2f5178b4..45d59b8a 100644 --- a/e2e/concourse/deploy-katee/pipelineExpected.yml +++ b/e2e/concourse/deploy-katee/pipelineExpected.yml @@ -174,11 +174,7 @@ jobs: - name: git - name: tagList params: - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml index 886dee73..59a77f02 100644 --- a/e2e/concourse/docker-push-paths/pipelineExpected.yml +++ b/e2e/concourse/docker-push-paths/pipelineExpected.yml @@ -121,11 +121,7 @@ jobs: - name: git - name: tagList params: - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: @@ -261,11 +257,7 @@ jobs: - name: git - name: tagList params: - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml index 75d575bd..4b3f62c4 100644 --- a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml @@ -134,13 +134,7 @@ jobs: - name: git - name: tagList params: - A: a - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) - B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml index 6e11de14..22278140 100644 --- a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml @@ -132,13 +132,7 @@ jobs: - name: git - name: tagList params: - A: a - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) - B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml index 040393c9..b0d47756 100644 --- a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml @@ -330,13 +330,7 @@ jobs: - name: git - name: tagList params: - A: a - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) - B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: @@ -535,13 +529,7 @@ jobs: - name: git - name: tagList params: - A: a - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) - B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml index e73a1afb..d7bb9783 100644 --- a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml @@ -180,13 +180,7 @@ jobs: - name: git - name: tagList params: - A: a - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) - B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: @@ -335,13 +329,7 @@ jobs: - name: git - name: tagList params: - A: a - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) - B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/docker-push/.halfpipe.io b/e2e/concourse/docker-push/.halfpipe.io index 4ee1cf6d..9782a497 100644 --- a/e2e/concourse/docker-push/.halfpipe.io +++ b/e2e/concourse/docker-push/.halfpipe.io @@ -15,6 +15,9 @@ tasks: vars: A: a B: b + secrets: + C: c + D: d - type: docker-push username: rob diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index 655e98da..505aa798 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -48,6 +48,8 @@ jobs: ARTIFACTORY_URL: ((artifactory.url)) ARTIFACTORY_USERNAME: ((artifactory.username)) B: b + C: c + D: d DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) RUNNING_IN_CI: "true" platform: linux @@ -71,6 +73,8 @@ jobs: --secret id=ARTIFACTORY_PASSWORD \ --secret id=ARTIFACTORY_URL \ --secret id=ARTIFACTORY_USERNAME \ + --secret id=C \ + --secret id=D \ git/e2e/concourse/docker-push docker buildx build \ -f git/e2e/concourse/docker-push/Dockerfile \ @@ -87,6 +91,8 @@ jobs: --secret id=ARTIFACTORY_PASSWORD \ --secret id=ARTIFACTORY_URL \ --secret id=ARTIFACTORY_USERNAME \ + --secret id=C \ + --secret id=D \ git/e2e/concourse/docker-push path: /bin/sh privileged: true @@ -127,13 +133,7 @@ jobs: - name: git - name: tagList params: - A: a - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) - B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: @@ -273,11 +273,7 @@ jobs: - name: git - name: tagList params: - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: @@ -423,11 +419,7 @@ jobs: - name: git - name: tagList params: - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/e2e/concourse/timer-trigger/pipelineExpected.yml b/e2e/concourse/timer-trigger/pipelineExpected.yml index 75e86b26..9c5ccd25 100644 --- a/e2e/concourse/timer-trigger/pipelineExpected.yml +++ b/e2e/concourse/timer-trigger/pipelineExpected.yml @@ -132,13 +132,7 @@ jobs: - name: git - name: tagList params: - A: a - ARTIFACTORY_PASSWORD: ((artifactory.password)) - ARTIFACTORY_URL: ((artifactory.url)) - ARTIFACTORY_USERNAME: ((artifactory.username)) - B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - RUNNING_IN_CI: "true" platform: linux run: args: diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index 86629d9f..332f2187 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -178,6 +178,7 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { for k, v := range convertVars(task.Secrets) { params[k] = v.(string) secrets = append(secrets, fmt.Sprintf("--secret id=%s", k)) + } slices.Sort(secrets) buildCommand = append(buildCommand, secrets...) @@ -245,7 +246,9 @@ func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { "username": "_json_key", }, }, - Params: params, + Params: atc.TaskEnv{ + "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", + }, Run: atc.TaskRunConfig{ Path: "/bin/sh", Args: []string{"-c", strings.Join([]string{ From c3da1471f616e73bc7f416f65731822379392ce7 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Fri, 8 Sep 2023 12:10:59 +0100 Subject: [PATCH 12/40] add lint warning when secret used in docker-push vars --- linters/docker-push.go | 8 ++++- linters/docker-push_test.go | 68 +++++++++++++++++++++++-------------- linters/errors.go | 7 ++-- linters/task.go | 4 +-- linters/task_test.go | 8 ++--- 5 files changed, 59 insertions(+), 36 deletions(-) diff --git a/linters/docker-push.go b/linters/docker-push.go index c9a45361..2c3b226c 100644 --- a/linters/docker-push.go +++ b/linters/docker-push.go @@ -11,7 +11,7 @@ import ( "github.com/springernature/halfpipe/manifest" ) -func LintDockerPushTask(docker manifest.DockerPush, manifest manifest.Manifest, fs afero.Afero) (errs []error) { +func LintDockerPushTask(docker manifest.DockerPush, fs afero.Afero) (errs []error) { if docker.Image == "" { errs = append(errs, NewErrMissingField("image")) } else { @@ -68,5 +68,11 @@ func LintDockerPushTask(docker manifest.DockerPush, manifest manifest.Manifest, } } + for k, v := range docker.Vars { + if strings.HasPrefix(v, "((") && strings.HasSuffix(v, "))") && !strings.HasPrefix(k, "ARTIFACTORY_") { + errs = append(errs, ErrDockerVarSecret.WithValue(k).AsWarning()) + } + } + return errs } diff --git a/linters/docker-push_test.go b/linters/docker-push_test.go index 032c610b..013f060f 100644 --- a/linters/docker-push_test.go +++ b/linters/docker-push_test.go @@ -13,7 +13,7 @@ var emptyManifest = manifest.Manifest{} func TestDockerPushTaskWithEmptyTask(t *testing.T) { fs := afero.Afero{Fs: afero.NewMemMapFs()} - errors := LintDockerPushTask(manifest.DockerPush{}, emptyManifest, fs) + errors := LintDockerPushTask(manifest.DockerPush{}, fs) assertContainsError(t, errors, NewErrMissingField("image")) } @@ -25,7 +25,7 @@ func TestDockerPushTaskWithBadRepo(t *testing.T) { Image: "asd", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assertContainsError(t, errors, ErrInvalidField.WithValue("image")) } @@ -40,7 +40,7 @@ func TestDockerPushTaskWithoutTeamDirectoryInHalfpipeRepo(t *testing.T) { DockerfilePath: "Dockerfile", } - errs := LintDockerPushTask(task, emptyManifest, fs) + errs := LintDockerPushTask(task, fs) assertContainsError(t, errs, ErrInvalidField.WithValue("image")) } @@ -55,7 +55,7 @@ func TestDockerPushTaskWithTeamDirectoryInHalfpipeRepo(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) } @@ -70,7 +70,7 @@ func TestDockerPushTaskWithoutTeamDirectoryInGCRRepo(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) } @@ -85,7 +85,7 @@ func TestDockerPushTaskWhenDockerfileIsMissing(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assertContainsError(t, errors, ErrFileNotFound) }) @@ -100,7 +100,7 @@ func TestDockerPushTaskWhenDockerfileIsMissing(t *testing.T) { RestoreArtifacts: true, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assertNotContainsError(t, errors, ErrFileNotFound) }) } @@ -122,7 +122,7 @@ func TestDockerPushTaskWithCorrectData(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -141,7 +141,7 @@ func TestDockerPushTaskWithCorrectData(t *testing.T) { DockerfilePath: "dockerfile/Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -160,7 +160,7 @@ func TestDockerPushTaskWithCorrectData(t *testing.T) { DockerfilePath: "../dockerfile/Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -182,7 +182,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { BuildPath: "buildPathDoesntExist", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 1) assertContainsError(t, errors, ErrInvalidField.WithValue("build_path")) }) @@ -206,7 +206,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { BuildPath: buildPath, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 1) assertContainsError(t, errors, ErrInvalidField.WithValue("build_path")) }) @@ -230,7 +230,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { BuildPath: buildPath, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -253,7 +253,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { BuildPath: buildPath, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -269,7 +269,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { RestoreArtifacts: true, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Empty(t, errors) }) @@ -291,15 +291,15 @@ func TestDockerPushRetries(t *testing.T) { } task.Retries = -1 - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assertContainsError(t, errors, ErrInvalidField.WithValue("retries")) task.Retries = 6 - errors = LintDockerPushTask(task, emptyManifest, fs) + errors = LintDockerPushTask(task, fs) assertContainsError(t, errors, ErrInvalidField.WithValue("retries")) task.Retries = 4 - errors = LintDockerPushTask(task, emptyManifest, fs) + errors = LintDockerPushTask(task, fs) assert.Len(t, errors, 0) } @@ -315,7 +315,7 @@ func TestDockerPushTag(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Empty(t, errors) }) @@ -331,7 +331,7 @@ func TestDockerPushTag(t *testing.T) { Tag: "yolo", } - errs := LintDockerPushTask(task, emptyManifest, fs) + errs := LintDockerPushTask(task, fs) assertContainsError(t, errs, ErrDockerPushTag) }) } @@ -349,9 +349,7 @@ func TestMultiplePlatforms(t *testing.T) { Platforms: []string{"linux/arm64", "linux/amd64"}, } - m := manifest.Manifest{Platform: "actions"} - - errors := LintDockerPushTask(task, m, fs) + errors := LintDockerPushTask(task, fs) assert.Empty(t, errors) }) @@ -367,9 +365,27 @@ func TestMultiplePlatforms(t *testing.T) { Platforms: []string{"linux/ad64"}, } - m := manifest.Manifest{Platform: "actions"} - - errors := LintDockerPushTask(task, m, fs) + errors := LintDockerPushTask(task, fs) assertContainsError(t, errors, ErrDockerPlatformUnknown) }) } + +func TestSecrets(t *testing.T) { + t.Run("no secrets in vars please", func(t *testing.T) { + fs := afero.Afero{Fs: afero.NewMemMapFs()} + fs.WriteFile("Dockerfile", []byte("FROM ubuntu"), 0777) + + task := manifest.DockerPush{ + Image: "asd/asd", + Username: "asd", + Password: "asdf", + DockerfilePath: "Dockerfile", + Platforms: []string{"linux/arm64", "linux/amd64"}, + Vars: manifest.Vars{"var1": "((a.secret))"}, + } + + errors := LintDockerPushTask(task, fs) + assertContainsError(t, errors, ErrDockerVarSecret.WithValue("var1").AsWarning()) + }) + +} diff --git a/linters/errors.go b/linters/errors.go index 0493c04b..035c092b 100644 --- a/linters/errors.go +++ b/linters/errors.go @@ -38,7 +38,9 @@ var ( ErrDockerPlatformUnknown = newError("only linux/amd64 and/or linux/arm64 are supported") ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") - ErrMultipleTriggers = newError("cannot have multiple triggers of this type") + ErrDockerVarSecret = newError("docker build var uses secret") + + ErrMultipleTriggers = newError("cannot have multiple triggers of this type") ErrVelaVariableMissing = newError("vela manifest variable is not specified in halfpipe manifest") ErrVelaNamespace = newError("vela namespace must start with 'katee-'") @@ -75,8 +77,7 @@ func (e Error) Error() string { } func (e Error) AsWarning() Error { - e.level = "warning" - return e + return Error{err: e.err, level: "warning", value: e.value} } func (e Error) IsWarning() bool { diff --git a/linters/task.go b/linters/task.go index 14ce294a..757d1051 100644 --- a/linters/task.go +++ b/linters/task.go @@ -17,7 +17,7 @@ type taskLinter struct { lintDeployCFTask func(task manifest.DeployCF, readCfManifest cf.ManifestReader, fs afero.Afero) []error lintDeployKateeTask func(task manifest.DeployKatee, man manifest.Manifest, fs afero.Afero) []error LintPrePromoteTask func(task manifest.Task) []error - lintDockerPushTask func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) []error + lintDockerPushTask func(task manifest.DockerPush, fs afero.Afero) []error lintDockerComposeTask func(task manifest.DockerCompose, fs afero.Afero) []error lintConsumerIntegrationTestTask func(task manifest.ConsumerIntegrationTest, providerHostRequired bool) []error lintDeployMLZipTask func(task manifest.DeployMLZip) []error @@ -97,7 +97,7 @@ func (linter taskLinter) lintTasks(listName string, ts []manifest.Task, man mani case manifest.DeployKatee: errs = linter.lintDeployKateeTask(task, man, linter.Fs) case manifest.DockerPush: - errs = linter.lintDockerPushTask(task, man, linter.Fs) + errs = linter.lintDockerPushTask(task, linter.Fs) case manifest.DockerCompose: errs = linter.lintDockerComposeTask(task, linter.Fs) case manifest.ConsumerIntegrationTest: diff --git a/linters/task_test.go b/linters/task_test.go index 454e69d0..5aa0aa5f 100644 --- a/linters/task_test.go +++ b/linters/task_test.go @@ -113,7 +113,7 @@ func TestCallsOutToTheLintersCorrectly(t *testing.T) { calledLintPrePromoteTasksNum++ return }, - lintDockerPushTask: func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) (errs []error) { + lintDockerPushTask: func(task manifest.DockerPush, fs afero.Afero) (errs []error) { calledLintDockerPushTask = true calledLintDockerPushTaskNum++ return @@ -232,7 +232,7 @@ func TestMergesTheErrorsAndWarningsCorrectlyWithPrePromote(t *testing.T) { LintPrePromoteTask: func(tasks manifest.Task) (errs []error) { return []error{prePromoteErr, prePromoteWarn} }, - lintDockerPushTask: func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) (errs []error) { + lintDockerPushTask: func(task manifest.DockerPush, fs afero.Afero) (errs []error) { return []error{dockerPushErr, dockerPushWarn} }, lintDeployMLZipTask: func(task manifest.DeployMLZip) (errs []error) { @@ -323,7 +323,7 @@ func TestMergesTheErrorsAndWarningsCorrectlyWithParallel(t *testing.T) { LintPrePromoteTask: func(tasks manifest.Task) (errs []error) { return []error{prePromoteErr, prePromoteWarn} }, - lintDockerPushTask: func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) (errs []error) { + lintDockerPushTask: func(task manifest.DockerPush, fs afero.Afero) (errs []error) { return []error{dockerPushErr, dockerPushWarn} }, lintDeployMLZipTask: func(task manifest.DeployMLZip) (errs []error) { @@ -557,7 +557,7 @@ func TestLintTimeout(t *testing.T) { return }, LintPrePromoteTask: func(task manifest.Task) (errs []error) { return }, - lintDockerPushTask: func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) (errs []error) { + lintDockerPushTask: func(task manifest.DockerPush, fs afero.Afero) (errs []error) { return }, lintDockerComposeTask: func(task manifest.DockerCompose, fs afero.Afero) (errs []error) { From c0cc2b46cadf5b0d28247b3bb350cb7ea1d9685e Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Fri, 8 Sep 2023 14:21:16 +0100 Subject: [PATCH 13/40] support docker-push secrets for actions --- .github/workflows/for-dependabot-to-check.yml | 4 + defaults/task_docker_push.go | 15 +++ defaults/task_docker_push_test.go | 23 ++--- defaults/tasks_env_defaulter.go | 2 - defaults/tasks_env_defaulter_test.go | 8 +- e2e/actions/deploy-katee/workflowExpected.yml | 4 + e2e/actions/docker-push/.halfpipe.io | 13 ++- e2e/actions/docker-push/workflowExpected.yml | 92 +++++++++++++++++++ linters/errors.go | 2 +- renderers/actions/docker_push.go | 1 + 10 files changed, 134 insertions(+), 30 deletions(-) diff --git a/.github/workflows/for-dependabot-to-check.yml b/.github/workflows/for-dependabot-to-check.yml index 4079417b..01369a39 100644 --- a/.github/workflows/for-dependabot-to-check.yml +++ b/.github/workflows/for-dependabot-to-check.yml @@ -165,6 +165,10 @@ jobs: platforms: linux/amd64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/blah:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy diff --git a/defaults/task_docker_push.go b/defaults/task_docker_push.go index 3b44bd7b..30ccb50a 100644 --- a/defaults/task_docker_push.go +++ b/defaults/task_docker_push.go @@ -27,5 +27,20 @@ func dockerPushDefaulter(original manifest.DockerPush, man manifest.Manifest, de updated.Platforms = []string{"linux/amd64"} } + if updated.Secrets == nil { + updated.Secrets = make(manifest.Vars) + } + + if man.Platform.IsConcourse() { + updated.Secrets["ARTIFACTORY_URL"] = defaults.Artifactory.URL + updated.Secrets["ARTIFACTORY_USERNAME"] = defaults.Artifactory.Username + updated.Secrets["ARTIFACTORY_PASSWORD"] = defaults.Artifactory.Password + } + if man.Platform.IsActions() { + updated.Secrets["ARTIFACTORY_URL"] = "${{ secrets.EE_ARTIFACTORY_URL }}" + updated.Secrets["ARTIFACTORY_USERNAME"] = "${{ secrets.EE_ARTIFACTORY_USERNAME }}" + updated.Secrets["ARTIFACTORY_PASSWORD"] = "${{ secrets.EE_ARTIFACTORY_PASSWORD }}" + } + return updated } diff --git a/defaults/task_docker_push_test.go b/defaults/task_docker_push_test.go index 0d621548..d71dbf26 100644 --- a/defaults/task_docker_push_test.go +++ b/defaults/task_docker_push_test.go @@ -8,27 +8,16 @@ import ( "testing" ) -func TestWhenPublicImage(t *testing.T) { +func TestWhenPublicImageDontSetUsernameAndPassword(t *testing.T) { task := manifest.DockerPush{Image: "asdf", DockerfilePath: "something", ScanTimeout: 15} - - defaultedTask := manifest.DockerPush{Image: "asdf", DockerfilePath: "something", ScanTimeout: 15, Platforms: []string{"linux/amd64"}} - - assert.Equal(t, defaultedTask, dockerPushDefaulter(task, manifest.Manifest{}, Concourse)) + assert.Empty(t, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Username) + assert.Empty(t, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Password) } -func TestPrivateImage(t *testing.T) { +func TestPrivateImageSetsUsernameAndPassword(t *testing.T) { task := manifest.DockerPush{Image: path.Join(config.DockerRegistry, "push-me"), DockerfilePath: "something"} - - expected := manifest.DockerPush{ - Image: path.Join(config.DockerRegistry, "push-me"), - DockerfilePath: "something", - Username: Concourse.Docker.Username, - Password: Concourse.Docker.Password, - ScanTimeout: 15, - Platforms: []string{"linux/amd64"}, - } - - assert.Equal(t, expected, dockerPushDefaulter(task, manifest.Manifest{}, Concourse)) + assert.Equal(t, Concourse.Docker.Username, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Username) + assert.Equal(t, Concourse.Docker.Password, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Password) } func TestSetsTheDockerFilePath(t *testing.T) { diff --git a/defaults/tasks_env_defaulter.go b/defaults/tasks_env_defaulter.go index 9609d274..9380921a 100644 --- a/defaults/tasks_env_defaulter.go +++ b/defaults/tasks_env_defaulter.go @@ -43,8 +43,6 @@ func (t tasksEnvVarsDefaulter) Apply(original manifest.TaskList, defaults Defaul tt = task case manifest.DockerPush: task.Vars = t.addDefaultsToVars(task.Vars, defaults) - task.Secrets = t.addDefaultsToVars(task.Secrets, defaults) - delete(task.Secrets, "RUNNING_IN_CI") tt = task case manifest.DeployCF: task.PrePromote = t.Apply(task.PrePromote, defaults) diff --git a/defaults/tasks_env_defaulter_test.go b/defaults/tasks_env_defaulter_test.go index e216a8aa..ae4583cb 100644 --- a/defaults/tasks_env_defaulter_test.go +++ b/defaults/tasks_env_defaulter_test.go @@ -14,12 +14,6 @@ func TestSetsCorrectEnvVarsToEmptyVars(t *testing.T) { "RUNNING_IN_CI": "true", } - expectedSecrets := map[string]string{ - "ARTIFACTORY_URL": Concourse.Artifactory.URL, - "ARTIFACTORY_USERNAME": Concourse.Artifactory.Username, - "ARTIFACTORY_PASSWORD": Concourse.Artifactory.Password, - } - input := manifest.TaskList{ manifest.Update{}, manifest.Run{}, @@ -69,7 +63,7 @@ func TestSetsCorrectEnvVarsToEmptyVars(t *testing.T) { }, }, }, - manifest.DockerPush{Vars: expectedVars, Secrets: expectedSecrets}, + manifest.DockerPush{Vars: expectedVars}, manifest.Parallel{ Tasks: manifest.TaskList{ manifest.Sequence{ diff --git a/e2e/actions/deploy-katee/workflowExpected.yml b/e2e/actions/deploy-katee/workflowExpected.yml index 74a43023..4a403a49 100644 --- a/e2e/actions/deploy-katee/workflowExpected.yml +++ b/e2e/actions/deploy-katee/workflowExpected.yml @@ -51,6 +51,10 @@ jobs: platforms: linux/amd64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy diff --git a/e2e/actions/docker-push/.halfpipe.io b/e2e/actions/docker-push/.halfpipe.io index 844dfc3d..730fce8f 100644 --- a/e2e/actions/docker-push/.halfpipe.io +++ b/e2e/actions/docker-push/.halfpipe.io @@ -19,7 +19,6 @@ tasks: name: Push default image: eu.gcr.io/halfpipe-io/someImage restore_artifacts: true - tag: version - type: docker-push name: Push custom @@ -39,7 +38,6 @@ tasks: name: Push multiple platforms image: eu.gcr.io/halfpipe-io/someImage restore_artifacts: true - tag: version platforms: - "linux/amd64" - "linux/arm64" @@ -48,8 +46,17 @@ tasks: name: Push multiple platforms and use cache image: eu.gcr.io/halfpipe-io/someImage restore_artifacts: true - tag: version use_cache: true platforms: - "linux/amd64" - "linux/arm64" + +- type: docker-push + name: Push with secrets + image: eu.gcr.io/halfpipe-io/someImage + vars: + A: a + B: b + secrets: + C: ((secret.c)) + D: d diff --git a/e2e/actions/docker-push/workflowExpected.yml b/e2e/actions/docker-push/workflowExpected.yml index 10f26533..4309d471 100644 --- a/e2e/actions/docker-push/workflowExpected.yml +++ b/e2e/actions/docker-push/workflowExpected.yml @@ -86,6 +86,10 @@ jobs: platforms: linux/amd64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -170,6 +174,10 @@ jobs: platforms: linux/amd64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/dockerhubusername/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -234,6 +242,10 @@ jobs: platforms: linux/amd64,linux/arm64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -300,6 +312,10 @@ jobs: platforms: linux/amd64,linux/arm64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: |- eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} eu.gcr.io/halfpipe-io/cache/someImage:buildcache @@ -328,3 +344,79 @@ jobs: echo "- eu.gcr.io/halfpipe-io/someImage:latest" >> $GITHUB_STEP_SUMMARY echo "- eu.gcr.io/halfpipe-io/someImage:${{ env.BUILD_VERSION }}" >> $GITHUB_STEP_SUMMARY echo "- eu.gcr.io/halfpipe-io/someImage:${{ env.GIT_REVISION }}" >> $GITHUB_STEP_SUMMARY + push_with_secrets: + name: Push with secrets + needs: + - push_multiple_platforms_and_use_cache + runs-on: ee-runner + timeout-minutes: 60 + steps: + - name: Vault secrets + id: secrets + uses: hashicorp/vault-action@v2.7.3 + with: + exportEnv: false + method: approle + roleId: ${{ env.VAULT_ROLE_ID }} + secretId: ${{ env.VAULT_SECRET_ID }} + secrets: | + /springernature/data/halfpipe-team/secret c | springernature_data_halfpipe-team_secret_c ; + url: https://vault.halfpipe.io + - name: Checkout code + uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }} + submodules: recursive + - name: Build Image + uses: docker/build-push-action@v4 + with: + build-args: | + A=a + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + B=b + BUILD_VERSION=2.${{ github.run_number }}.0 + GIT_REVISION=${{ github.sha }} + RUNNING_IN_CI=true + VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} + VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + context: e2e/actions/docker-push + file: e2e/actions/docker-push/Dockerfile + platforms: linux/amd64 + provenance: false + push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + C=${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_c }} + D=d + tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} + - name: Run Trivy vulnerability scanner + uses: docker://aquasec/trivy + with: + args: -c "cd e2e/actions/docker-push; [ -f .trivyignore ] && echo \"Ignoring the following CVE's due to .trivyignore\" || true; [ -f .trivyignore ] && cat .trivyignore; echo || true; trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 1 eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }}" + entrypoint: /bin/sh + - name: Push Image + run: |- + docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} --tag eu.gcr.io/halfpipe-io/someImage:latest + docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} --tag eu.gcr.io/halfpipe-io/someImage:${{ env.BUILD_VERSION }} + docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} --tag eu.gcr.io/halfpipe-io/someImage:${{ env.GIT_REVISION }} + - name: Repository dispatch + uses: peter-evans/repository-dispatch@v2 + with: + event-type: docker-push:eu.gcr.io/halfpipe-io/someImage + token: ${{ secrets.EE_REPOSITORY_DISPATCH_TOKEN }} + - name: Summary + run: |- + echo ":ship: **Image Pushed Successfully**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "[eu.gcr.io/halfpipe-io/someImage](https://eu.gcr.io/halfpipe-io/someImage)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Tags:" >> $GITHUB_STEP_SUMMARY + echo "- eu.gcr.io/halfpipe-io/someImage:latest" >> $GITHUB_STEP_SUMMARY + echo "- eu.gcr.io/halfpipe-io/someImage:${{ env.BUILD_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- eu.gcr.io/halfpipe-io/someImage:${{ env.GIT_REVISION }}" >> $GITHUB_STEP_SUMMARY diff --git a/linters/errors.go b/linters/errors.go index 035c092b..95ebd478 100644 --- a/linters/errors.go +++ b/linters/errors.go @@ -38,7 +38,7 @@ var ( ErrDockerPlatformUnknown = newError("only linux/amd64 and/or linux/arm64 are supported") ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") - ErrDockerVarSecret = newError("docker build var uses secret") + ErrDockerVarSecret = newError("using a secret in docker build vars is not secure. See the 'secrets' option of the docker-push task") ErrMultipleTriggers = newError("cannot have multiple triggers of this type") diff --git a/renderers/actions/docker_push.go b/renderers/actions/docker_push.go index 31d16068..7faae963 100644 --- a/renderers/actions/docker_push.go +++ b/renderers/actions/docker_push.go @@ -58,6 +58,7 @@ func buildImage(a *Actions, task manifest.DockerPush, buildArgs map[string]strin "build-args": MultiLine{buildArgs}, "platforms": strings.Join(task.Platforms, ","), "provenance": false, + "secrets": MultiLine{task.Secrets}, }, } From aa86312e621e856c75598e6965776a0fa9ff211c Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Fri, 8 Sep 2023 14:42:59 +0100 Subject: [PATCH 14/40] remove vault build args docker-push --- .github/workflows/for-dependabot-to-check.yml | 20 ++-- defaults/task_docker_push.go | 6 +- e2e/actions/deploy-katee/workflowExpected.yml | 20 ++-- e2e/actions/docker-push/workflowExpected.yml | 100 ++++++++---------- renderers/actions/docker_push.go | 24 +++-- renderers/actions/workflow.go | 6 +- 6 files changed, 85 insertions(+), 91 deletions(-) diff --git a/.github/workflows/for-dependabot-to-check.yml b/.github/workflows/for-dependabot-to-check.yml index 01369a39..274271f8 100644 --- a/.github/workflows/for-dependabot-to-check.yml +++ b/.github/workflows/for-dependabot-to-check.yml @@ -152,23 +152,21 @@ jobs: uses: docker/build-push-action@v4 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: . file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64 provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME tags: eu.gcr.io/halfpipe-io/cache/blah:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy diff --git a/defaults/task_docker_push.go b/defaults/task_docker_push.go index 30ccb50a..6552079c 100644 --- a/defaults/task_docker_push.go +++ b/defaults/task_docker_push.go @@ -37,9 +37,9 @@ func dockerPushDefaulter(original manifest.DockerPush, man manifest.Manifest, de updated.Secrets["ARTIFACTORY_PASSWORD"] = defaults.Artifactory.Password } if man.Platform.IsActions() { - updated.Secrets["ARTIFACTORY_URL"] = "${{ secrets.EE_ARTIFACTORY_URL }}" - updated.Secrets["ARTIFACTORY_USERNAME"] = "${{ secrets.EE_ARTIFACTORY_USERNAME }}" - updated.Secrets["ARTIFACTORY_PASSWORD"] = "${{ secrets.EE_ARTIFACTORY_PASSWORD }}" + updated.Secrets["ARTIFACTORY_URL"] = "" + updated.Secrets["ARTIFACTORY_USERNAME"] = "" + updated.Secrets["ARTIFACTORY_PASSWORD"] = "" } return updated diff --git a/e2e/actions/deploy-katee/workflowExpected.yml b/e2e/actions/deploy-katee/workflowExpected.yml index 4a403a49..5a1b7e22 100644 --- a/e2e/actions/deploy-katee/workflowExpected.yml +++ b/e2e/actions/deploy-katee/workflowExpected.yml @@ -38,23 +38,21 @@ jobs: uses: docker/build-push-action@v4 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/deploy-katee file: e2e/actions/deploy-katee/Dockerfile platforms: linux/amd64 provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME tags: eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy diff --git a/e2e/actions/docker-push/workflowExpected.yml b/e2e/actions/docker-push/workflowExpected.yml index 4309d471..b92d095b 100644 --- a/e2e/actions/docker-push/workflowExpected.yml +++ b/e2e/actions/docker-push/workflowExpected.yml @@ -73,23 +73,21 @@ jobs: uses: docker/build-push-action@v4 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64 provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -158,26 +156,24 @@ jobs: uses: docker/build-push-action@v4 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME BAR=bar BLAH=${{ steps.secrets.outputs.springernature_data_halfpipe-team_very_secret }} - BUILD_VERSION=2.${{ github.run_number }}.0 + BUILD_VERSION FOO=foo - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile2 platforms: linux/amd64 provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME tags: eu.gcr.io/halfpipe-io/cache/dockerhubusername/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -229,23 +225,21 @@ jobs: uses: docker/build-push-action@v4 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64,linux/arm64 provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -297,14 +291,12 @@ jobs: uses: docker/build-push-action@v4 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI cache-from: type=registry,ref=eu.gcr.io/halfpipe-io/cache/someImage:buildcache cache-to: type=inline context: e2e/actions/docker-push @@ -313,9 +305,9 @@ jobs: provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME tags: |- eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} eu.gcr.io/halfpipe-io/cache/someImage:buildcache @@ -374,24 +366,22 @@ jobs: with: build-args: | A=a - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME B=b - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64 provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME C=${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_c }} D=d tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} diff --git a/renderers/actions/docker_push.go b/renderers/actions/docker_push.go index 7faae963..64f001a9 100644 --- a/renderers/actions/docker_push.go +++ b/renderers/actions/docker_push.go @@ -11,15 +11,7 @@ import ( func (a *Actions) dockerPushSteps(task manifest.DockerPush) (steps Steps) { steps = dockerLogin(task.Image, task.Username, task.Password) - buildArgs := map[string]string{} - for k, v := range globalEnv { - buildArgs[k] = v - } - for k, v := range task.Vars { - buildArgs[k] = v - } - - steps = append(steps, buildImage(a, task, buildArgs)) + steps = append(steps, buildImage(a, task)) steps = append(steps, scanImage(a, task)) steps = append(steps, pushImage(task)) steps = append(steps, repositoryDispatch(task.Image)) @@ -46,7 +38,19 @@ func repositoryDispatch(eventName string) Step { } } -func buildImage(a *Actions, task manifest.DockerPush, buildArgs map[string]string) Step { +func buildImage(a *Actions, task manifest.DockerPush) Step { + buildArgs := map[string]string{ + "ARTIFACTORY_PASSWORD": "", + "ARTIFACTORY_URL": "", + "ARTIFACTORY_USERNAME": "", + "BUILD_VERSION": "", + "GIT_REVISION": "", + "RUNNING_IN_CI": "", + } + for k, v := range task.Vars { + buildArgs[k] = v + } + step := Step{ Name: "Build Image", Uses: "docker/build-push-action@v4", diff --git a/renderers/actions/workflow.go b/renderers/actions/workflow.go index a97fd351..4b1e9e1f 100644 --- a/renderers/actions/workflow.go +++ b/renderers/actions/workflow.go @@ -92,7 +92,11 @@ type MultiLine struct { func (ml MultiLine) MarshalYAML() (interface{}, error) { var out []string for k, v := range ml.m { - out = append(out, fmt.Sprintf("%s=%s\n", k, v)) + if v == "" { + out = append(out, fmt.Sprintf("%s\n", k)) + } else { + out = append(out, fmt.Sprintf("%s=%s\n", k, v)) + } } sort.Strings(out) From bdac6a0a5c0c1dcf6a55edd4649d529ac7fbd2f3 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Tue, 12 Sep 2023 15:17:24 +0100 Subject: [PATCH 15/40] secrets in the docker push action require values (unlike build args) --- .github/workflows/for-dependabot-to-check.yml | 6 ++-- defaults/task_docker_push.go | 6 ++-- e2e/actions/deploy-katee/workflowExpected.yml | 6 ++-- e2e/actions/docker-push/workflowExpected.yml | 30 +++++++++---------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/for-dependabot-to-check.yml b/.github/workflows/for-dependabot-to-check.yml index 274271f8..2ea5e16b 100644 --- a/.github/workflows/for-dependabot-to-check.yml +++ b/.github/workflows/for-dependabot-to-check.yml @@ -164,9 +164,9 @@ jobs: provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD - ARTIFACTORY_URL - ARTIFACTORY_USERNAME + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/blah:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy diff --git a/defaults/task_docker_push.go b/defaults/task_docker_push.go index 6552079c..30ccb50a 100644 --- a/defaults/task_docker_push.go +++ b/defaults/task_docker_push.go @@ -37,9 +37,9 @@ func dockerPushDefaulter(original manifest.DockerPush, man manifest.Manifest, de updated.Secrets["ARTIFACTORY_PASSWORD"] = defaults.Artifactory.Password } if man.Platform.IsActions() { - updated.Secrets["ARTIFACTORY_URL"] = "" - updated.Secrets["ARTIFACTORY_USERNAME"] = "" - updated.Secrets["ARTIFACTORY_PASSWORD"] = "" + updated.Secrets["ARTIFACTORY_URL"] = "${{ secrets.EE_ARTIFACTORY_URL }}" + updated.Secrets["ARTIFACTORY_USERNAME"] = "${{ secrets.EE_ARTIFACTORY_USERNAME }}" + updated.Secrets["ARTIFACTORY_PASSWORD"] = "${{ secrets.EE_ARTIFACTORY_PASSWORD }}" } return updated diff --git a/e2e/actions/deploy-katee/workflowExpected.yml b/e2e/actions/deploy-katee/workflowExpected.yml index 5a1b7e22..f6f49dfb 100644 --- a/e2e/actions/deploy-katee/workflowExpected.yml +++ b/e2e/actions/deploy-katee/workflowExpected.yml @@ -50,9 +50,9 @@ jobs: provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD - ARTIFACTORY_URL - ARTIFACTORY_USERNAME + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy diff --git a/e2e/actions/docker-push/workflowExpected.yml b/e2e/actions/docker-push/workflowExpected.yml index b92d095b..583fe0f2 100644 --- a/e2e/actions/docker-push/workflowExpected.yml +++ b/e2e/actions/docker-push/workflowExpected.yml @@ -85,9 +85,9 @@ jobs: provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD - ARTIFACTORY_URL - ARTIFACTORY_USERNAME + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -171,9 +171,9 @@ jobs: provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD - ARTIFACTORY_URL - ARTIFACTORY_USERNAME + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/dockerhubusername/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -237,9 +237,9 @@ jobs: provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD - ARTIFACTORY_URL - ARTIFACTORY_USERNAME + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -305,9 +305,9 @@ jobs: provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD - ARTIFACTORY_URL - ARTIFACTORY_USERNAME + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: |- eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} eu.gcr.io/halfpipe-io/cache/someImage:buildcache @@ -379,9 +379,9 @@ jobs: provenance: false push: true secrets: | - ARTIFACTORY_PASSWORD - ARTIFACTORY_URL - ARTIFACTORY_USERNAME + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} C=${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_c }} D=d tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} From 862335030ea7cd8dddbb53fa8375a2c2e1e83b0d Mon Sep 17 00:00:00 2001 From: Simon Johansson Date: Wed, 27 Sep 2023 13:54:15 +0200 Subject: [PATCH 16/40] Run a cf logs --recent in case of cf push error --- e2e/concourse/artifacts/pipelineExpected.yml | 8 ++++ .../pipelineExpected.yml | 40 +++++++++++++++++++ .../deploy-cf-rolling/pipelineExpected.yml | 24 +++++++++++ .../pipelineExpected.yml | 8 ++++ e2e/concourse/deploy-cf/pipelineExpected.yml | 32 +++++++++++++++ .../notifications/pipelineExpected.yml | 8 ++++ renderers/concourse/deploy_cf.go | 26 ++++++++++-- renderers/concourse/pipeline.go | 1 - 8 files changed, 143 insertions(+), 4 deletions(-) diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index 49e43d77..51e0393e 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -252,6 +252,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/artifacts/manifest.yml + put: cf logs --recent + resource: cf-snpaas-pe-staging params: appPath: git/e2e/concourse/artifacts cliVersion: cf7 diff --git a/e2e/concourse/deploy-cf-docker-image/pipelineExpected.yml b/e2e/concourse/deploy-cf-docker-image/pipelineExpected.yml index 29b5a889..2459ac16 100644 --- a/e2e/concourse/deploy-cf-docker-image/pipelineExpected.yml +++ b/e2e/concourse/deploy-cf-docker-image/pipelineExpected.yml @@ -62,6 +62,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 @@ -105,6 +113,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 @@ -221,6 +237,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: rolling-cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 @@ -254,6 +278,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: rolling-cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 @@ -333,6 +365,14 @@ jobs: timeout: 1h - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: rolling-cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 diff --git a/e2e/concourse/deploy-cf-rolling/pipelineExpected.yml b/e2e/concourse/deploy-cf-rolling/pipelineExpected.yml index c37216a4..8e0a35c0 100644 --- a/e2e/concourse/deploy-cf-rolling/pipelineExpected.yml +++ b/e2e/concourse/deploy-cf-rolling/pipelineExpected.yml @@ -10,6 +10,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-rolling/manifest.yml + put: cf logs --recent + resource: rolling-cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf-rolling cliVersion: cf7 @@ -34,6 +42,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-rolling/manifest.yml + put: cf logs --recent + resource: rolling-cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf-rolling cliVersion: cf7 @@ -108,6 +124,14 @@ jobs: timeout: 1h - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-rolling/manifest.yml + put: cf logs --recent + resource: rolling-cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf-rolling cliVersion: cf7 diff --git a/e2e/concourse/deploy-cf-with-artefact/pipelineExpected.yml b/e2e/concourse/deploy-cf-with-artefact/pipelineExpected.yml index f2520428..929d8d66 100644 --- a/e2e/concourse/deploy-cf-with-artefact/pipelineExpected.yml +++ b/e2e/concourse/deploy-cf-with-artefact/pipelineExpected.yml @@ -129,6 +129,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-with-artefact/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: artifacts/e2e/concourse/deploy-cf-with-artefact/build/linux/binary cliVersion: cf7 diff --git a/e2e/concourse/deploy-cf/pipelineExpected.yml b/e2e/concourse/deploy-cf/pipelineExpected.yml index c6bbb7f2..5be34d4c 100644 --- a/e2e/concourse/deploy-cf/pipelineExpected.yml +++ b/e2e/concourse/deploy-cf/pipelineExpected.yml @@ -10,6 +10,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf cliVersion: cf7 @@ -34,6 +42,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf8 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf cliVersion: cf8 @@ -69,6 +85,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf cliVersion: cf7 @@ -502,6 +526,14 @@ jobs: timeout: 1h - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf cliVersion: cf7 diff --git a/e2e/concourse/notifications/pipelineExpected.yml b/e2e/concourse/notifications/pipelineExpected.yml index db8515c6..9318f7e6 100644 --- a/e2e/concourse/notifications/pipelineExpected.yml +++ b/e2e/concourse/notifications/pipelineExpected.yml @@ -239,6 +239,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/notifications/manifest.yml + put: cf logs --recent + resource: cf-snpaas-pe-staging params: appPath: git/e2e/concourse/notifications cliVersion: cf7 diff --git a/renderers/concourse/deploy_cf.go b/renderers/concourse/deploy_cf.go index 237c7930..46319ade 100644 --- a/renderers/concourse/deploy_cf.go +++ b/renderers/concourse/deploy_cf.go @@ -43,12 +43,12 @@ func (c Concourse) deployCFJob(task manifest.DeployCF, man manifest.Manifest, ba if len(task.PrePromote) == 0 { steps = append(steps, deploy.pushApp()) } else if task.Rolling { - steps = append(steps, deploy.pushCandidateApp()) + steps = append(steps, deploy.logsOnFailure(deploy.pushCandidateApp())) steps = append(steps, c.prePromoteTasks(deploy)...) steps = append(steps, deploy.pushApp()) steps = append(steps, deploy.removeTestApp()) } else { - steps = append(steps, deploy.pushCandidateApp()) + steps = append(steps, deploy.logsOnFailure(deploy.pushCandidateApp())) steps = append(steps, deploy.checkApp()) steps = append(steps, c.prePromoteTasks(deploy)...) steps = append(steps, deploy.promoteCandidateAppToLive()) @@ -232,7 +232,27 @@ func (d deployCF) pushApp() atc.Step { push.Params["buildVersionPath"] = path.Join("version", "version") } - return stepWithAttemptsAndTimeout(&push, d.task.GetAttempts(), d.task.GetTimeout()) + return d.logsOnFailure(stepWithAttemptsAndTimeout(&push, d.task.GetAttempts(), d.task.GetTimeout())) +} + +func (d deployCF) logsOnFailure(stepConfig atc.Step) atc.Step { + return atc.Step{ + Config: &atc.OnFailureStep{ + Step: stepConfig.Config, + Hook: atc.Step{ + Config: &atc.PutStep{ + Name: "cf logs --recent", + Resource: d.resourceName, + Params: atc.Params{ + "command": "halfpipe-logs", + "cliVersion": d.task.CliVersion, + "manifestPath": d.manifestPath, + }, + NoGet: true, + }, + }, + }, + } } func (d deployCF) configureSSO() atc.Step { diff --git a/renderers/concourse/pipeline.go b/renderers/concourse/pipeline.go index fe54b4b3..630c3c6b 100644 --- a/renderers/concourse/pipeline.go +++ b/renderers/concourse/pipeline.go @@ -551,5 +551,4 @@ func stepWithAttemptsAndTimeout(stepConfig atc.StepConfig, attempts int, timeout Attempts: attempts, }, } - } From 3a64e21dcdeffad933f2deabc8c4d25417364fd8 Mon Sep 17 00:00:00 2001 From: Simon Johansson Date: Wed, 27 Sep 2023 14:28:03 +0200 Subject: [PATCH 17/40] cf logs --recent for failed cf push in actions --- .github/workflows/for-dependabot-to-check.yml | 14 ++++ e2e/actions/deploy-cf/workflowExpected.yml | 84 +++++++++++++++++++ renderers/actions/deploy_cf.go | 9 ++ 3 files changed, 107 insertions(+) diff --git a/.github/workflows/for-dependabot-to-check.yml b/.github/workflows/for-dependabot-to-check.yml index 2ea5e16b..3bc1138d 100644 --- a/.github/workflows/for-dependabot-to-check.yml +++ b/.github/workflows/for-dependabot-to-check.yml @@ -81,6 +81,20 @@ jobs: username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: . + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: cf-space + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: diff --git a/e2e/actions/deploy-cf/workflowExpected.yml b/e2e/actions/deploy-cf/workflowExpected.yml index eb854307..6ccda3fc 100644 --- a/e2e/actions/deploy-cf/workflowExpected.yml +++ b/e2e/actions/deploy-cf/workflowExpected.yml @@ -104,6 +104,20 @@ jobs: CF_ENV_VAR_ENV2: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_something }} CF_ENV_VAR_ENV3: '{"a": "b", "c": "d"}' CF_ENV_VAR_ENV4: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_another_secret }} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf/foo.html + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -205,6 +219,20 @@ jobs: CF_ENV_VAR_ENV2: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_something }} CF_ENV_VAR_ENV3: '{"a": "b", "c": "d"}' CF_ENV_VAR_ENV4: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_another_secret }} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf/foo.html + cli_version: cf8 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -305,6 +333,20 @@ jobs: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} CF_ENV_VAR_ENV2: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_something }} CF_ENV_VAR_ENV4: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_another_secret }} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf/foo.html + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -481,6 +523,20 @@ jobs: username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest-docker.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -594,6 +650,20 @@ jobs: username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf/foo.html + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -700,6 +770,20 @@ jobs: username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: diff --git a/renderers/actions/deploy_cf.go b/renderers/actions/deploy_cf.go index 0f47917b..4cc7f4fd 100644 --- a/renderers/actions/deploy_cf.go +++ b/renderers/actions/deploy_cf.go @@ -75,6 +75,15 @@ func (a *Actions) deployCFSteps(task manifest.DeployCF, man manifest.Manifest) ( } deploySteps = append(deploySteps, push) + deploySteps = append(deploySteps, Step{ + Name: "cf logs --recent", + If: "failure()", + Uses: uses, + With: addCommonParams(With{ + "command": "halfpipe-logs", + }), + }) + deploySteps = append(deploySteps, Step{ Name: "Check", Uses: uses, From 8aac3f1e895a4b25b8716c33e74567d471327e86 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 27 Sep 2023 15:15:34 +0100 Subject: [PATCH 18/40] add go option to check loop vars --- build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.sh b/build.sh index 237af414..5bf851a2 100755 --- a/build.sh +++ b/build.sh @@ -13,6 +13,10 @@ if [ "${1-}" = "ci" ]; then go_opts="-mod=readonly" fi + +# https://github.com/golang/go/wiki/LoopvarExperiment +export GOEXPERIMENT=loopvar + echo [1/6] fmt go fmt ./... From 1a514c516e8faceed8b6454c555edf64b67a510d Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 27 Sep 2023 16:27:36 +0100 Subject: [PATCH 19/40] update list of shared secrets --- renderers/actions/secrets.go | 45 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/renderers/actions/secrets.go b/renderers/actions/secrets.go index f670bb7b..7985eb13 100644 --- a/renderers/actions/secrets.go +++ b/renderers/actions/secrets.go @@ -47,27 +47,30 @@ func (s *Secret) outputVar() string { // vault kv list /springernature/shared func isShared(s string) bool { return map[string]bool{ - "PPG-gradle-version-reporter": true, - "PPG-owasp-dependency-reporter": true, - "artifactory": true, - "artifactory-support": true, - "artifactory_test": true, - "bla": true, - "burpsuiteenterprise": true, - "contrastsecurity": true, - "ee-sso-route-service": true, - "fastly": true, - "grafana": true, - "halfpipe-artifacts": true, - "halfpipe-docker-config": true, - "halfpipe-gcr": true, - "halfpipe-github": true, - "halfpipe-ml-deploy": true, - "halfpipe-semver": true, - "halfpipe-slack": true, - "katee-tls-dev": true, - "katee-tls-prod": true, - "sentry-release-integration": true, + "PPG-gradle-version-reporter": true, + "PPG-owasp-dependency-reporter": true, + "artifactory": true, + "artifactory-support": true, + "artifactory_test": true, + "bla": true, + "burpsuiteenterprise": true, + "content_hub-casper-credentials-live": true, + "content_hub-casper-credentials-qa": true, + "contrastsecurity": true, + "eas-sigrid": true, + "ee-sso-route-service": true, + "fastly": true, + "grafana": true, + "halfpipe-artifacts": true, + "halfpipe-docker-config": true, + "halfpipe-gcr": true, + "halfpipe-github": true, + "halfpipe-ml-deploy": true, + "halfpipe-semver": true, + "halfpipe-slack": true, + "katee-tls-dev": true, + "katee-tls-prod": true, + "sentry-release-integration": true, }[s] } From 1525d3ffa23002ca2cfec95b38da620396b2e357 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 27 Sep 2023 16:29:54 +0100 Subject: [PATCH 20/40] wip - add exec command to run a task locally --- cmd/cmds/exec.go | 120 +++++++++++++++++++++++++++++++++++++++++++ manifest/manifest.go | 9 ++++ 2 files changed, 129 insertions(+) create mode 100644 cmd/cmds/exec.go diff --git a/cmd/cmds/exec.go b/cmd/cmds/exec.go new file mode 100644 index 00000000..cb702a67 --- /dev/null +++ b/cmd/cmds/exec.go @@ -0,0 +1,120 @@ +package cmds + +import ( + "fmt" + "github.com/spf13/cobra" + "github.com/springernature/halfpipe/manifest" + "os" + "strings" +) + +func init() { + rootCmd.AddCommand(execCmd) +} + +var execCmd = &cobra.Command{ + Use: "exec ", + Short: "Execute a task locally", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + taskName := args[0] + man, controller := getManifestAndController(formatInput(Input)) + man, err := controller.DefaultAndMap(man) + if err != nil { + printErr(err) + os.Exit(1) + } + + found, task := man.Tasks.GetRunTask(taskName) + if !found { + printErr(fmt.Errorf("run task not found '%s'", taskName)) + os.Exit(1) + } + fmt.Println(renderShellCommand(task, man.Team)) + }, +} + +func renderShellCommand(task manifest.Run, team string) string { + s := []string{ + "docker run -it", + `-v "$PWD":/app`, + "-w /app", + } + + for k, v := range task.Vars { + s = append(s, fmt.Sprintf("-e %s=%s", k, vaultLookup(v, team))) + } + + s = append(s, task.Docker.Image, task.Script) + + return strings.Join(s, " \\ \n ") +} + +func vaultLookup(s string, team string) string { + if !isSecret(s) { + return s + } + s = strings.TrimSpace(s[2 : len(s)-2]) + + if isKeyValueSecret(s) { + parts := strings.Split(s, ".") + vaultFolder := team + if isShared(parts[0]) { + vaultFolder = "shared" + } + return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s/%s)", parts[1], vaultFolder, parts[0]) + } + + if isAbsolutePathSecret(s) { + parts := strings.Split(s, " ") + return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s/%s)", parts[1], team, parts[0]) + } + + return s +} + +// ************************************************** +// all this copied from renderers/actions/secrets.go + +// check if a secret matches one of the shared secrets +// vault kv list /springernature/shared +func isShared(s string) bool { + return map[string]bool{ + "PPG-gradle-version-reporter": true, + "PPG-owasp-dependency-reporter": true, + "artifactory": true, + "artifactory-support": true, + "artifactory_test": true, + "bla": true, + "burpsuiteenterprise": true, + "content_hub-casper-credentials-live": true, + "content_hub-casper-credentials-qa": true, + "contrastsecurity": true, + "eas-sigrid": true, + "ee-sso-route-service": true, + "fastly": true, + "grafana": true, + "halfpipe-artifacts": true, + "halfpipe-docker-config": true, + "halfpipe-gcr": true, + "halfpipe-github": true, + "halfpipe-ml-deploy": true, + "halfpipe-semver": true, + "halfpipe-slack": true, + "katee-tls-dev": true, + "katee-tls-prod": true, + "sentry-release-integration": true, + }[s] +} + +func isSecret(s string) bool { + return strings.HasPrefix(s, "((") && strings.HasSuffix(s, "))") +} + +func isAbsolutePathSecret(s string) bool { + return len(strings.Split(s, " ")) == 2 +} + +func isKeyValueSecret(s string) bool { + return len(strings.Split(s, ".")) == 2 +} diff --git a/manifest/manifest.go b/manifest/manifest.go index 8b855794..74cf3d62 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -69,6 +69,15 @@ func (tl TaskList) Flatten() (updated TaskList) { return } +func (tl TaskList) GetRunTask(name string) (found bool, task Run) { + for _, t := range tl.Flatten() { + if t.GetName() == name { + return true, t.(Run) + } + } + return false, Run{} +} + func (tl TaskList) PreviousTaskNames(currentIndex int) []string { if currentIndex == 0 { return []string{} From 0f7e2efe99bbc0d04e240dbecd63c4dd99cae1b8 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 27 Sep 2023 16:31:09 +0100 Subject: [PATCH 21/40] bump deps --- go.mod | 14 +++++++------- go.sum | 44 ++++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index b0bbeed5..45e3b22c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( code.cloudfoundry.org/cli v7.1.0+incompatible github.com/blang/semver v3.5.1+incompatible github.com/cloudfoundry/bosh-cli v6.4.1+incompatible - github.com/cloudfoundry/bosh-utils v0.0.388 // indirect + github.com/cloudfoundry/bosh-utils v0.0.398 // indirect github.com/concourse/concourse v1.6.1-0.20230204041300-d289c02f878d github.com/cppforlife/go-patch v0.2.0 // indirect github.com/gookit/color v1.5.4 @@ -19,7 +19,7 @@ require ( github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/onsi/gomega v1.27.10 github.com/pkg/errors v0.9.1 - github.com/spf13/afero v1.9.5 + github.com/spf13/afero v1.10.0 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 github.com/tcnksm/go-gitconfig v0.1.2 @@ -30,7 +30,7 @@ require ( require ( github.com/sirupsen/logrus v1.9.3 - golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 gopkg.in/yaml.v3 v3.0.1 ) @@ -50,9 +50,9 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/tedsuo/rata v1.0.1-0.20170830210128-07d200713958 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/crypto v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 56ad325e..d2b48677 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudfoundry/bosh-cli v6.4.1+incompatible h1:n5/+NIF9QxvGINOrjh6DmO+GTen78MoCj5+LU9L8bR4= github.com/cloudfoundry/bosh-cli v6.4.1+incompatible/go.mod h1:rzIB+e1sn7wQL/TJ54bl/FemPKRhXby5BIMS3tLuWFM= -github.com/cloudfoundry/bosh-utils v0.0.388 h1:5UwnFgDDioddiWzRuQbxeP+ERcJwFQ1FM0x+udN97nM= -github.com/cloudfoundry/bosh-utils v0.0.388/go.mod h1:bKQU0etIkZD0cTeqvv+im2K6yWzQIv0kx3IXnkKAxzU= +github.com/cloudfoundry/bosh-utils v0.0.398 h1:kPtAoO7GmwEFolaQDXDjz7rB5HTytYKoz4PCjaiewBI= +github.com/cloudfoundry/bosh-utils v0.0.398/go.mod h1:kVhLBy5Tnur0NOTtSNeERBp359KZAgZNPXt2md1fUHI= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -140,8 +140,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8 h1:gpptm606MZYGaMHMsB4Srmb6EbW/IVHnt04rcMXnkBQ= +github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -194,8 +194,8 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= -github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= +github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= +github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -213,8 +213,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= -github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -248,8 +248,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -260,8 +260,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -317,8 +317,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -377,11 +377,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -390,8 +390,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -442,8 +442,8 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 84653d824069bba82ed7bb419a33a380a5b5ddd2 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 27 Sep 2023 16:45:29 +0100 Subject: [PATCH 22/40] wip - halfpipe exec docker-compose tasks too --- cmd/cmds/exec.go | 37 ++++++++++++++++++++++++++++++++----- manifest/manifest.go | 6 +++--- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/cmd/cmds/exec.go b/cmd/cmds/exec.go index cb702a67..fabf539d 100644 --- a/cmd/cmds/exec.go +++ b/cmd/cmds/exec.go @@ -25,16 +25,21 @@ var execCmd = &cobra.Command{ os.Exit(1) } - found, task := man.Tasks.GetRunTask(taskName) - if !found { - printErr(fmt.Errorf("run task not found '%s'", taskName)) + task := man.Tasks.GetTask(taskName) + + switch t := task.(type) { + case manifest.Run: + fmt.Println(renderRunCommand(t, man.Team)) + case manifest.DockerCompose: + fmt.Println(renderDockerComposeCommand(t, man.Team)) + default: + printErr(fmt.Errorf("task not found with name '%s' and type 'run' or 'docker-compose'", taskName)) os.Exit(1) } - fmt.Println(renderShellCommand(task, man.Team)) }, } -func renderShellCommand(task manifest.Run, team string) string { +func renderRunCommand(task manifest.Run, team string) string { s := []string{ "docker run -it", `-v "$PWD":/app`, @@ -50,6 +55,28 @@ func renderShellCommand(task manifest.Run, team string) string { return strings.Join(s, " \\ \n ") } +func renderDockerComposeCommand(task manifest.DockerCompose, team string) string { + s := []string{ + "docker compose", + fmt.Sprintf("-f %s", task.ComposeFile), + "run", + `-v "$PWD":/app`, + "-w /app", + } + + for k, v := range task.Vars { + s = append(s, fmt.Sprintf("-e %s=%s", k, vaultLookup(v, team))) + } + + s = append(s, "--use-aliases", task.Service) + + if task.Command != "" { + s = append(s, task.Command) + } + + return strings.Join(s, " \\ \n ") +} + func vaultLookup(s string, team string) string { if !isSecret(s) { return s diff --git a/manifest/manifest.go b/manifest/manifest.go index 74cf3d62..772dfdd0 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -69,13 +69,13 @@ func (tl TaskList) Flatten() (updated TaskList) { return } -func (tl TaskList) GetRunTask(name string) (found bool, task Run) { +func (tl TaskList) GetTask(name string) Task { for _, t := range tl.Flatten() { if t.GetName() == name { - return true, t.(Run) + return t } } - return false, Run{} + return nil } func (tl TaskList) PreviousTaskNames(currentIndex int) []string { From 318475f33543c387045c5aa24814767fba8a255c Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Thu, 28 Sep 2023 11:04:45 +0100 Subject: [PATCH 23/40] wip shell renderer --- cmd/cmds/actions-migration-help.go | 2 +- cmd/cmds/exec.go | 133 ++------------------------- cmd/cmds/helpers.go | 13 +-- cmd/cmds/internal-representation.go | 2 +- cmd/cmds/pipeline_name.go | 2 +- cmd/cmds/root.go | 2 +- controller.go | 4 + renderers/shell/shell.go | 136 ++++++++++++++++++++++++++++ 8 files changed, 158 insertions(+), 136 deletions(-) create mode 100644 renderers/shell/shell.go diff --git a/cmd/cmds/actions-migration-help.go b/cmd/cmds/actions-migration-help.go index ad7c43cc..3b909060 100644 --- a/cmd/cmds/actions-migration-help.go +++ b/cmd/cmds/actions-migration-help.go @@ -14,7 +14,7 @@ var actionsMigrationHelp = &cobra.Command{ Use: "actions-migration-help", Short: "Prints out the steps needed to migrate from Concourse to Actions", Run: func(cmd *cobra.Command, args []string) { - man, controller := getManifestAndController(formatInput(Input)) + man, controller := getManifestAndController(formatInput(Input), nil) response := controller.Process(man) tpl, _ := template.New("").Parse(` diff --git a/cmd/cmds/exec.go b/cmd/cmds/exec.go index fabf539d..c9c6e0a2 100644 --- a/cmd/cmds/exec.go +++ b/cmd/cmds/exec.go @@ -3,9 +3,7 @@ package cmds import ( "fmt" "github.com/spf13/cobra" - "github.com/springernature/halfpipe/manifest" - "os" - "strings" + "github.com/springernature/halfpipe/renderers/shell" ) func init() { @@ -18,130 +16,13 @@ var execCmd = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { taskName := args[0] - man, controller := getManifestAndController(formatInput(Input)) - man, err := controller.DefaultAndMap(man) - if err != nil { - printErr(err) - os.Exit(1) - } - task := man.Tasks.GetTask(taskName) + shellRenderer := shell.NewShell(taskName) + man, controller := getManifestAndController(formatInput(Input), shellRenderer) - switch t := task.(type) { - case manifest.Run: - fmt.Println(renderRunCommand(t, man.Team)) - case manifest.DockerCompose: - fmt.Println(renderDockerComposeCommand(t, man.Team)) - default: - printErr(fmt.Errorf("task not found with name '%s' and type 'run' or 'docker-compose'", taskName)) - os.Exit(1) - } - }, -} - -func renderRunCommand(task manifest.Run, team string) string { - s := []string{ - "docker run -it", - `-v "$PWD":/app`, - "-w /app", - } - - for k, v := range task.Vars { - s = append(s, fmt.Sprintf("-e %s=%s", k, vaultLookup(v, team))) - } - - s = append(s, task.Docker.Image, task.Script) - - return strings.Join(s, " \\ \n ") -} - -func renderDockerComposeCommand(task manifest.DockerCompose, team string) string { - s := []string{ - "docker compose", - fmt.Sprintf("-f %s", task.ComposeFile), - "run", - `-v "$PWD":/app`, - "-w /app", - } - - for k, v := range task.Vars { - s = append(s, fmt.Sprintf("-e %s=%s", k, vaultLookup(v, team))) - } - - s = append(s, "--use-aliases", task.Service) - - if task.Command != "" { - s = append(s, task.Command) - } - - return strings.Join(s, " \\ \n ") -} + response := controller.Process(man) + outputLintResults(response.LintResults) + fmt.Println(response) -func vaultLookup(s string, team string) string { - if !isSecret(s) { - return s - } - s = strings.TrimSpace(s[2 : len(s)-2]) - - if isKeyValueSecret(s) { - parts := strings.Split(s, ".") - vaultFolder := team - if isShared(parts[0]) { - vaultFolder = "shared" - } - return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s/%s)", parts[1], vaultFolder, parts[0]) - } - - if isAbsolutePathSecret(s) { - parts := strings.Split(s, " ") - return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s/%s)", parts[1], team, parts[0]) - } - - return s -} - -// ************************************************** -// all this copied from renderers/actions/secrets.go - -// check if a secret matches one of the shared secrets -// vault kv list /springernature/shared -func isShared(s string) bool { - return map[string]bool{ - "PPG-gradle-version-reporter": true, - "PPG-owasp-dependency-reporter": true, - "artifactory": true, - "artifactory-support": true, - "artifactory_test": true, - "bla": true, - "burpsuiteenterprise": true, - "content_hub-casper-credentials-live": true, - "content_hub-casper-credentials-qa": true, - "contrastsecurity": true, - "eas-sigrid": true, - "ee-sso-route-service": true, - "fastly": true, - "grafana": true, - "halfpipe-artifacts": true, - "halfpipe-docker-config": true, - "halfpipe-gcr": true, - "halfpipe-github": true, - "halfpipe-ml-deploy": true, - "halfpipe-semver": true, - "halfpipe-slack": true, - "katee-tls-dev": true, - "katee-tls-prod": true, - "sentry-release-integration": true, - }[s] -} - -func isSecret(s string) bool { - return strings.HasPrefix(s, "((") && strings.HasSuffix(s, "))") -} - -func isAbsolutePathSecret(s string) bool { - return len(strings.Split(s, " ")) == 2 -} - -func isKeyValueSecret(s string) bool { - return len(strings.Split(s, ".")) == 2 + }, } diff --git a/cmd/cmds/helpers.go b/cmd/cmds/helpers.go index 9e85244a..6eca8294 100644 --- a/cmd/cmds/helpers.go +++ b/cmd/cmds/helpers.go @@ -132,7 +132,7 @@ func createController(projectData project.Data, fs afero.Afero, currentDir strin } -func getManifestAndController(halfpipeFilenameOptions []string) (manifest.Manifest, halfpipe.Controller) { +func getManifestAndController(halfpipeFilenameOptions []string, renderer halfpipe.Renderer) (manifest.Manifest, halfpipe.Controller) { if err := checkVersion(); err != nil { printErr(err) os.Exit(1) @@ -157,11 +157,12 @@ func getManifestAndController(halfpipeFilenameOptions []string) (manifest.Manife outputLintResults(linters.LintResults{linters.NewLintResult("Halfpipe Manifest", "https://ee.public.springernature.app/rel-eng/halfpipe/manifest/", manErrors)}) } - var renderer halfpipe.Renderer - if man.Platform.IsActions() { - renderer = actions.NewActions(projectData.GitURI, projectData.HalfpipeFilePath) - } else { - renderer = concourse.NewPipeline(projectData.HalfpipeFilePath) + if renderer == nil { + if man.Platform.IsActions() { + renderer = actions.NewActions(projectData.GitURI, projectData.HalfpipeFilePath) + } else { + renderer = concourse.NewPipeline(projectData.HalfpipeFilePath) + } } controller := createController(projectData, fs, currentDir, renderer) diff --git a/cmd/cmds/internal-representation.go b/cmd/cmds/internal-representation.go index 0b42d497..90b60908 100644 --- a/cmd/cmds/internal-representation.go +++ b/cmd/cmds/internal-representation.go @@ -16,7 +16,7 @@ var internalRepresentation = &cobra.Command{ Short: `Prints the internal representation of the manifest`, Run: func(cmd *cobra.Command, args []string) { - man, controller := getManifestAndController(formatInput(Input)) + man, controller := getManifestAndController(formatInput(Input), nil) defaultedAndMappedManifest, _ := controller.DefaultAndMap(man) diff --git a/cmd/cmds/pipeline_name.go b/cmd/cmds/pipeline_name.go index e091ef92..88c5f277 100644 --- a/cmd/cmds/pipeline_name.go +++ b/cmd/cmds/pipeline_name.go @@ -14,7 +14,7 @@ var pipelineNameCmd = &cobra.Command{ Use: "pipeline-name", Short: "Prints the name of the pipeline", Run: func(cmd *cobra.Command, args []string) { - man, _ := getManifestAndController(formatInput(Input)) + man, _ := getManifestAndController(formatInput(Input), nil) if man.PipelineName() == "" { os.Exit(1) } diff --git a/cmd/cmds/root.go b/cmd/cmds/root.go index 8f97379a..b0ea1c62 100644 --- a/cmd/cmds/root.go +++ b/cmd/cmds/root.go @@ -11,7 +11,7 @@ var rootCmd = &cobra.Command{ Short: `halfpipe is a tool to lint and render pipelines Invoke without any arguments to lint your .halfpipe.io file and render a pipeline`, Run: func(cmd *cobra.Command, args []string) { - man, controller := getManifestAndController(formatInput(Input)) + man, controller := getManifestAndController(formatInput(Input), nil) response := controller.Process(man) if man.Platform.IsActions() && output == "" { diff --git a/controller.go b/controller.go index 4d6f23a5..de6a0715 100644 --- a/controller.go +++ b/controller.go @@ -20,6 +20,10 @@ type Response struct { Platform manifest.Platform } +func (r Response) String() string { + return r.ConfigYaml +} + type Renderer interface { Render(manifest manifest.Manifest) (string, error) } diff --git a/renderers/shell/shell.go b/renderers/shell/shell.go new file mode 100644 index 00000000..24b009c0 --- /dev/null +++ b/renderers/shell/shell.go @@ -0,0 +1,136 @@ +package shell + +import ( + "fmt" + "github.com/springernature/halfpipe" + "github.com/springernature/halfpipe/manifest" + "strings" +) + +type shell struct { + taskName string +} + +func NewShell(taskName string) halfpipe.Renderer { + return shell{taskName: taskName} +} + +func (s shell) Render(man manifest.Manifest) (string, error) { + task := man.Tasks.GetTask(s.taskName) + + switch t := task.(type) { + case manifest.Run: + return renderRunCommand(t, man.Team), nil + case manifest.DockerCompose: + return renderDockerComposeCommand(t, man.Team), nil + } + + return "", fmt.Errorf("task not found with name '%s' and type 'run' or 'docker-compose'", s.taskName) +} + +func renderRunCommand(task manifest.Run, team string) string { + s := []string{ + "docker run -it", + `-v "$PWD":/app`, + "-w /app", + } + + for k, v := range task.Vars { + s = append(s, fmt.Sprintf("-e %s=%s", k, vaultLookup(v, team))) + } + + s = append(s, task.Docker.Image, task.Script) + + return strings.Join(s, " \\ \n ") +} + +func renderDockerComposeCommand(task manifest.DockerCompose, team string) string { + s := []string{ + "docker compose", + fmt.Sprintf("-f %s", task.ComposeFile), + "run", + `-v "$PWD":/app`, + "-w /app", + } + + for k, v := range task.Vars { + s = append(s, fmt.Sprintf("-e %s=%s", k, vaultLookup(v, team))) + } + + s = append(s, "--use-aliases", task.Service) + + if task.Command != "" { + s = append(s, task.Command) + } + + return strings.Join(s, " \\ \n ") +} + +func vaultLookup(s string, team string) string { + if !isSecret(s) { + return s + } + s = strings.TrimSpace(s[2 : len(s)-2]) + + if isKeyValueSecret(s) { + parts := strings.Split(s, ".") + vaultFolder := team + if isShared(parts[0]) { + vaultFolder = "shared" + } + return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s/%s)", parts[1], vaultFolder, parts[0]) + } + + if isAbsolutePathSecret(s) { + parts := strings.Split(s, " ") + return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s/%s)", parts[1], team, parts[0]) + } + + return s +} + +// ************************************************** +// all this copied from renderers/actions/secrets.go + +// check if a secret matches one of the shared secrets +// vault kv list /springernature/shared +func isShared(s string) bool { + return map[string]bool{ + "PPG-gradle-version-reporter": true, + "PPG-owasp-dependency-reporter": true, + "artifactory": true, + "artifactory-support": true, + "artifactory_test": true, + "bla": true, + "burpsuiteenterprise": true, + "content_hub-casper-credentials-live": true, + "content_hub-casper-credentials-qa": true, + "contrastsecurity": true, + "eas-sigrid": true, + "ee-sso-route-service": true, + "fastly": true, + "grafana": true, + "halfpipe-artifacts": true, + "halfpipe-docker-config": true, + "halfpipe-gcr": true, + "halfpipe-github": true, + "halfpipe-ml-deploy": true, + "halfpipe-semver": true, + "halfpipe-slack": true, + "katee-tls-dev": true, + "katee-tls-prod": true, + "sentry-release-integration": true, + }[s] +} + +func isSecret(s string) bool { + return strings.HasPrefix(s, "((") && strings.HasSuffix(s, "))") +} + +func isAbsolutePathSecret(s string) bool { + return len(strings.Split(s, " ")) == 2 +} + +func isKeyValueSecret(s string) bool { + return len(strings.Split(s, ".")) == 2 +} From b748c5f21478649cf303ff52fffcbb585fc4450e Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Thu, 28 Sep 2023 12:46:48 +0100 Subject: [PATCH 24/40] tweak --- cmd/cmds/exec.go | 2 +- renderers/shell/shell.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/cmds/exec.go b/cmd/cmds/exec.go index c9c6e0a2..f6ed1cca 100644 --- a/cmd/cmds/exec.go +++ b/cmd/cmds/exec.go @@ -17,7 +17,7 @@ var execCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { taskName := args[0] - shellRenderer := shell.NewShell(taskName) + shellRenderer := shell.New(taskName) man, controller := getManifestAndController(formatInput(Input), shellRenderer) response := controller.Process(man) diff --git a/renderers/shell/shell.go b/renderers/shell/shell.go index 24b009c0..28f7bed8 100644 --- a/renderers/shell/shell.go +++ b/renderers/shell/shell.go @@ -11,7 +11,7 @@ type shell struct { taskName string } -func NewShell(taskName string) halfpipe.Renderer { +func New(taskName string) halfpipe.Renderer { return shell{taskName: taskName} } From 583490654652ac2535ec617a1b6425af89801341 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Fri, 29 Sep 2023 13:39:06 +0100 Subject: [PATCH 25/40] refactor secret rendering --- renderers/actions/secrets.go | 119 ++++++---------------------- renderers/shared/secrets/secrets.go | 87 ++++++++++++++++++++ renderers/shell/shell.go | 74 ++--------------- 3 files changed, 119 insertions(+), 161 deletions(-) create mode 100644 renderers/shared/secrets/secrets.go diff --git a/renderers/actions/secrets.go b/renderers/actions/secrets.go index 7985eb13..1df62842 100644 --- a/renderers/actions/secrets.go +++ b/renderers/actions/secrets.go @@ -2,6 +2,7 @@ package actions import ( "fmt" + "github.com/springernature/halfpipe/renderers/shared/secrets" "sort" "strings" ) @@ -28,95 +29,25 @@ var githubSecrets = struct { VaultSecretID: "${{ secrets.VAULT_SECRET_ID }}", } -type Secret struct { - vaultPath string +func secretVar(s *secrets.Secret) string { + return fmt.Sprintf("${{ steps.secrets.outputs.%s }}", secretOutputVar(s)) } -func (s *Secret) actionsVar() string { - return fmt.Sprintf("${{ steps.secrets.outputs.%s }}", s.outputVar()) +func secretVaultPath(s *secrets.Secret) string { + return fmt.Sprintf("/springernature/data/%s %s", s.MapPath, s.Key) } - -func (s *Secret) outputVar() string { - ov := strings.ReplaceAll(s.vaultPath, "/", "_") +func secretOutputVar(s *secrets.Secret) string { + ov := strings.ReplaceAll(secretVaultPath(s), "/", "_") ov = strings.ReplaceAll(ov, " ", "_") ov = strings.TrimPrefix(ov, "_") return ov } -// check if a secret matches one of the shared secrets -// vault kv list /springernature/shared -func isShared(s string) bool { - return map[string]bool{ - "PPG-gradle-version-reporter": true, - "PPG-owasp-dependency-reporter": true, - "artifactory": true, - "artifactory-support": true, - "artifactory_test": true, - "bla": true, - "burpsuiteenterprise": true, - "content_hub-casper-credentials-live": true, - "content_hub-casper-credentials-qa": true, - "contrastsecurity": true, - "eas-sigrid": true, - "ee-sso-route-service": true, - "fastly": true, - "grafana": true, - "halfpipe-artifacts": true, - "halfpipe-docker-config": true, - "halfpipe-gcr": true, - "halfpipe-github": true, - "halfpipe-ml-deploy": true, - "halfpipe-semver": true, - "halfpipe-slack": true, - "katee-tls-dev": true, - "katee-tls-prod": true, - "sentry-release-integration": true, - }[s] -} - -func toSecret(s string, team string) *Secret { - if !isSecret(s) { - return nil - } - - secretValue := s[2 : len(s)-2] - - if isKeyValueSecret(secretValue) { - parts := strings.Split(secretValue, ".") - if isShared(parts[0]) { - team = "shared" - } - return &Secret{ - vaultPath: fmt.Sprintf("/springernature/data/%s/%s %s", team, parts[0], parts[1]), - } - } - - if isAbsolutePathSecret(secretValue) { - return &Secret{ - vaultPath: secretValue, - } - } - - return nil -} - -func isSecret(s string) bool { - return strings.HasPrefix(s, "((") && strings.HasSuffix(s, "))") -} - -func isAbsolutePathSecret(s string) bool { - return len(strings.Split(s, " ")) == 2 -} - -func isKeyValueSecret(s string) bool { - return len(strings.Split(s, ".")) == 2 -} - -func secretsToActionsSecret(secrets []*Secret) string { +func secretsToActionsSecret(secrets []*secrets.Secret) string { uniqueSecrets := map[string]string{} for _, s := range secrets { - x := fmt.Sprintf("%s | %s ;\n", s.vaultPath, s.outputVar()) - uniqueSecrets[s.outputVar()] = x + x := fmt.Sprintf("%s | %s ;\n", secretVaultPath(s), secretOutputVar(s)) + uniqueSecrets[secretOutputVar(s)] = x } var secs []string @@ -128,7 +59,7 @@ func secretsToActionsSecret(secrets []*Secret) string { return strings.Join(secs, "") } -func fetchSecrets(secrets []*Secret, team string) Step { +func fetchSecrets(secrets []*secrets.Secret) Step { return Step{ Name: "Vault secrets", ID: "secrets", @@ -145,7 +76,7 @@ func fetchSecrets(secrets []*Secret, team string) Step { } func convertSecrets(steps Steps, team string) (newSteps Steps) { - secrets := []*Secret{} + allSecrets := []*secrets.Secret{} for _, step := range steps { newWith := With{} @@ -153,39 +84,39 @@ func convertSecrets(steps Steps, team string) (newSteps Steps) { switch v := value.(type) { case MultiLine: secretList, multiLineStringWithActionSecret := multiLineStringToSecret(v.m, team) - secrets = append(secrets, secretList...) + allSecrets = append(allSecrets, secretList...) value = MultiLine{multiLineStringWithActionSecret} default: - if s := toSecret(fmt.Sprintf("%v", value), team); s != nil { - secrets = append(secrets, s) - value = s.actionsVar() + if s := secrets.New(fmt.Sprintf("%v", value), team); s != nil { + allSecrets = append(allSecrets, s) + value = secretVar(s) } } newWith[key] = value } step.With = newWith for k, v := range step.Env { - if s := toSecret(v, team); s != nil { - secrets = append(secrets, s) - step.Env[k] = s.actionsVar() + if s := secrets.New(v, team); s != nil { + allSecrets = append(allSecrets, s) + step.Env[k] = secretVar(s) } } newSteps = append(newSteps, step) } - if len(secrets) > 0 { - newSteps = append(Steps{fetchSecrets(secrets, team)}, newSteps...) + if len(allSecrets) > 0 { + newSteps = append(Steps{fetchSecrets(allSecrets)}, newSteps...) } return newSteps } -func multiLineStringToSecret(ml map[string]string, team string) ([]*Secret, map[string]string) { +func multiLineStringToSecret(ml map[string]string, team string) ([]*secrets.Secret, map[string]string) { m := make(map[string]string) - var sec []*Secret + var sec []*secrets.Secret for k, v := range ml { - if a := toSecret(v, team); a != nil { + if a := secrets.New(v, team); a != nil { sec = append(sec, a) - m[k] = a.actionsVar() + m[k] = secretVar(a) } else { m[k] = v } diff --git a/renderers/shared/secrets/secrets.go b/renderers/shared/secrets/secrets.go new file mode 100644 index 00000000..ed80b01c --- /dev/null +++ b/renderers/shared/secrets/secrets.go @@ -0,0 +1,87 @@ +package secrets + +import ( + "fmt" + "strings" +) + +// Secret models a Vault secret +// MapPath is root-relative path e.g. "/myteam/myproject/mysecretmap" +type Secret struct { + MapPath string + Key string +} + +// New returns a Secret from a string in the "halfpipe" format +// "((map.key))" or "((/path/to/map key))" +func New(s string, team string) *Secret { + if !isSecret(s) { + return nil + } + + secretValue := strings.TrimSpace(s[2 : len(s)-2]) + + if isKeyValueSecret(secretValue) { + parts := strings.Split(secretValue, ".") + if isSharedSecret(parts[0]) { + team = "shared" + } + return &Secret{ + MapPath: fmt.Sprintf("%s/%s", team, parts[0]), + Key: parts[1], + } + } + + if isAbsolutePathSecret(secretValue) { + parts := strings.Split(secretValue, " ") + mapPath := strings.TrimPrefix(parts[0], "/springernature/data/") + mapPath = strings.TrimPrefix(mapPath, "/springernature/") + return &Secret{ + MapPath: mapPath, + Key: parts[1], + } + } + + return nil +} + +func isSecret(s string) bool { + return strings.HasPrefix(s, "((") && strings.HasSuffix(s, "))") +} + +func isAbsolutePathSecret(s string) bool { + return len(strings.Split(s, " ")) == 2 +} + +func isKeyValueSecret(s string) bool { + return len(strings.Split(s, ".")) == 2 +} + +func isSharedSecret(s string) bool { + return map[string]bool{ + "PPG-gradle-version-reporter": true, + "PPG-owasp-dependency-reporter": true, + "artifactory": true, + "artifactory-support": true, + "artifactory_test": true, + "bla": true, + "burpsuiteenterprise": true, + "content_hub-casper-credentials-live": true, + "content_hub-casper-credentials-qa": true, + "contrastsecurity": true, + "eas-sigrid": true, + "ee-sso-route-service": true, + "fastly": true, + "grafana": true, + "halfpipe-artifacts": true, + "halfpipe-docker-config": true, + "halfpipe-gcr": true, + "halfpipe-github": true, + "halfpipe-ml-deploy": true, + "halfpipe-semver": true, + "halfpipe-slack": true, + "katee-tls-dev": true, + "katee-tls-prod": true, + "sentry-release-integration": true, + }[s] +} diff --git a/renderers/shell/shell.go b/renderers/shell/shell.go index 28f7bed8..bdf3a92c 100644 --- a/renderers/shell/shell.go +++ b/renderers/shell/shell.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/springernature/halfpipe" "github.com/springernature/halfpipe/manifest" + "github.com/springernature/halfpipe/renderers/shared/secrets" "strings" ) @@ -36,7 +37,7 @@ func renderRunCommand(task manifest.Run, team string) string { } for k, v := range task.Vars { - s = append(s, fmt.Sprintf("-e %s=%s", k, vaultLookup(v, team))) + s = append(s, fmt.Sprintf("-e %s=%s", k, convertSecret(v, team))) } s = append(s, task.Docker.Image, task.Script) @@ -54,7 +55,7 @@ func renderDockerComposeCommand(task manifest.DockerCompose, team string) string } for k, v := range task.Vars { - s = append(s, fmt.Sprintf("-e %s=%s", k, vaultLookup(v, team))) + s = append(s, fmt.Sprintf("-e %s=%s", k, convertSecret(v, team))) } s = append(s, "--use-aliases", task.Service) @@ -66,71 +67,10 @@ func renderDockerComposeCommand(task manifest.DockerCompose, team string) string return strings.Join(s, " \\ \n ") } -func vaultLookup(s string, team string) string { - if !isSecret(s) { +func convertSecret(s string, team string) string { + secret := secrets.New(s, team) + if secret == nil { return s } - s = strings.TrimSpace(s[2 : len(s)-2]) - - if isKeyValueSecret(s) { - parts := strings.Split(s, ".") - vaultFolder := team - if isShared(parts[0]) { - vaultFolder = "shared" - } - return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s/%s)", parts[1], vaultFolder, parts[0]) - } - - if isAbsolutePathSecret(s) { - parts := strings.Split(s, " ") - return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s/%s)", parts[1], team, parts[0]) - } - - return s -} - -// ************************************************** -// all this copied from renderers/actions/secrets.go - -// check if a secret matches one of the shared secrets -// vault kv list /springernature/shared -func isShared(s string) bool { - return map[string]bool{ - "PPG-gradle-version-reporter": true, - "PPG-owasp-dependency-reporter": true, - "artifactory": true, - "artifactory-support": true, - "artifactory_test": true, - "bla": true, - "burpsuiteenterprise": true, - "content_hub-casper-credentials-live": true, - "content_hub-casper-credentials-qa": true, - "contrastsecurity": true, - "eas-sigrid": true, - "ee-sso-route-service": true, - "fastly": true, - "grafana": true, - "halfpipe-artifacts": true, - "halfpipe-docker-config": true, - "halfpipe-gcr": true, - "halfpipe-github": true, - "halfpipe-ml-deploy": true, - "halfpipe-semver": true, - "halfpipe-slack": true, - "katee-tls-dev": true, - "katee-tls-prod": true, - "sentry-release-integration": true, - }[s] -} - -func isSecret(s string) bool { - return strings.HasPrefix(s, "((") && strings.HasSuffix(s, "))") -} - -func isAbsolutePathSecret(s string) bool { - return len(strings.Split(s, " ")) == 2 -} - -func isKeyValueSecret(s string) bool { - return len(strings.Split(s, ".")) == 2 + return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s)", secret.Key, secret.MapPath) } From 6008e20d1d85c60c4127eaa4c5253cb66073dc35 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Mon, 2 Oct 2023 11:10:47 +0100 Subject: [PATCH 26/40] sort the output vars in shell renderer --- renderers/shell/shell.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/renderers/shell/shell.go b/renderers/shell/shell.go index bdf3a92c..a65881a2 100644 --- a/renderers/shell/shell.go +++ b/renderers/shell/shell.go @@ -5,6 +5,7 @@ import ( "github.com/springernature/halfpipe" "github.com/springernature/halfpipe/manifest" "github.com/springernature/halfpipe/renderers/shared/secrets" + "sort" "strings" ) @@ -36,9 +37,12 @@ func renderRunCommand(task manifest.Run, team string) string { "-w /app", } + vars := []string{} for k, v := range task.Vars { - s = append(s, fmt.Sprintf("-e %s=%s", k, convertSecret(v, team))) + vars = append(vars, fmt.Sprintf("-e %s=%s", k, convertSecret(v, team))) } + sort.Strings(vars) + s = append(s, vars...) s = append(s, task.Docker.Image, task.Script) @@ -54,9 +58,12 @@ func renderDockerComposeCommand(task manifest.DockerCompose, team string) string "-w /app", } + vars := []string{} for k, v := range task.Vars { - s = append(s, fmt.Sprintf("-e %s=%s", k, convertSecret(v, team))) + vars = append(vars, fmt.Sprintf("-e %s=%s", k, convertSecret(v, team))) } + sort.Strings(vars) + s = append(s, vars...) s = append(s, "--use-aliases", task.Service) From 5228064c489f0350616d006e54cd6410fa4f0061 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Mon, 2 Oct 2023 11:11:57 +0100 Subject: [PATCH 27/40] tests for shell renderer sad path --- renderers/shell/shell_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 renderers/shell/shell_test.go diff --git a/renderers/shell/shell_test.go b/renderers/shell/shell_test.go new file mode 100644 index 00000000..918e0cd4 --- /dev/null +++ b/renderers/shell/shell_test.go @@ -0,0 +1,25 @@ +package shell + +import ( + "github.com/springernature/halfpipe/manifest" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestShell_Render_SadPath(t *testing.T) { + + t.Run("task doesn't exist", func(t *testing.T) { + renderer := New("task name that doesn't exist") + actual, err := renderer.Render(manifest.Manifest{Tasks: manifest.TaskList{manifest.Run{Name: "task name"}}}) + assert.Error(t, err) + assert.Empty(t, actual) + }) + + t.Run("task exists but type not supported", func(t *testing.T) { + renderer := New("task name") + actual, err := renderer.Render(manifest.Manifest{Tasks: manifest.TaskList{manifest.DockerPush{Name: "task name"}}}) + assert.Error(t, err) + assert.Empty(t, actual) + }) + +} From 5963a8f77bd2b7a5dd4becbd1518b4cb4e0afc63 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Mon, 2 Oct 2023 11:12:24 +0100 Subject: [PATCH 28/40] e2e tests for shell renderer --- .gitignore | 1 + cmd/cmds/actions-migration-help.go | 2 +- cmd/cmds/exec.go | 8 ++- cmd/cmds/root.go | 6 ++- controller.go | 8 +-- controller_test.go | 10 ++-- e2e/shell/all/.halfpipe.io | 33 ++++++++++++ e2e/shell/all/Dockerfile | 1 + e2e/shell/all/custom-docker-compose.yml | 5 ++ .../all/docker-compose-complex_expected.txt | 17 +++++++ .../all/docker-compose-simple_expected.txt | 11 ++++ e2e/shell/all/docker-compose.yml | 5 ++ e2e/shell/all/run_expected.txt | 14 ++++++ e2e/shell/all/test.sh | 8 +++ e2e/test.sh | 50 ++++++++----------- 15 files changed, 138 insertions(+), 41 deletions(-) create mode 100644 e2e/shell/all/.halfpipe.io create mode 100644 e2e/shell/all/Dockerfile create mode 100644 e2e/shell/all/custom-docker-compose.yml create mode 100644 e2e/shell/all/docker-compose-complex_expected.txt create mode 100644 e2e/shell/all/docker-compose-simple_expected.txt create mode 100644 e2e/shell/all/docker-compose.yml create mode 100644 e2e/shell/all/run_expected.txt create mode 100755 e2e/shell/all/test.sh diff --git a/.gitignore b/.gitignore index d9c9f157..2f682bc2 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ vendor pipelineActual.yml workflowActual.yml e2e/concourse/init/.halfpipe.io +e2e/shell/all/*_actual.txt diff --git a/cmd/cmds/actions-migration-help.go b/cmd/cmds/actions-migration-help.go index 3b909060..6353b4f6 100644 --- a/cmd/cmds/actions-migration-help.go +++ b/cmd/cmds/actions-migration-help.go @@ -15,7 +15,7 @@ var actionsMigrationHelp = &cobra.Command{ Short: "Prints out the steps needed to migrate from Concourse to Actions", Run: func(cmd *cobra.Command, args []string) { man, controller := getManifestAndController(formatInput(Input), nil) - response := controller.Process(man) + response, _ := controller.Process(man) tpl, _ := template.New("").Parse(` To migrate a Concourse pipeline to Actions you must do the following steps diff --git a/cmd/cmds/exec.go b/cmd/cmds/exec.go index f6ed1cca..82828fa2 100644 --- a/cmd/cmds/exec.go +++ b/cmd/cmds/exec.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/spf13/cobra" "github.com/springernature/halfpipe/renderers/shell" + "os" ) func init() { @@ -20,9 +21,12 @@ var execCmd = &cobra.Command{ shellRenderer := shell.New(taskName) man, controller := getManifestAndController(formatInput(Input), shellRenderer) - response := controller.Process(man) + response, err := controller.Process(man) + if err != nil { + printErr(err) + os.Exit(1) + } outputLintResults(response.LintResults) fmt.Println(response) - }, } diff --git a/cmd/cmds/root.go b/cmd/cmds/root.go index b0ea1c62..22316058 100644 --- a/cmd/cmds/root.go +++ b/cmd/cmds/root.go @@ -12,7 +12,11 @@ var rootCmd = &cobra.Command{ Invoke without any arguments to lint your .halfpipe.io file and render a pipeline`, Run: func(cmd *cobra.Command, args []string) { man, controller := getManifestAndController(formatInput(Input), nil) - response := controller.Process(man) + response, err := controller.Process(man) + if err != nil { + printErr(err) + os.Exit(1) + } if man.Platform.IsActions() && output == "" { output = path.Join(response.Project.GitRootPath, ".github/workflows/", man.PipelineName()+".yml") diff --git a/controller.go b/controller.go index de6a0715..0a7b7fdc 100644 --- a/controller.go +++ b/controller.go @@ -9,7 +9,7 @@ import ( ) type Controller interface { - Process(man manifest.Manifest) Response + Process(man manifest.Manifest) (Response, error) DefaultAndMap(man manifest.Manifest) (updated manifest.Manifest, err error) } @@ -44,7 +44,7 @@ func NewController(defaulter defaults.Defaults, mapper mapper.Mapper, linters [] } } -func (c controller) Process(man manifest.Manifest) (response Response) { +func (c controller) Process(man manifest.Manifest) (response Response, err error) { defaultedManifest := c.defaulter.Apply(man) for _, linter := range c.linters { @@ -61,11 +61,11 @@ func (c controller) Process(man manifest.Manifest) (response Response) { return } - config, _ := c.renderer.Render(mappedManifest) + config, err := c.renderer.Render(mappedManifest) response.ConfigYaml = config response.Project = c.defaulter.Project response.Platform = man.Platform - return response + return } func (c controller) DefaultAndMap(man manifest.Manifest) (updated manifest.Manifest, err error) { diff --git a/controller_test.go b/controller_test.go index 7a02d695..a4e15991 100644 --- a/controller_test.go +++ b/controller_test.go @@ -41,14 +41,14 @@ func testController() controller { func TestWorksForHalfpipeFileWithYMLExtension(t *testing.T) { c := testController() - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Len(t, response.LintResults.Error(), 0) } func TestWorksForHalfpipeFile(t *testing.T) { c := testController() - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Len(t, response.LintResults.Error(), 0) } @@ -68,7 +68,7 @@ func TestAppliesAllLinters(t *testing.T) { linter2 := fakeLinter{errors.New("error from linter2")} c.linters = []linters.Linter{linter1, linter2} - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Empty(t, response.ConfigYaml) assert.Len(t, response.LintResults, 2) @@ -79,7 +79,7 @@ func TestAppliesAllLinters(t *testing.T) { func TestGivesBackConfigWhenLinterPasses(t *testing.T) { c := testController() - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Len(t, response.LintResults, 0) assert.Equal(t, "fake output", response.ConfigYaml) } @@ -95,7 +95,7 @@ func (f FakeMapper) Apply(original manifest.Manifest) (updated manifest.Manifest func TestGivesBackABadTestResultWhenAMapperFails(t *testing.T) { c := testController() c.mapper = FakeMapper{err: errors.New("blurgh")} - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Len(t, response.LintResults, 1) assert.True(t, response.LintResults.HasErrors()) diff --git a/e2e/shell/all/.halfpipe.io b/e2e/shell/all/.halfpipe.io new file mode 100644 index 00000000..881fee30 --- /dev/null +++ b/e2e/shell/all/.halfpipe.io @@ -0,0 +1,33 @@ +team: halfpipe-team +pipeline: pipeline-name + +feature_toggles: +- update-pipeline + +tasks: +- type: run + name: run + script: \echo hello + docker: + image: alpine:test + vars: + ENV1: 1234 + ENV2: ((secret.something)) + ENV3: '{"a": "b", "c": "d"}' + ENV4: ((another.secret)) + VERY_SECRET: blah + +- type: docker-compose + name: docker-compose-simple + +- type: docker-compose + name: docker-compose-complex + command: \echo hello + compose_file: custom-docker-compose.yml + service: customservice + vars: + ENV1: 1234 + ENV2: ((secret.something)) + ENV3: '{"a": "b", "c": "d"}' + ENV4: ((another.secret)) + VERY_SECRET: blah diff --git a/e2e/shell/all/Dockerfile b/e2e/shell/all/Dockerfile new file mode 100644 index 00000000..76737680 --- /dev/null +++ b/e2e/shell/all/Dockerfile @@ -0,0 +1 @@ +from alpine diff --git a/e2e/shell/all/custom-docker-compose.yml b/e2e/shell/all/custom-docker-compose.yml new file mode 100644 index 00000000..88f55fec --- /dev/null +++ b/e2e/shell/all/custom-docker-compose.yml @@ -0,0 +1,5 @@ +version: '3' + +services: + customservice: + image: appropriate/curl diff --git a/e2e/shell/all/docker-compose-complex_expected.txt b/e2e/shell/all/docker-compose-complex_expected.txt new file mode 100644 index 00000000..72e62ed5 --- /dev/null +++ b/e2e/shell/all/docker-compose-complex_expected.txt @@ -0,0 +1,17 @@ +docker compose \ + -f custom-docker-compose.yml \ + run \ + -v "$PWD":/app \ + -w /app \ + -e ARTIFACTORY_PASSWORD=$(vault kv get -field=password /springernature/shared/artifactory) \ + -e ARTIFACTORY_URL=$(vault kv get -field=url /springernature/shared/artifactory) \ + -e ARTIFACTORY_USERNAME=$(vault kv get -field=username /springernature/shared/artifactory) \ + -e ENV1=1234 \ + -e ENV2=$(vault kv get -field=something /springernature/halfpipe-team/secret) \ + -e ENV3={"a": "b", "c": "d"} \ + -e ENV4=$(vault kv get -field=secret /springernature/halfpipe-team/another) \ + -e RUNNING_IN_CI=true \ + -e VERY_SECRET=blah \ + --use-aliases \ + customservice \ + \echo hello diff --git a/e2e/shell/all/docker-compose-simple_expected.txt b/e2e/shell/all/docker-compose-simple_expected.txt new file mode 100644 index 00000000..fa8145d7 --- /dev/null +++ b/e2e/shell/all/docker-compose-simple_expected.txt @@ -0,0 +1,11 @@ +docker compose \ + -f docker-compose.yml \ + run \ + -v "$PWD":/app \ + -w /app \ + -e ARTIFACTORY_PASSWORD=$(vault kv get -field=password /springernature/shared/artifactory) \ + -e ARTIFACTORY_URL=$(vault kv get -field=url /springernature/shared/artifactory) \ + -e ARTIFACTORY_USERNAME=$(vault kv get -field=username /springernature/shared/artifactory) \ + -e RUNNING_IN_CI=true \ + --use-aliases \ + app diff --git a/e2e/shell/all/docker-compose.yml b/e2e/shell/all/docker-compose.yml new file mode 100644 index 00000000..e8c0c0f0 --- /dev/null +++ b/e2e/shell/all/docker-compose.yml @@ -0,0 +1,5 @@ +version: '3' + +services: + app: + image: appropriate/curl diff --git a/e2e/shell/all/run_expected.txt b/e2e/shell/all/run_expected.txt new file mode 100644 index 00000000..e8df368b --- /dev/null +++ b/e2e/shell/all/run_expected.txt @@ -0,0 +1,14 @@ +docker run -it \ + -v "$PWD":/app \ + -w /app \ + -e ARTIFACTORY_PASSWORD=$(vault kv get -field=password /springernature/shared/artifactory) \ + -e ARTIFACTORY_URL=$(vault kv get -field=url /springernature/shared/artifactory) \ + -e ARTIFACTORY_USERNAME=$(vault kv get -field=username /springernature/shared/artifactory) \ + -e ENV1=1234 \ + -e ENV2=$(vault kv get -field=something /springernature/halfpipe-team/secret) \ + -e ENV3={"a": "b", "c": "d"} \ + -e ENV4=$(vault kv get -field=secret /springernature/halfpipe-team/another) \ + -e RUNNING_IN_CI=true \ + -e VERY_SECRET=blah \ + alpine:test \ + \echo hello diff --git a/e2e/shell/all/test.sh b/e2e/shell/all/test.sh new file mode 100755 index 00000000..b9e78644 --- /dev/null +++ b/e2e/shell/all/test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +for f in `find . -name '*_expected.txt'`; do + taskName="${f:2:(-13)}" + echo " task name: $taskName" + ../../../halfpipe -q exec "$taskName" > "${f/expected/actual}" + diff -w "$f" "${f/expected/actual}" +done diff --git a/e2e/test.sh b/e2e/test.sh index ebda5859..4366053c 100755 --- a/e2e/test.sh +++ b/e2e/test.sh @@ -1,36 +1,30 @@ #!/usr/bin/env bash runTest() { dir=${1%*/} - yml="/tmp/halfpipe-e2e/$dir.yml" - log="/tmp/halfpipe-e2e/$dir.log" - ( - set -e - echo "* Running ${dir}" - cd ${dir} - if [[ -f test.sh ]]; then - ./test.sh - elif [[ -f workflowExpected.yml ]]; then - # actions - ../../../halfpipe -q -o workflowActual.yml - sed '6s/\"\"/main/' workflowActual.yml > $yml - diff --ignore-blank-lines $yml workflowExpected.yml - else - # concourse - ../../../halfpipe -q -o pipelineActual.yml - sed 's/ branch: ""/ branch: main/g' pipelineActual.yml | sed -E 's/(key:.+)\-$/\1/g' > $yml - diff --ignore-blank-lines $yml pipelineExpected.yml - if command -v fly > /dev/null; then - fly validate-pipeline -c pipelineActual.yml &> /dev/null - fi + echo "* Running ${dir}" + mkdir -p /tmp/halfpipe-e2e/$dir + tmpYml="/tmp/halfpipe-e2e/$dir.yml" + + cd ${dir} + if [[ -f test.sh ]]; then + ./test.sh + elif [[ -f workflowExpected.yml ]]; then + # actions + ../../../halfpipe -q -o workflowActual.yml + sed '6s/\"\"/main/' workflowActual.yml > $tmpYml + diff --ignore-blank-lines $tmpYml workflowExpected.yml + elif [[ -f pipelineExpected.yml ]]; then + # concourse + ../../../halfpipe -q -o pipelineActual.yml + sed 's/ branch: ""/ branch: main/g' pipelineActual.yml | sed -E 's/(key:.+)\-$/\1/g' > $tmpYml + diff --ignore-blank-lines $tmpYml pipelineExpected.yml + if command -v fly > /dev/null; then + fly validate-pipeline -c pipelineActual.yml &> /dev/null fi - ) &>> $log + fi } -export -f runTest rm -rf /tmp/halfpipe-e2e -mkdir -p /tmp/halfpipe-e2e/{actions,concourse} -ls -d */*/ | xargs -ID -P16 bash -c "runTest D" -RET_CODE=$? -cat /tmp/halfpipe-e2e/*/*.log -exit $RET_CODE +export -f runTest +ls -d */*/ | xargs -ID -P1 bash -c "runTest D || exit 1" From 9e6577f1cd84e95e438c00f4fa0d1cfecaefba70 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Mon, 2 Oct 2023 11:13:10 +0100 Subject: [PATCH 29/40] tweak readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 8a986acc..fd7490aa 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,7 @@ We also use [GitHub Actions](https://github.com/springernature/halfpipe/actions) dependabot will raise PRs. Alternatively, to manually update all deps: ```bash -go get -t -u ./... -go mod tidy +go get -t -u ./... && go mod tidy ``` ### GitHub actions From 84ed70cf7c045c6a0077a83c7dfa218638198de2 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Mon, 2 Oct 2023 11:13:41 +0100 Subject: [PATCH 30/40] bump deps --- go.mod | 4 ++-- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 45e3b22c..e13a6302 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( code.cloudfoundry.org/cli v7.1.0+incompatible github.com/blang/semver v3.5.1+incompatible github.com/cloudfoundry/bosh-cli v6.4.1+incompatible - github.com/cloudfoundry/bosh-utils v0.0.398 // indirect + github.com/cloudfoundry/bosh-utils v0.0.399 // indirect github.com/concourse/concourse v1.6.1-0.20230204041300-d289c02f878d github.com/cppforlife/go-patch v0.2.0 // indirect github.com/gookit/color v1.5.4 @@ -17,7 +17,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mbrevoort/cronexpr v0.0.0-20170805223836-93a834a6c23a github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/onsi/gomega v1.27.10 + github.com/onsi/gomega v1.28.0 github.com/pkg/errors v0.9.1 github.com/spf13/afero v1.10.0 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index d2b48677..767b798a 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudfoundry/bosh-cli v6.4.1+incompatible h1:n5/+NIF9QxvGINOrjh6DmO+GTen78MoCj5+LU9L8bR4= github.com/cloudfoundry/bosh-cli v6.4.1+incompatible/go.mod h1:rzIB+e1sn7wQL/TJ54bl/FemPKRhXby5BIMS3tLuWFM= -github.com/cloudfoundry/bosh-utils v0.0.398 h1:kPtAoO7GmwEFolaQDXDjz7rB5HTytYKoz4PCjaiewBI= -github.com/cloudfoundry/bosh-utils v0.0.398/go.mod h1:kVhLBy5Tnur0NOTtSNeERBp359KZAgZNPXt2md1fUHI= +github.com/cloudfoundry/bosh-utils v0.0.399 h1:FSPMpvGjr/F/snhc0yUPObGoGl6jn/ZqmKkoW6ZSjj4= +github.com/cloudfoundry/bosh-utils v0.0.399/go.mod h1:mLJ0MTXBI0uH9QC7PqLktoWbmqcTVM02IfQShcFZLDw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -140,8 +140,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8 h1:gpptm606MZYGaMHMsB4Srmb6EbW/IVHnt04rcMXnkBQ= -github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -196,8 +196,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c= +github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= From 612ca85ed929b782941a36ab4b2e92221abe6c84 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Mon, 2 Oct 2023 11:37:43 +0100 Subject: [PATCH 31/40] use parallel for the e2e tests, xargs is unreliable for some reason --- e2e/test.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/e2e/test.sh b/e2e/test.sh index 4366053c..724ccd21 100755 --- a/e2e/test.sh +++ b/e2e/test.sh @@ -27,4 +27,10 @@ runTest() { rm -rf /tmp/halfpipe-e2e export -f runTest -ls -d */*/ | xargs -ID -P1 bash -c "runTest D || exit 1" + +if command -v parallel > /dev/null; then + ls -d */*/ | parallel -j16 runTest +else + # xargs doesn't return exit code reliably with parallel + ls -d */*/ | xargs -ID -P1 bash -c "runTest D" +fi From 77a792b6f4277b6fef65483643c5be8f5e49c10e Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Mon, 2 Oct 2023 14:32:47 +0100 Subject: [PATCH 32/40] allow actions runner name to be overridden with env var --- renderers/actions/actions.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/renderers/actions/actions.go b/renderers/actions/actions.go index 1fe796a4..a9fd8d2d 100644 --- a/renderers/actions/actions.go +++ b/renderers/actions/actions.go @@ -3,6 +3,7 @@ package actions import ( "fmt" "golang.org/x/exp/slices" + "os" "regexp" "strings" "time" @@ -13,7 +14,16 @@ import ( "github.com/springernature/halfpipe/manifest" ) -const eeRunner = "ee-runner" +var eeRunner = "ee-runner" + +// allow runner name to be overridden with env var +func runnerName() string { + fromEnv := os.Getenv("HALFPIPE_ACTIONS_RUNNER") + if fromEnv != "" { + return fromEnv + } + return eeRunner +} var globalEnv = Env{ "ARTIFACTORY_PASSWORD": githubSecrets.ArtifactoryPassword, @@ -81,7 +91,7 @@ func (a *Actions) jobs(tasks manifest.TaskList, man manifest.Manifest, parent *p job := Job{ Name: task.GetName(), - RunsOn: eeRunner, + RunsOn: runnerName(), Steps: convertSecrets(steps, man.Team), TimeoutMinutes: timeoutInMinutes(task.GetTimeout()), Needs: needs, From 32bf9e4b9382767de599c1290d3acd5b4d03ad23 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Mon, 2 Oct 2023 14:58:26 +0100 Subject: [PATCH 33/40] set the actions runner in config --- config/config.go | 2 ++ renderers/actions/actions.go | 14 +------------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/config/config.go b/config/config.go index 2a7edf76..f2d3114a 100644 --- a/config/config.go +++ b/config/config.go @@ -25,6 +25,8 @@ var ( ConcourseURL = "https://concourse." + Domain + ActionsRunnerName = getEnv("HALFPIPE_ACTIONS_RUNNER", "ee-runner") + CacheDirs = []string{ "../../../var/halfpipe/cache", "../../../halfpipe-cache", // deprecated and should be removed after a while diff --git a/renderers/actions/actions.go b/renderers/actions/actions.go index a9fd8d2d..00327712 100644 --- a/renderers/actions/actions.go +++ b/renderers/actions/actions.go @@ -3,7 +3,6 @@ package actions import ( "fmt" "golang.org/x/exp/slices" - "os" "regexp" "strings" "time" @@ -14,17 +13,6 @@ import ( "github.com/springernature/halfpipe/manifest" ) -var eeRunner = "ee-runner" - -// allow runner name to be overridden with env var -func runnerName() string { - fromEnv := os.Getenv("HALFPIPE_ACTIONS_RUNNER") - if fromEnv != "" { - return fromEnv - } - return eeRunner -} - var globalEnv = Env{ "ARTIFACTORY_PASSWORD": githubSecrets.ArtifactoryPassword, "ARTIFACTORY_URL": githubSecrets.ArtifactoryURL, @@ -91,7 +79,7 @@ func (a *Actions) jobs(tasks manifest.TaskList, man manifest.Manifest, parent *p job := Job{ Name: task.GetName(), - RunsOn: runnerName(), + RunsOn: config.ActionsRunnerName, Steps: convertSecrets(steps, man.Team), TimeoutMinutes: timeoutInMinutes(task.GetTimeout()), Needs: needs, From d84dc9485ac83b37461de62a13082ef3a7cc73e4 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Mon, 9 Oct 2023 14:36:34 +0100 Subject: [PATCH 34/40] bump deps --- .github/workflows/codeql.yml | 2 +- .github/workflows/for-dependabot-to-check.yml | 2 +- e2e/actions/deploy-katee/workflowExpected.yml | 2 +- e2e/actions/docker-push/workflowExpected.yml | 10 +++---- go.mod | 10 +++---- go.sum | 28 +++++++++---------- renderers/actions/docker_push.go | 5 ++-- 7 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index fe8c5c4b..ae8ae38d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/.github/workflows/for-dependabot-to-check.yml b/.github/workflows/for-dependabot-to-check.yml index 3bc1138d..08b61743 100644 --- a/.github/workflows/for-dependabot-to-check.yml +++ b/.github/workflows/for-dependabot-to-check.yml @@ -163,7 +163,7 @@ jobs: ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }} submodules: recursive - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | ARTIFACTORY_PASSWORD diff --git a/e2e/actions/deploy-katee/workflowExpected.yml b/e2e/actions/deploy-katee/workflowExpected.yml index f6f49dfb..c0299854 100644 --- a/e2e/actions/deploy-katee/workflowExpected.yml +++ b/e2e/actions/deploy-katee/workflowExpected.yml @@ -35,7 +35,7 @@ jobs: ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }} submodules: recursive - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | ARTIFACTORY_PASSWORD diff --git a/e2e/actions/docker-push/workflowExpected.yml b/e2e/actions/docker-push/workflowExpected.yml index 583fe0f2..b58b37ee 100644 --- a/e2e/actions/docker-push/workflowExpected.yml +++ b/e2e/actions/docker-push/workflowExpected.yml @@ -70,7 +70,7 @@ jobs: run: tar -xvf halfpipe-artifacts.tar; rm halfpipe-artifacts.tar working-directory: ${{ github.workspace }} - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | ARTIFACTORY_PASSWORD @@ -153,7 +153,7 @@ jobs: password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_foo_bar }} username: user - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | ARTIFACTORY_PASSWORD @@ -222,7 +222,7 @@ jobs: run: tar -xvf halfpipe-artifacts.tar; rm halfpipe-artifacts.tar working-directory: ${{ github.workspace }} - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | ARTIFACTORY_PASSWORD @@ -288,7 +288,7 @@ jobs: run: tar -xvf halfpipe-artifacts.tar; rm halfpipe-artifacts.tar working-directory: ${{ github.workspace }} - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | ARTIFACTORY_PASSWORD @@ -362,7 +362,7 @@ jobs: ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }} submodules: recursive - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | A=a diff --git a/go.mod b/go.mod index e13a6302..f6281627 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( code.cloudfoundry.org/cli v7.1.0+incompatible github.com/blang/semver v3.5.1+incompatible github.com/cloudfoundry/bosh-cli v6.4.1+incompatible - github.com/cloudfoundry/bosh-utils v0.0.399 // indirect + github.com/cloudfoundry/bosh-utils v0.0.402 // indirect github.com/concourse/concourse v1.6.1-0.20230204041300-d289c02f878d github.com/cppforlife/go-patch v0.2.0 // indirect github.com/gookit/color v1.5.4 @@ -30,7 +30,7 @@ require ( require ( github.com/sirupsen/logrus v1.9.3 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d gopkg.in/yaml.v3 v3.0.1 ) @@ -50,9 +50,9 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/tedsuo/rata v1.0.1-0.20170830210128-07d200713958 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/net v0.15.0 // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.16.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 767b798a..e52c46a2 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudfoundry/bosh-cli v6.4.1+incompatible h1:n5/+NIF9QxvGINOrjh6DmO+GTen78MoCj5+LU9L8bR4= github.com/cloudfoundry/bosh-cli v6.4.1+incompatible/go.mod h1:rzIB+e1sn7wQL/TJ54bl/FemPKRhXby5BIMS3tLuWFM= -github.com/cloudfoundry/bosh-utils v0.0.399 h1:FSPMpvGjr/F/snhc0yUPObGoGl6jn/ZqmKkoW6ZSjj4= -github.com/cloudfoundry/bosh-utils v0.0.399/go.mod h1:mLJ0MTXBI0uH9QC7PqLktoWbmqcTVM02IfQShcFZLDw= +github.com/cloudfoundry/bosh-utils v0.0.402 h1:uvVgTcbaF0x04sVG6kyK+OgeVP+PZzv5F/R7dJFKBx8= +github.com/cloudfoundry/bosh-utils v0.0.402/go.mod h1:TiXvpLZjeUXGEdUGhY8Sdnv51h8/g5X0EJ+8HLqApCM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -248,8 +248,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -260,8 +260,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -317,8 +317,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -377,11 +377,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -442,8 +442,8 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/renderers/actions/docker_push.go b/renderers/actions/docker_push.go index 64f001a9..c3637002 100644 --- a/renderers/actions/docker_push.go +++ b/renderers/actions/docker_push.go @@ -2,10 +2,11 @@ package actions import ( "fmt" - "github.com/springernature/halfpipe/renderers/shared" "path" "strings" + "github.com/springernature/halfpipe/renderers/shared" + "github.com/springernature/halfpipe/manifest" ) @@ -53,7 +54,7 @@ func buildImage(a *Actions, task manifest.DockerPush) Step { step := Step{ Name: "Build Image", - Uses: "docker/build-push-action@v4", + Uses: "docker/build-push-action@v5", With: With{ "context": path.Join(a.workingDir, task.BuildPath), "file": path.Join(a.workingDir, task.DockerfilePath), From 1597f3a88cc65411ac880ba06b188231d2b7e978 Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 11 Oct 2023 16:34:35 +0100 Subject: [PATCH 35/40] better error message for 'halfpipe exec' --- renderers/shell/shell.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/renderers/shell/shell.go b/renderers/shell/shell.go index a65881a2..d0c6b442 100644 --- a/renderers/shell/shell.go +++ b/renderers/shell/shell.go @@ -27,7 +27,14 @@ func (s shell) Render(man manifest.Manifest) (string, error) { return renderDockerComposeCommand(t, man.Team), nil } - return "", fmt.Errorf("task not found with name '%s' and type 'run' or 'docker-compose'", s.taskName) + errMsg := "task not found with name '%s' and type 'run' or 'docker-compose\n\navailable tasks:\n" + for _, t := range man.Tasks.Flatten() { + switch t := t.(type) { + case manifest.Run, manifest.DockerCompose: + errMsg += fmt.Sprintf(" %s", t.GetName()) + } + } + return "", fmt.Errorf(errMsg, s.taskName) } func renderRunCommand(task manifest.Run, team string) string { From b39ff9fc3d0f14e249f38406947eab62e43b746a Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 11 Oct 2023 16:50:06 +0100 Subject: [PATCH 36/40] halfpipe exec - quote variables --- .../all/docker-compose-complex_expected.txt | 18 +++++++++--------- .../all/docker-compose-simple_expected.txt | 8 ++++---- e2e/shell/all/run_expected.txt | 18 +++++++++--------- renderers/shell/shell.go | 4 ++-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/e2e/shell/all/docker-compose-complex_expected.txt b/e2e/shell/all/docker-compose-complex_expected.txt index 72e62ed5..f52f444a 100644 --- a/e2e/shell/all/docker-compose-complex_expected.txt +++ b/e2e/shell/all/docker-compose-complex_expected.txt @@ -3,15 +3,15 @@ docker compose \ run \ -v "$PWD":/app \ -w /app \ - -e ARTIFACTORY_PASSWORD=$(vault kv get -field=password /springernature/shared/artifactory) \ - -e ARTIFACTORY_URL=$(vault kv get -field=url /springernature/shared/artifactory) \ - -e ARTIFACTORY_USERNAME=$(vault kv get -field=username /springernature/shared/artifactory) \ - -e ENV1=1234 \ - -e ENV2=$(vault kv get -field=something /springernature/halfpipe-team/secret) \ - -e ENV3={"a": "b", "c": "d"} \ - -e ENV4=$(vault kv get -field=secret /springernature/halfpipe-team/another) \ - -e RUNNING_IN_CI=true \ - -e VERY_SECRET=blah \ + -e ARTIFACTORY_PASSWORD="$(vault kv get -field=password /springernature/shared/artifactory)" \ + -e ARTIFACTORY_URL="$(vault kv get -field=url /springernature/shared/artifactory)" \ + -e ARTIFACTORY_USERNAME="$(vault kv get -field=username /springernature/shared/artifactory)" \ + -e ENV1="1234" \ + -e ENV2="$(vault kv get -field=something /springernature/halfpipe-team/secret)" \ + -e ENV3="{"a": "b", "c": "d"}" \ + -e ENV4="$(vault kv get -field=secret /springernature/halfpipe-team/another)" \ + -e RUNNING_IN_CI="true" \ + -e VERY_SECRET="blah" \ --use-aliases \ customservice \ \echo hello diff --git a/e2e/shell/all/docker-compose-simple_expected.txt b/e2e/shell/all/docker-compose-simple_expected.txt index fa8145d7..bd5995fc 100644 --- a/e2e/shell/all/docker-compose-simple_expected.txt +++ b/e2e/shell/all/docker-compose-simple_expected.txt @@ -3,9 +3,9 @@ docker compose \ run \ -v "$PWD":/app \ -w /app \ - -e ARTIFACTORY_PASSWORD=$(vault kv get -field=password /springernature/shared/artifactory) \ - -e ARTIFACTORY_URL=$(vault kv get -field=url /springernature/shared/artifactory) \ - -e ARTIFACTORY_USERNAME=$(vault kv get -field=username /springernature/shared/artifactory) \ - -e RUNNING_IN_CI=true \ + -e ARTIFACTORY_PASSWORD="$(vault kv get -field=password /springernature/shared/artifactory)" \ + -e ARTIFACTORY_URL="$(vault kv get -field=url /springernature/shared/artifactory)" \ + -e ARTIFACTORY_USERNAME="$(vault kv get -field=username /springernature/shared/artifactory)" \ + -e RUNNING_IN_CI="true" \ --use-aliases \ app diff --git a/e2e/shell/all/run_expected.txt b/e2e/shell/all/run_expected.txt index e8df368b..c0304d54 100644 --- a/e2e/shell/all/run_expected.txt +++ b/e2e/shell/all/run_expected.txt @@ -1,14 +1,14 @@ docker run -it \ -v "$PWD":/app \ -w /app \ - -e ARTIFACTORY_PASSWORD=$(vault kv get -field=password /springernature/shared/artifactory) \ - -e ARTIFACTORY_URL=$(vault kv get -field=url /springernature/shared/artifactory) \ - -e ARTIFACTORY_USERNAME=$(vault kv get -field=username /springernature/shared/artifactory) \ - -e ENV1=1234 \ - -e ENV2=$(vault kv get -field=something /springernature/halfpipe-team/secret) \ - -e ENV3={"a": "b", "c": "d"} \ - -e ENV4=$(vault kv get -field=secret /springernature/halfpipe-team/another) \ - -e RUNNING_IN_CI=true \ - -e VERY_SECRET=blah \ + -e ARTIFACTORY_PASSWORD="$(vault kv get -field=password /springernature/shared/artifactory)" \ + -e ARTIFACTORY_URL="$(vault kv get -field=url /springernature/shared/artifactory)" \ + -e ARTIFACTORY_USERNAME="$(vault kv get -field=username /springernature/shared/artifactory)" \ + -e ENV1="1234" \ + -e ENV2="$(vault kv get -field=something /springernature/halfpipe-team/secret)" \ + -e ENV3="{"a": "b", "c": "d"}" \ + -e ENV4="$(vault kv get -field=secret /springernature/halfpipe-team/another)" \ + -e RUNNING_IN_CI="true" \ + -e VERY_SECRET="blah" \ alpine:test \ \echo hello diff --git a/renderers/shell/shell.go b/renderers/shell/shell.go index d0c6b442..95781d6c 100644 --- a/renderers/shell/shell.go +++ b/renderers/shell/shell.go @@ -46,7 +46,7 @@ func renderRunCommand(task manifest.Run, team string) string { vars := []string{} for k, v := range task.Vars { - vars = append(vars, fmt.Sprintf("-e %s=%s", k, convertSecret(v, team))) + vars = append(vars, fmt.Sprintf(`-e %s="%s"`, k, convertSecret(v, team))) } sort.Strings(vars) s = append(s, vars...) @@ -67,7 +67,7 @@ func renderDockerComposeCommand(task manifest.DockerCompose, team string) string vars := []string{} for k, v := range task.Vars { - vars = append(vars, fmt.Sprintf("-e %s=%s", k, convertSecret(v, team))) + vars = append(vars, fmt.Sprintf(`-e %s="%s"`, k, convertSecret(v, team))) } sort.Strings(vars) s = append(s, vars...) From 55ff4678f3373532ffe8ca9b7137b709093cfb6c Mon Sep 17 00:00:00 2001 From: Rob Whitby Date: Wed, 11 Oct 2023 16:55:04 +0100 Subject: [PATCH 37/40] update halfpipe exec run test --- e2e/shell/all/.halfpipe.io | 2 +- e2e/shell/all/run_expected.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/shell/all/.halfpipe.io b/e2e/shell/all/.halfpipe.io index 881fee30..fd3de502 100644 --- a/e2e/shell/all/.halfpipe.io +++ b/e2e/shell/all/.halfpipe.io @@ -7,7 +7,7 @@ feature_toggles: tasks: - type: run name: run - script: \echo hello + script: ./test.sh docker: image: alpine:test vars: diff --git a/e2e/shell/all/run_expected.txt b/e2e/shell/all/run_expected.txt index c0304d54..272878d4 100644 --- a/e2e/shell/all/run_expected.txt +++ b/e2e/shell/all/run_expected.txt @@ -11,4 +11,4 @@ docker run -it \ -e RUNNING_IN_CI="true" \ -e VERY_SECRET="blah" \ alpine:test \ - \echo hello + ./test.sh From 11ef4779309fca816acbae7f964532dc8fe91b48 Mon Sep 17 00:00:00 2001 From: Simon Johansson Date: Mon, 16 Oct 2023 12:35:52 +0200 Subject: [PATCH 38/40] github-deployment enables a deployment in github for actions --- e2e/actions/deploy-cf/.halfpipe.io | 22 ++- e2e/actions/deploy-cf/manifest-docker.yml | 2 +- e2e/actions/deploy-cf/manifest-worker.yml | 9 + e2e/actions/deploy-cf/manifest.yml | 2 +- e2e/actions/deploy-cf/workflowExpected.yml | 194 +++++++++++++++++---- manifest/deploy_cf.go | 2 +- manifest/feature_toggles.go | 9 +- manifest/secret_validator_test.go | 5 +- renderers/actions/actions.go | 15 ++ renderers/actions/workflow.go | 6 + 10 files changed, 218 insertions(+), 48 deletions(-) create mode 100644 e2e/actions/deploy-cf/manifest-worker.yml diff --git a/e2e/actions/deploy-cf/.halfpipe.io b/e2e/actions/deploy-cf/.halfpipe.io index 267e5ae9..899af036 100644 --- a/e2e/actions/deploy-cf/.halfpipe.io +++ b/e2e/actions/deploy-cf/.halfpipe.io @@ -2,6 +2,9 @@ team: halfpipe-team pipeline: pipeline-name platform: actions +feature_toggles: +- github-deployment + triggers: - type: git watched_paths: @@ -20,7 +23,7 @@ tasks: name: deploy to cf api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: dev + space: space1 manifest: manifest.yml deploy_artifact: foo.html vars: @@ -33,7 +36,7 @@ tasks: name: deploy to cf with cf8 api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: dev + space: space2 manifest: manifest.yml deploy_artifact: foo.html cli_version: cf8 @@ -47,7 +50,7 @@ tasks: name: deploy to cf with pre-promote api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: dev + space: space3 manifest: manifest.yml deploy_artifact: foo.html vars: @@ -70,7 +73,7 @@ tasks: name: deploy to cf with docker image api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: dev + space: space4 manifest: manifest-docker.yml docker_tag: version @@ -78,7 +81,7 @@ tasks: name: deploy with sso api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: dev + space: space5 deploy_artifact: foo.html sso_route: my-route.public.springernature.app rolling: true @@ -87,6 +90,13 @@ tasks: name: deploy without artifact api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: dev + space: space6 sso_route: my-route.public.springernature.app rolling: true + + - type: deploy-cf + name: deploy to cf worker app + api: ((cloudfoundry.api-snpaas)) + org: ((cloudfoundry.org-snpaas)) + space: space7 + manifest: manifest-worker.yml diff --git a/e2e/actions/deploy-cf/manifest-docker.yml b/e2e/actions/deploy-cf/manifest-docker.yml index 4c8292a8..8bb552cd 100644 --- a/e2e/actions/deploy-cf/manifest-docker.yml +++ b/e2e/actions/deploy-cf/manifest-docker.yml @@ -4,6 +4,6 @@ applications: instances: 1 memory: 32M routes: - - route: "test-route" + - route: my-route.public.springernature.app docker: image: eu.gcr.io/halfpipe-io/someImage diff --git a/e2e/actions/deploy-cf/manifest-worker.yml b/e2e/actions/deploy-cf/manifest-worker.yml new file mode 100644 index 00000000..2ff98f00 --- /dev/null +++ b/e2e/actions/deploy-cf/manifest-worker.yml @@ -0,0 +1,9 @@ +--- +applications: +- name: halfpipe-example-worker + instances: 1 + memory: 32M + no-route: true + health-check-type: process + buildpacks: + - java diff --git a/e2e/actions/deploy-cf/manifest.yml b/e2e/actions/deploy-cf/manifest.yml index 3ed7f866..1e1bd036 100644 --- a/e2e/actions/deploy-cf/manifest.yml +++ b/e2e/actions/deploy-cf/manifest.yml @@ -4,7 +4,7 @@ applications: instances: 1 memory: 32M routes: - - route: test-route - route: my-route.public.springernature.app + - route: test-route buildpacks: - java diff --git a/e2e/actions/deploy-cf/workflowExpected.yml b/e2e/actions/deploy-cf/workflowExpected.yml index 6ccda3fc..b2c05f16 100644 --- a/e2e/actions/deploy-cf/workflowExpected.yml +++ b/e2e/actions/deploy-cf/workflowExpected.yml @@ -54,6 +54,9 @@ jobs: - make_binary runs-on: ee-runner timeout-minutes: 60 + environment: + name: halfpipe-example/space1 + url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -95,7 +98,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space1 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -115,7 +118,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space1 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -128,7 +131,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space1 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -141,7 +144,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space1 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -160,7 +163,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space1 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_to_cf_with_cf8: @@ -169,6 +172,9 @@ jobs: - deploy_to_cf runs-on: ee-runner timeout-minutes: 60 + environment: + name: halfpipe-example/space2 + url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -210,7 +216,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space2 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -230,7 +236,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space2 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -243,7 +249,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space2 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -256,7 +262,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space2 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -275,7 +281,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space2 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_to_cf_with_pre-promote: @@ -284,6 +290,9 @@ jobs: - deploy_to_cf_with_cf8 runs-on: ee-runner timeout-minutes: 60 + environment: + name: halfpipe-example/space3 + url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -326,7 +335,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space3 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -344,7 +353,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space3 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -357,7 +366,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space3 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: run smoke-test.sh @@ -367,7 +376,7 @@ jobs: entrypoint: /bin/sh env: ENV5: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_some_secret }} - TEST_ROUTE: halfpipe-example-dev-CANDIDATE.springernature.app + TEST_ROUTE: halfpipe-example-space3-CANDIDATE.springernature.app - name: docker-compose run: |- docker-compose -f docker-compose.yml pull @@ -390,7 +399,7 @@ jobs: -v /mnt/halfpipe-cache/:/var/halfpipe/shared-cache \ app env: - TEST_ROUTE: halfpipe-example-dev-CANDIDATE.springernature.app + TEST_ROUTE: halfpipe-example-space3-CANDIDATE.springernature.app - name: Docker cleanup if: always() run: docker-compose -f docker-compose.yml down @@ -440,10 +449,10 @@ jobs: CONSUMER_SCRIPT: ci/run-external-and-cdcs-dev DOCKER_COMPOSE_SERVICE: "" GIT_CLONE_OPTIONS: "" - PROVIDER_HOST: halfpipe-example-dev-CANDIDATE.springernature.app + PROVIDER_HOST: halfpipe-example-space3-CANDIDATE.springernature.app PROVIDER_HOST_KEY: PIPELINE_NAME_DEPLOYED_HOST PROVIDER_NAME: pipeline-name - TEST_ROUTE: halfpipe-example-dev-CANDIDATE.springernature.app + TEST_ROUTE: halfpipe-example-space3-CANDIDATE.springernature.app - name: Promote uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -454,7 +463,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space3 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -473,7 +482,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space3 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_to_cf_with_docker_image: @@ -482,6 +491,9 @@ jobs: - deploy_to_cf_with_pre-promote runs-on: ee-runner timeout-minutes: 60 + environment: + name: halfpipe-example-docker/space4 + url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -518,7 +530,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space4 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -534,7 +546,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space4 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -547,7 +559,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space4 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -560,7 +572,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space4 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -579,7 +591,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space4 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_with_sso: @@ -588,6 +600,9 @@ jobs: - deploy_to_cf_with_docker_image runs-on: ee-runner timeout-minutes: 60 + environment: + name: halfpipe-example/space5 + url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -632,7 +647,7 @@ jobs: CF_API: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} CF_ORG: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} CF_PASSWORD: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - CF_SPACE: dev + CF_SPACE: space5 CF_USERNAME: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} SSO_HOST: my-route - name: Push @@ -645,7 +660,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space5 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -661,7 +676,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space5 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -674,7 +689,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space5 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -687,7 +702,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space5 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -706,7 +721,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space5 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_without_artifact: @@ -715,6 +730,9 @@ jobs: - deploy_with_sso runs-on: ee-runner timeout-minutes: 60 + environment: + name: halfpipe-example/space6 + url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -752,7 +770,7 @@ jobs: CF_API: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} CF_ORG: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} CF_PASSWORD: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - CF_SPACE: dev + CF_SPACE: space6 CF_USERNAME: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} SSO_HOST: my-route - name: Push @@ -765,7 +783,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space6 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -781,7 +799,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space6 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -794,7 +812,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space6 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -807,7 +825,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space6 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -826,6 +844,110 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: dev + space: space6 + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} + deploy_to_cf_worker_app: + name: deploy to cf worker app + needs: + - deploy_without_artifact + runs-on: ee-runner + timeout-minutes: 60 + environment: + name: halfpipe-example-worker/space7 + steps: + - name: Vault secrets + id: secrets + uses: hashicorp/vault-action@v2.7.3 + with: + exportEnv: false + method: approle + roleId: ${{ env.VAULT_ROLE_ID }} + secretId: ${{ env.VAULT_SECRET_ID }} + secrets: | + /springernature/data/halfpipe-team/cloudfoundry api-snpaas | springernature_data_halfpipe-team_cloudfoundry_api-snpaas ; + /springernature/data/halfpipe-team/cloudfoundry org-snpaas | springernature_data_halfpipe-team_cloudfoundry_org-snpaas ; + /springernature/data/halfpipe-team/cloudfoundry password-snpaas | springernature_data_halfpipe-team_cloudfoundry_password-snpaas ; + /springernature/data/halfpipe-team/cloudfoundry username-snpaas | springernature_data_halfpipe-team_cloudfoundry_username-snpaas ; + url: https://vault.halfpipe.io + - name: Checkout code + uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }} + submodules: recursive + - name: Push + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf + cli_version: cf7 + command: halfpipe-push + manifestPath: e2e/actions/deploy-cf/manifest-worker.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: space7 + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} + env: + CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest-worker.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: space7 + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} + - name: Check + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf + cli_version: cf7 + command: halfpipe-check + manifestPath: e2e/actions/deploy-cf/manifest-worker.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: space7 + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} + - name: Promote + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf + cli_version: cf7 + command: halfpipe-promote + manifestPath: e2e/actions/deploy-cf/manifest-worker.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: space7 + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} + - name: Summary + run: |- + echo ":rocket: **Deployment Successful**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "[SNPaaS Mission Control](https://mission-control.snpaas.eu/)" >> $GITHUB_STEP_SUMMARY + - name: Cleanup + if: always() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf + cli_version: cf7 + command: halfpipe-cleanup + manifestPath: e2e/actions/deploy-cf/manifest-worker.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: space7 testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} diff --git a/manifest/deploy_cf.go b/manifest/deploy_cf.go index 36f812c2..c4c084fb 100644 --- a/manifest/deploy_cf.go +++ b/manifest/deploy_cf.go @@ -11,7 +11,7 @@ type DeployCF struct { Name string `yaml:"name,omitempty"` ManualTrigger bool `json:"manual_trigger" yaml:"manual_trigger,omitempty"` API string `yaml:"api,omitempty" secretAllowed:"true"` - Space string `yaml:"space,omitempty" secretAllowed:"true"` + Space string `yaml:"space,omitempty"` Org string `yaml:"org,omitempty" secretAllowed:"true"` Username string `yaml:"username,omitempty" secretAllowed:"true"` Password string `yaml:"password,omitempty" secretAllowed:"true"` diff --git a/manifest/feature_toggles.go b/manifest/feature_toggles.go index a43019d9..5d86154d 100644 --- a/manifest/feature_toggles.go +++ b/manifest/feature_toggles.go @@ -1,6 +1,8 @@ package manifest -import "golang.org/x/exp/slices" +import ( + "golang.org/x/exp/slices" +) type FeatureToggles []string @@ -8,6 +10,7 @@ const ( FeatureUpdatePipeline = "update-pipeline" FeatureUpdatePipelineAndTag = "update-pipeline-and-tag" FeatureGithubStatuses = "github-statuses" + FeatureGithubDeployment = "github-deployment" ) var AvailableFeatureToggles = FeatureToggles{ @@ -27,3 +30,7 @@ func (f FeatureToggles) TagGitRepo() bool { func (f FeatureToggles) GithubStatuses() bool { return slices.Contains(f, FeatureGithubStatuses) } + +func (f FeatureToggles) GithubDeployment() bool { + return slices.Contains(f, FeatureGithubDeployment) +} diff --git a/manifest/secret_validator_test.go b/manifest/secret_validator_test.go index 4c24c073..2886108f 100644 --- a/manifest/secret_validator_test.go +++ b/manifest/secret_validator_test.go @@ -285,7 +285,7 @@ func TestDeployCF(t *testing.T) { Type: "((not.ok))", Name: "((not.ok))", API: "((super.ok))", - Space: "((super.ok))", + Space: "((not.ok))", Org: "((super.ok))", Username: "((super.ok))", Password: "((super.ok))", @@ -302,7 +302,7 @@ func TestDeployCF(t *testing.T) { } errors := secretValidator.Validate(bad) - assert.Len(t, errors, 8) + assert.Len(t, errors, 9) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].type")) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].name")) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].manifest")) @@ -311,6 +311,7 @@ func TestDeployCF(t *testing.T) { assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].timeout")) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].notifications.on_failure[1]")) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].notifications.on_success[2]")) + assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].space")) badPrePromote := manifest.Manifest{ Tasks: manifest.TaskList{ diff --git a/renderers/actions/actions.go b/renderers/actions/actions.go index 00327712..eff7f3b5 100644 --- a/renderers/actions/actions.go +++ b/renderers/actions/actions.go @@ -85,6 +85,21 @@ func (a *Actions) jobs(tasks manifest.TaskList, man manifest.Manifest, parent *p Needs: needs, } + switch t := task.(type) { + case manifest.DeployCF: + if man.FeatureToggles.GithubDeployment() { + job.Environment = Environment{ + Name: fmt.Sprintf("%s/%s", t.CfApplication.Name, t.Space), + } + maybeRoutes := t.CfApplication.RemainingManifestFields["routes"] + if maybeRoutes != nil { + if len(maybeRoutes.([]any)) > 0 { + job.Environment.Url = fmt.Sprintf("https://%v", maybeRoutes.([]any)[0].(map[any]any)["route"]) + } + } + } + } + if job.Name == "update" { job.Outputs = Outputs{"synced": "${{ steps.sync.outputs.synced }}"} } diff --git a/renderers/actions/workflow.go b/renderers/actions/workflow.go index 4b1e9e1f..6b26073c 100644 --- a/renderers/actions/workflow.go +++ b/renderers/actions/workflow.go @@ -51,6 +51,11 @@ type Run struct { type Jobs yaml.MapSlice +type Environment struct { + Name string `yaml:"name,omitempty"` + Url string `yaml:"url,omitempty"` +} + type Job struct { Name string `yaml:"name,omitempty"` Needs []string `yaml:"needs,omitempty"` @@ -59,6 +64,7 @@ type Job struct { Container Container `yaml:"container,omitempty"` TimeoutMinutes int `yaml:"timeout-minutes,omitempty"` Outputs map[string]string `yaml:"outputs,omitempty"` + Environment Environment `yaml:"environment,omitempty"` Steps Steps `yaml:"steps,omitempty"` } From 14277a38f27a8694a1f32459c3d75253209459d9 Mon Sep 17 00:00:00 2001 From: Simon Johansson Date: Mon, 30 Oct 2023 09:11:30 +0100 Subject: [PATCH 39/40] Revert "github-deployment enables a deployment in github for actions" This reverts commit 11ef4779309fca816acbae7f964532dc8fe91b48. --- e2e/actions/deploy-cf/.halfpipe.io | 22 +-- e2e/actions/deploy-cf/manifest-docker.yml | 2 +- e2e/actions/deploy-cf/manifest-worker.yml | 9 - e2e/actions/deploy-cf/manifest.yml | 2 +- e2e/actions/deploy-cf/workflowExpected.yml | 194 ++++----------------- manifest/deploy_cf.go | 2 +- manifest/feature_toggles.go | 9 +- manifest/secret_validator_test.go | 5 +- renderers/actions/actions.go | 15 -- renderers/actions/workflow.go | 6 - 10 files changed, 48 insertions(+), 218 deletions(-) delete mode 100644 e2e/actions/deploy-cf/manifest-worker.yml diff --git a/e2e/actions/deploy-cf/.halfpipe.io b/e2e/actions/deploy-cf/.halfpipe.io index 899af036..267e5ae9 100644 --- a/e2e/actions/deploy-cf/.halfpipe.io +++ b/e2e/actions/deploy-cf/.halfpipe.io @@ -2,9 +2,6 @@ team: halfpipe-team pipeline: pipeline-name platform: actions -feature_toggles: -- github-deployment - triggers: - type: git watched_paths: @@ -23,7 +20,7 @@ tasks: name: deploy to cf api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: space1 + space: dev manifest: manifest.yml deploy_artifact: foo.html vars: @@ -36,7 +33,7 @@ tasks: name: deploy to cf with cf8 api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: space2 + space: dev manifest: manifest.yml deploy_artifact: foo.html cli_version: cf8 @@ -50,7 +47,7 @@ tasks: name: deploy to cf with pre-promote api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: space3 + space: dev manifest: manifest.yml deploy_artifact: foo.html vars: @@ -73,7 +70,7 @@ tasks: name: deploy to cf with docker image api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: space4 + space: dev manifest: manifest-docker.yml docker_tag: version @@ -81,7 +78,7 @@ tasks: name: deploy with sso api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: space5 + space: dev deploy_artifact: foo.html sso_route: my-route.public.springernature.app rolling: true @@ -90,13 +87,6 @@ tasks: name: deploy without artifact api: ((cloudfoundry.api-snpaas)) org: ((cloudfoundry.org-snpaas)) - space: space6 + space: dev sso_route: my-route.public.springernature.app rolling: true - - - type: deploy-cf - name: deploy to cf worker app - api: ((cloudfoundry.api-snpaas)) - org: ((cloudfoundry.org-snpaas)) - space: space7 - manifest: manifest-worker.yml diff --git a/e2e/actions/deploy-cf/manifest-docker.yml b/e2e/actions/deploy-cf/manifest-docker.yml index 8bb552cd..4c8292a8 100644 --- a/e2e/actions/deploy-cf/manifest-docker.yml +++ b/e2e/actions/deploy-cf/manifest-docker.yml @@ -4,6 +4,6 @@ applications: instances: 1 memory: 32M routes: - - route: my-route.public.springernature.app + - route: "test-route" docker: image: eu.gcr.io/halfpipe-io/someImage diff --git a/e2e/actions/deploy-cf/manifest-worker.yml b/e2e/actions/deploy-cf/manifest-worker.yml deleted file mode 100644 index 2ff98f00..00000000 --- a/e2e/actions/deploy-cf/manifest-worker.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -applications: -- name: halfpipe-example-worker - instances: 1 - memory: 32M - no-route: true - health-check-type: process - buildpacks: - - java diff --git a/e2e/actions/deploy-cf/manifest.yml b/e2e/actions/deploy-cf/manifest.yml index 1e1bd036..3ed7f866 100644 --- a/e2e/actions/deploy-cf/manifest.yml +++ b/e2e/actions/deploy-cf/manifest.yml @@ -4,7 +4,7 @@ applications: instances: 1 memory: 32M routes: - - route: my-route.public.springernature.app - route: test-route + - route: my-route.public.springernature.app buildpacks: - java diff --git a/e2e/actions/deploy-cf/workflowExpected.yml b/e2e/actions/deploy-cf/workflowExpected.yml index b2c05f16..6ccda3fc 100644 --- a/e2e/actions/deploy-cf/workflowExpected.yml +++ b/e2e/actions/deploy-cf/workflowExpected.yml @@ -54,9 +54,6 @@ jobs: - make_binary runs-on: ee-runner timeout-minutes: 60 - environment: - name: halfpipe-example/space1 - url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -98,7 +95,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space1 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -118,7 +115,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space1 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -131,7 +128,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space1 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -144,7 +141,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space1 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -163,7 +160,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space1 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_to_cf_with_cf8: @@ -172,9 +169,6 @@ jobs: - deploy_to_cf runs-on: ee-runner timeout-minutes: 60 - environment: - name: halfpipe-example/space2 - url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -216,7 +210,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space2 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -236,7 +230,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space2 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -249,7 +243,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space2 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -262,7 +256,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space2 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -281,7 +275,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space2 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_to_cf_with_pre-promote: @@ -290,9 +284,6 @@ jobs: - deploy_to_cf_with_cf8 runs-on: ee-runner timeout-minutes: 60 - environment: - name: halfpipe-example/space3 - url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -335,7 +326,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space3 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -353,7 +344,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space3 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -366,7 +357,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space3 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: run smoke-test.sh @@ -376,7 +367,7 @@ jobs: entrypoint: /bin/sh env: ENV5: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_some_secret }} - TEST_ROUTE: halfpipe-example-space3-CANDIDATE.springernature.app + TEST_ROUTE: halfpipe-example-dev-CANDIDATE.springernature.app - name: docker-compose run: |- docker-compose -f docker-compose.yml pull @@ -399,7 +390,7 @@ jobs: -v /mnt/halfpipe-cache/:/var/halfpipe/shared-cache \ app env: - TEST_ROUTE: halfpipe-example-space3-CANDIDATE.springernature.app + TEST_ROUTE: halfpipe-example-dev-CANDIDATE.springernature.app - name: Docker cleanup if: always() run: docker-compose -f docker-compose.yml down @@ -449,10 +440,10 @@ jobs: CONSUMER_SCRIPT: ci/run-external-and-cdcs-dev DOCKER_COMPOSE_SERVICE: "" GIT_CLONE_OPTIONS: "" - PROVIDER_HOST: halfpipe-example-space3-CANDIDATE.springernature.app + PROVIDER_HOST: halfpipe-example-dev-CANDIDATE.springernature.app PROVIDER_HOST_KEY: PIPELINE_NAME_DEPLOYED_HOST PROVIDER_NAME: pipeline-name - TEST_ROUTE: halfpipe-example-space3-CANDIDATE.springernature.app + TEST_ROUTE: halfpipe-example-dev-CANDIDATE.springernature.app - name: Promote uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -463,7 +454,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space3 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -482,7 +473,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space3 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_to_cf_with_docker_image: @@ -491,9 +482,6 @@ jobs: - deploy_to_cf_with_pre-promote runs-on: ee-runner timeout-minutes: 60 - environment: - name: halfpipe-example-docker/space4 - url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -530,7 +518,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space4 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -546,7 +534,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space4 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -559,7 +547,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space4 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -572,7 +560,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space4 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -591,7 +579,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest-docker.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space4 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_with_sso: @@ -600,9 +588,6 @@ jobs: - deploy_to_cf_with_docker_image runs-on: ee-runner timeout-minutes: 60 - environment: - name: halfpipe-example/space5 - url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -647,7 +632,7 @@ jobs: CF_API: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} CF_ORG: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} CF_PASSWORD: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - CF_SPACE: space5 + CF_SPACE: dev CF_USERNAME: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} SSO_HOST: my-route - name: Push @@ -660,7 +645,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space5 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -676,7 +661,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space5 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -689,7 +674,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space5 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -702,7 +687,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space5 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -721,7 +706,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space5 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} deploy_without_artifact: @@ -730,9 +715,6 @@ jobs: - deploy_with_sso runs-on: ee-runner timeout-minutes: 60 - environment: - name: halfpipe-example/space6 - url: https://my-route.public.springernature.app steps: - name: Vault secrets id: secrets @@ -770,7 +752,7 @@ jobs: CF_API: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} CF_ORG: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} CF_PASSWORD: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - CF_SPACE: space6 + CF_SPACE: dev CF_USERNAME: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} SSO_HOST: my-route - name: Push @@ -783,7 +765,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space6 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: @@ -799,7 +781,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space6 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check @@ -812,7 +794,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space6 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Promote @@ -825,7 +807,7 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space6 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Summary @@ -844,110 +826,6 @@ jobs: manifestPath: e2e/actions/deploy-cf/manifest.yml org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space6 - testDomain: springernature.app - username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - deploy_to_cf_worker_app: - name: deploy to cf worker app - needs: - - deploy_without_artifact - runs-on: ee-runner - timeout-minutes: 60 - environment: - name: halfpipe-example-worker/space7 - steps: - - name: Vault secrets - id: secrets - uses: hashicorp/vault-action@v2.7.3 - with: - exportEnv: false - method: approle - roleId: ${{ env.VAULT_ROLE_ID }} - secretId: ${{ env.VAULT_SECRET_ID }} - secrets: | - /springernature/data/halfpipe-team/cloudfoundry api-snpaas | springernature_data_halfpipe-team_cloudfoundry_api-snpaas ; - /springernature/data/halfpipe-team/cloudfoundry org-snpaas | springernature_data_halfpipe-team_cloudfoundry_org-snpaas ; - /springernature/data/halfpipe-team/cloudfoundry password-snpaas | springernature_data_halfpipe-team_cloudfoundry_password-snpaas ; - /springernature/data/halfpipe-team/cloudfoundry username-snpaas | springernature_data_halfpipe-team_cloudfoundry_username-snpaas ; - url: https://vault.halfpipe.io - - name: Checkout code - uses: actions/checkout@v4 - with: - lfs: true - show-progress: false - ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }} - submodules: recursive - - name: Push - uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable - with: - api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} - appPath: e2e/actions/deploy-cf - cli_version: cf7 - command: halfpipe-push - manifestPath: e2e/actions/deploy-cf/manifest-worker.yml - org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} - password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space7 - testDomain: springernature.app - username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - env: - CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - - name: cf logs --recent - if: failure() - uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable - with: - api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} - appPath: e2e/actions/deploy-cf - cli_version: cf7 - command: halfpipe-logs - manifestPath: e2e/actions/deploy-cf/manifest-worker.yml - org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} - password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space7 - testDomain: springernature.app - username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - - name: Check - uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable - with: - api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} - appPath: e2e/actions/deploy-cf - cli_version: cf7 - command: halfpipe-check - manifestPath: e2e/actions/deploy-cf/manifest-worker.yml - org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} - password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space7 - testDomain: springernature.app - username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - - name: Promote - uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable - with: - api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} - appPath: e2e/actions/deploy-cf - cli_version: cf7 - command: halfpipe-promote - manifestPath: e2e/actions/deploy-cf/manifest-worker.yml - org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} - password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space7 - testDomain: springernature.app - username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - - name: Summary - run: |- - echo ":rocket: **Deployment Successful**" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "[SNPaaS Mission Control](https://mission-control.snpaas.eu/)" >> $GITHUB_STEP_SUMMARY - - name: Cleanup - if: always() - uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable - with: - api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} - appPath: e2e/actions/deploy-cf - cli_version: cf7 - command: halfpipe-cleanup - manifestPath: e2e/actions/deploy-cf/manifest-worker.yml - org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} - password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} - space: space7 + space: dev testDomain: springernature.app username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} diff --git a/manifest/deploy_cf.go b/manifest/deploy_cf.go index c4c084fb..36f812c2 100644 --- a/manifest/deploy_cf.go +++ b/manifest/deploy_cf.go @@ -11,7 +11,7 @@ type DeployCF struct { Name string `yaml:"name,omitempty"` ManualTrigger bool `json:"manual_trigger" yaml:"manual_trigger,omitempty"` API string `yaml:"api,omitempty" secretAllowed:"true"` - Space string `yaml:"space,omitempty"` + Space string `yaml:"space,omitempty" secretAllowed:"true"` Org string `yaml:"org,omitempty" secretAllowed:"true"` Username string `yaml:"username,omitempty" secretAllowed:"true"` Password string `yaml:"password,omitempty" secretAllowed:"true"` diff --git a/manifest/feature_toggles.go b/manifest/feature_toggles.go index 5d86154d..a43019d9 100644 --- a/manifest/feature_toggles.go +++ b/manifest/feature_toggles.go @@ -1,8 +1,6 @@ package manifest -import ( - "golang.org/x/exp/slices" -) +import "golang.org/x/exp/slices" type FeatureToggles []string @@ -10,7 +8,6 @@ const ( FeatureUpdatePipeline = "update-pipeline" FeatureUpdatePipelineAndTag = "update-pipeline-and-tag" FeatureGithubStatuses = "github-statuses" - FeatureGithubDeployment = "github-deployment" ) var AvailableFeatureToggles = FeatureToggles{ @@ -30,7 +27,3 @@ func (f FeatureToggles) TagGitRepo() bool { func (f FeatureToggles) GithubStatuses() bool { return slices.Contains(f, FeatureGithubStatuses) } - -func (f FeatureToggles) GithubDeployment() bool { - return slices.Contains(f, FeatureGithubDeployment) -} diff --git a/manifest/secret_validator_test.go b/manifest/secret_validator_test.go index 2886108f..4c24c073 100644 --- a/manifest/secret_validator_test.go +++ b/manifest/secret_validator_test.go @@ -285,7 +285,7 @@ func TestDeployCF(t *testing.T) { Type: "((not.ok))", Name: "((not.ok))", API: "((super.ok))", - Space: "((not.ok))", + Space: "((super.ok))", Org: "((super.ok))", Username: "((super.ok))", Password: "((super.ok))", @@ -302,7 +302,7 @@ func TestDeployCF(t *testing.T) { } errors := secretValidator.Validate(bad) - assert.Len(t, errors, 9) + assert.Len(t, errors, 8) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].type")) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].name")) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].manifest")) @@ -311,7 +311,6 @@ func TestDeployCF(t *testing.T) { assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].timeout")) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].notifications.on_failure[1]")) assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].notifications.on_success[2]")) - assert.Contains(t, errors, manifest.UnsupportedSecretError("tasks[0].space")) badPrePromote := manifest.Manifest{ Tasks: manifest.TaskList{ diff --git a/renderers/actions/actions.go b/renderers/actions/actions.go index eff7f3b5..00327712 100644 --- a/renderers/actions/actions.go +++ b/renderers/actions/actions.go @@ -85,21 +85,6 @@ func (a *Actions) jobs(tasks manifest.TaskList, man manifest.Manifest, parent *p Needs: needs, } - switch t := task.(type) { - case manifest.DeployCF: - if man.FeatureToggles.GithubDeployment() { - job.Environment = Environment{ - Name: fmt.Sprintf("%s/%s", t.CfApplication.Name, t.Space), - } - maybeRoutes := t.CfApplication.RemainingManifestFields["routes"] - if maybeRoutes != nil { - if len(maybeRoutes.([]any)) > 0 { - job.Environment.Url = fmt.Sprintf("https://%v", maybeRoutes.([]any)[0].(map[any]any)["route"]) - } - } - } - } - if job.Name == "update" { job.Outputs = Outputs{"synced": "${{ steps.sync.outputs.synced }}"} } diff --git a/renderers/actions/workflow.go b/renderers/actions/workflow.go index 6b26073c..4b1e9e1f 100644 --- a/renderers/actions/workflow.go +++ b/renderers/actions/workflow.go @@ -51,11 +51,6 @@ type Run struct { type Jobs yaml.MapSlice -type Environment struct { - Name string `yaml:"name,omitempty"` - Url string `yaml:"url,omitempty"` -} - type Job struct { Name string `yaml:"name,omitempty"` Needs []string `yaml:"needs,omitempty"` @@ -64,7 +59,6 @@ type Job struct { Container Container `yaml:"container,omitempty"` TimeoutMinutes int `yaml:"timeout-minutes,omitempty"` Outputs map[string]string `yaml:"outputs,omitempty"` - Environment Environment `yaml:"environment,omitempty"` Steps Steps `yaml:"steps,omitempty"` } From 127f452a2ae35c77bf2d6967e3f415c4968349c9 Mon Sep 17 00:00:00 2001 From: Simon Johansson Date: Mon, 30 Oct 2023 09:47:10 +0100 Subject: [PATCH 40/40] Bump vault action --- .github/workflows/codeql.yml | 2 +- .github/workflows/for-dependabot-to-check.yml | 2 +- .../consumer-integration-test/workflowExpected.yml | 4 ++-- e2e/actions/deploy-cf/workflowExpected.yml | 12 ++++++------ e2e/actions/deploy-katee/workflowExpected.yml | 4 ++-- e2e/actions/deploy-ml/workflowExpected.yml | 2 +- e2e/actions/docker-compose/workflowExpected.yml | 2 +- e2e/actions/docker-push/workflowExpected.yml | 4 ++-- e2e/actions/run/workflowExpected.yml | 2 +- e2e/actions/trigger-git-options/workflowExpected.yml | 4 ++-- renderers/actions/secrets.go | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ae8ae38d..2e94c393 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/.github/workflows/for-dependabot-to-check.yml b/.github/workflows/for-dependabot-to-check.yml index 08b61743..cb6e17f1 100644 --- a/.github/workflows/for-dependabot-to-check.yml +++ b/.github/workflows/for-dependabot-to-check.yml @@ -47,7 +47,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/consumer-integration-test/workflowExpected.yml b/e2e/actions/consumer-integration-test/workflowExpected.yml index 8f98bbdc..45c3421e 100644 --- a/e2e/actions/consumer-integration-test/workflowExpected.yml +++ b/e2e/actions/consumer-integration-test/workflowExpected.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -103,7 +103,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/deploy-cf/workflowExpected.yml b/e2e/actions/deploy-cf/workflowExpected.yml index 6ccda3fc..3dfa1bd3 100644 --- a/e2e/actions/deploy-cf/workflowExpected.yml +++ b/e2e/actions/deploy-cf/workflowExpected.yml @@ -57,7 +57,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -172,7 +172,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -287,7 +287,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -485,7 +485,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -591,7 +591,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -718,7 +718,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/deploy-katee/workflowExpected.yml b/e2e/actions/deploy-katee/workflowExpected.yml index c0299854..982decd7 100644 --- a/e2e/actions/deploy-katee/workflowExpected.yml +++ b/e2e/actions/deploy-katee/workflowExpected.yml @@ -88,7 +88,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -141,7 +141,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/deploy-ml/workflowExpected.yml b/e2e/actions/deploy-ml/workflowExpected.yml index c4f03fa8..baec31cd 100644 --- a/e2e/actions/deploy-ml/workflowExpected.yml +++ b/e2e/actions/deploy-ml/workflowExpected.yml @@ -87,7 +87,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/docker-compose/workflowExpected.yml b/e2e/actions/docker-compose/workflowExpected.yml index c6a39224..63ac4c34 100644 --- a/e2e/actions/docker-compose/workflowExpected.yml +++ b/e2e/actions/docker-compose/workflowExpected.yml @@ -83,7 +83,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/docker-push/workflowExpected.yml b/e2e/actions/docker-push/workflowExpected.yml index b58b37ee..8ac5485b 100644 --- a/e2e/actions/docker-push/workflowExpected.yml +++ b/e2e/actions/docker-push/workflowExpected.yml @@ -123,7 +123,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -345,7 +345,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/run/workflowExpected.yml b/e2e/actions/run/workflowExpected.yml index 6d6046b6..0673712b 100644 --- a/e2e/actions/run/workflowExpected.yml +++ b/e2e/actions/run/workflowExpected.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/trigger-git-options/workflowExpected.yml b/e2e/actions/trigger-git-options/workflowExpected.yml index a9f39c52..63de160c 100644 --- a/e2e/actions/trigger-git-options/workflowExpected.yml +++ b/e2e/actions/trigger-git-options/workflowExpected.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -63,7 +63,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/renderers/actions/secrets.go b/renderers/actions/secrets.go index 1df62842..935d9841 100644 --- a/renderers/actions/secrets.go +++ b/renderers/actions/secrets.go @@ -63,7 +63,7 @@ func fetchSecrets(secrets []*secrets.Secret) Step { return Step{ Name: "Vault secrets", ID: "secrets", - Uses: "hashicorp/vault-action@v2.7.3", + Uses: "hashicorp/vault-action@v2.7.4", With: With{ "url": "https://vault.halfpipe.io", "method": "approle",