diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ae8ae38d..dfb69e25 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -16,7 +16,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: diff --git a/.github/workflows/for-dependabot-to-check.yml b/.github/workflows/for-dependabot-to-check.yml index 4079417b..cb6e17f1 100644 --- a/.github/workflows/for-dependabot-to-check.yml +++ b/.github/workflows/for-dependabot-to-check.yml @@ -47,7 +47,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -81,6 +81,20 @@ jobs: username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: . + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: cf-space + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -149,22 +163,24 @@ jobs: ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }} submodules: recursive - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: . file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/blah:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy diff --git a/.gitignore b/.gitignore index d9c9f157..2f682bc2 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ vendor pipelineActual.yml workflowActual.yml e2e/concourse/init/.halfpipe.io +e2e/shell/all/*_actual.txt diff --git a/README.md b/README.md index 8a986acc..fd7490aa 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,7 @@ We also use [GitHub Actions](https://github.com/springernature/halfpipe/actions) dependabot will raise PRs. Alternatively, to manually update all deps: ```bash -go get -t -u ./... -go mod tidy +go get -t -u ./... && go mod tidy ``` ### GitHub actions diff --git a/build.sh b/build.sh index 237af414..5bf851a2 100755 --- a/build.sh +++ b/build.sh @@ -13,6 +13,10 @@ if [ "${1-}" = "ci" ]; then go_opts="-mod=readonly" fi + +# https://github.com/golang/go/wiki/LoopvarExperiment +export GOEXPERIMENT=loopvar + echo [1/6] fmt go fmt ./... diff --git a/cmd/cmds/actions-migration-help.go b/cmd/cmds/actions-migration-help.go index ad7c43cc..6353b4f6 100644 --- a/cmd/cmds/actions-migration-help.go +++ b/cmd/cmds/actions-migration-help.go @@ -14,8 +14,8 @@ var actionsMigrationHelp = &cobra.Command{ Use: "actions-migration-help", Short: "Prints out the steps needed to migrate from Concourse to Actions", Run: func(cmd *cobra.Command, args []string) { - man, controller := getManifestAndController(formatInput(Input)) - response := controller.Process(man) + man, controller := getManifestAndController(formatInput(Input), nil) + response, _ := controller.Process(man) tpl, _ := template.New("").Parse(` To migrate a Concourse pipeline to Actions you must do the following steps diff --git a/cmd/cmds/exec.go b/cmd/cmds/exec.go new file mode 100644 index 00000000..82828fa2 --- /dev/null +++ b/cmd/cmds/exec.go @@ -0,0 +1,32 @@ +package cmds + +import ( + "fmt" + "github.com/spf13/cobra" + "github.com/springernature/halfpipe/renderers/shell" + "os" +) + +func init() { + rootCmd.AddCommand(execCmd) +} + +var execCmd = &cobra.Command{ + Use: "exec ", + Short: "Execute a task locally", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + taskName := args[0] + + shellRenderer := shell.New(taskName) + man, controller := getManifestAndController(formatInput(Input), shellRenderer) + + response, err := controller.Process(man) + if err != nil { + printErr(err) + os.Exit(1) + } + outputLintResults(response.LintResults) + fmt.Println(response) + }, +} diff --git a/cmd/cmds/helpers.go b/cmd/cmds/helpers.go index 9e85244a..6eca8294 100644 --- a/cmd/cmds/helpers.go +++ b/cmd/cmds/helpers.go @@ -132,7 +132,7 @@ func createController(projectData project.Data, fs afero.Afero, currentDir strin } -func getManifestAndController(halfpipeFilenameOptions []string) (manifest.Manifest, halfpipe.Controller) { +func getManifestAndController(halfpipeFilenameOptions []string, renderer halfpipe.Renderer) (manifest.Manifest, halfpipe.Controller) { if err := checkVersion(); err != nil { printErr(err) os.Exit(1) @@ -157,11 +157,12 @@ func getManifestAndController(halfpipeFilenameOptions []string) (manifest.Manife outputLintResults(linters.LintResults{linters.NewLintResult("Halfpipe Manifest", "https://ee.public.springernature.app/rel-eng/halfpipe/manifest/", manErrors)}) } - var renderer halfpipe.Renderer - if man.Platform.IsActions() { - renderer = actions.NewActions(projectData.GitURI, projectData.HalfpipeFilePath) - } else { - renderer = concourse.NewPipeline(projectData.HalfpipeFilePath) + if renderer == nil { + if man.Platform.IsActions() { + renderer = actions.NewActions(projectData.GitURI, projectData.HalfpipeFilePath) + } else { + renderer = concourse.NewPipeline(projectData.HalfpipeFilePath) + } } controller := createController(projectData, fs, currentDir, renderer) diff --git a/cmd/cmds/internal-representation.go b/cmd/cmds/internal-representation.go index 0b42d497..90b60908 100644 --- a/cmd/cmds/internal-representation.go +++ b/cmd/cmds/internal-representation.go @@ -16,7 +16,7 @@ var internalRepresentation = &cobra.Command{ Short: `Prints the internal representation of the manifest`, Run: func(cmd *cobra.Command, args []string) { - man, controller := getManifestAndController(formatInput(Input)) + man, controller := getManifestAndController(formatInput(Input), nil) defaultedAndMappedManifest, _ := controller.DefaultAndMap(man) diff --git a/cmd/cmds/pipeline_name.go b/cmd/cmds/pipeline_name.go index e091ef92..88c5f277 100644 --- a/cmd/cmds/pipeline_name.go +++ b/cmd/cmds/pipeline_name.go @@ -14,7 +14,7 @@ var pipelineNameCmd = &cobra.Command{ Use: "pipeline-name", Short: "Prints the name of the pipeline", Run: func(cmd *cobra.Command, args []string) { - man, _ := getManifestAndController(formatInput(Input)) + man, _ := getManifestAndController(formatInput(Input), nil) if man.PipelineName() == "" { os.Exit(1) } diff --git a/cmd/cmds/root.go b/cmd/cmds/root.go index 8f97379a..22316058 100644 --- a/cmd/cmds/root.go +++ b/cmd/cmds/root.go @@ -11,8 +11,12 @@ var rootCmd = &cobra.Command{ Short: `halfpipe is a tool to lint and render pipelines Invoke without any arguments to lint your .halfpipe.io file and render a pipeline`, Run: func(cmd *cobra.Command, args []string) { - man, controller := getManifestAndController(formatInput(Input)) - response := controller.Process(man) + man, controller := getManifestAndController(formatInput(Input), nil) + response, err := controller.Process(man) + if err != nil { + printErr(err) + os.Exit(1) + } if man.Platform.IsActions() && output == "" { output = path.Join(response.Project.GitRootPath, ".github/workflows/", man.PipelineName()+".yml") diff --git a/config/config.go b/config/config.go index 2a7edf76..f2d3114a 100644 --- a/config/config.go +++ b/config/config.go @@ -25,6 +25,8 @@ var ( ConcourseURL = "https://concourse." + Domain + ActionsRunnerName = getEnv("HALFPIPE_ACTIONS_RUNNER", "ee-runner") + CacheDirs = []string{ "../../../var/halfpipe/cache", "../../../halfpipe-cache", // deprecated and should be removed after a while diff --git a/controller.go b/controller.go index 4d6f23a5..0a7b7fdc 100644 --- a/controller.go +++ b/controller.go @@ -9,7 +9,7 @@ import ( ) type Controller interface { - Process(man manifest.Manifest) Response + Process(man manifest.Manifest) (Response, error) DefaultAndMap(man manifest.Manifest) (updated manifest.Manifest, err error) } @@ -20,6 +20,10 @@ type Response struct { Platform manifest.Platform } +func (r Response) String() string { + return r.ConfigYaml +} + type Renderer interface { Render(manifest manifest.Manifest) (string, error) } @@ -40,7 +44,7 @@ func NewController(defaulter defaults.Defaults, mapper mapper.Mapper, linters [] } } -func (c controller) Process(man manifest.Manifest) (response Response) { +func (c controller) Process(man manifest.Manifest) (response Response, err error) { defaultedManifest := c.defaulter.Apply(man) for _, linter := range c.linters { @@ -57,11 +61,11 @@ func (c controller) Process(man manifest.Manifest) (response Response) { return } - config, _ := c.renderer.Render(mappedManifest) + config, err := c.renderer.Render(mappedManifest) response.ConfigYaml = config response.Project = c.defaulter.Project response.Platform = man.Platform - return response + return } func (c controller) DefaultAndMap(man manifest.Manifest) (updated manifest.Manifest, err error) { diff --git a/controller_test.go b/controller_test.go index 7a02d695..a4e15991 100644 --- a/controller_test.go +++ b/controller_test.go @@ -41,14 +41,14 @@ func testController() controller { func TestWorksForHalfpipeFileWithYMLExtension(t *testing.T) { c := testController() - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Len(t, response.LintResults.Error(), 0) } func TestWorksForHalfpipeFile(t *testing.T) { c := testController() - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Len(t, response.LintResults.Error(), 0) } @@ -68,7 +68,7 @@ func TestAppliesAllLinters(t *testing.T) { linter2 := fakeLinter{errors.New("error from linter2")} c.linters = []linters.Linter{linter1, linter2} - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Empty(t, response.ConfigYaml) assert.Len(t, response.LintResults, 2) @@ -79,7 +79,7 @@ func TestAppliesAllLinters(t *testing.T) { func TestGivesBackConfigWhenLinterPasses(t *testing.T) { c := testController() - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Len(t, response.LintResults, 0) assert.Equal(t, "fake output", response.ConfigYaml) } @@ -95,7 +95,7 @@ func (f FakeMapper) Apply(original manifest.Manifest) (updated manifest.Manifest func TestGivesBackABadTestResultWhenAMapperFails(t *testing.T) { c := testController() c.mapper = FakeMapper{err: errors.New("blurgh")} - response := c.Process(validHalfpipeManifest) + response, _ := c.Process(validHalfpipeManifest) assert.Len(t, response.LintResults, 1) assert.True(t, response.LintResults.HasErrors()) diff --git a/defaults/task_docker_push.go b/defaults/task_docker_push.go index 3b44bd7b..30ccb50a 100644 --- a/defaults/task_docker_push.go +++ b/defaults/task_docker_push.go @@ -27,5 +27,20 @@ func dockerPushDefaulter(original manifest.DockerPush, man manifest.Manifest, de updated.Platforms = []string{"linux/amd64"} } + if updated.Secrets == nil { + updated.Secrets = make(manifest.Vars) + } + + if man.Platform.IsConcourse() { + updated.Secrets["ARTIFACTORY_URL"] = defaults.Artifactory.URL + updated.Secrets["ARTIFACTORY_USERNAME"] = defaults.Artifactory.Username + updated.Secrets["ARTIFACTORY_PASSWORD"] = defaults.Artifactory.Password + } + if man.Platform.IsActions() { + updated.Secrets["ARTIFACTORY_URL"] = "${{ secrets.EE_ARTIFACTORY_URL }}" + updated.Secrets["ARTIFACTORY_USERNAME"] = "${{ secrets.EE_ARTIFACTORY_USERNAME }}" + updated.Secrets["ARTIFACTORY_PASSWORD"] = "${{ secrets.EE_ARTIFACTORY_PASSWORD }}" + } + return updated } diff --git a/defaults/task_docker_push_test.go b/defaults/task_docker_push_test.go index 0d621548..d71dbf26 100644 --- a/defaults/task_docker_push_test.go +++ b/defaults/task_docker_push_test.go @@ -8,27 +8,16 @@ import ( "testing" ) -func TestWhenPublicImage(t *testing.T) { +func TestWhenPublicImageDontSetUsernameAndPassword(t *testing.T) { task := manifest.DockerPush{Image: "asdf", DockerfilePath: "something", ScanTimeout: 15} - - defaultedTask := manifest.DockerPush{Image: "asdf", DockerfilePath: "something", ScanTimeout: 15, Platforms: []string{"linux/amd64"}} - - assert.Equal(t, defaultedTask, dockerPushDefaulter(task, manifest.Manifest{}, Concourse)) + assert.Empty(t, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Username) + assert.Empty(t, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Password) } -func TestPrivateImage(t *testing.T) { +func TestPrivateImageSetsUsernameAndPassword(t *testing.T) { task := manifest.DockerPush{Image: path.Join(config.DockerRegistry, "push-me"), DockerfilePath: "something"} - - expected := manifest.DockerPush{ - Image: path.Join(config.DockerRegistry, "push-me"), - DockerfilePath: "something", - Username: Concourse.Docker.Username, - Password: Concourse.Docker.Password, - ScanTimeout: 15, - Platforms: []string{"linux/amd64"}, - } - - assert.Equal(t, expected, dockerPushDefaulter(task, manifest.Manifest{}, Concourse)) + assert.Equal(t, Concourse.Docker.Username, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Username) + assert.Equal(t, Concourse.Docker.Password, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Password) } func TestSetsTheDockerFilePath(t *testing.T) { diff --git a/e2e/actions/consumer-integration-test/workflowExpected.yml b/e2e/actions/consumer-integration-test/workflowExpected.yml index 8f98bbdc..45c3421e 100644 --- a/e2e/actions/consumer-integration-test/workflowExpected.yml +++ b/e2e/actions/consumer-integration-test/workflowExpected.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -103,7 +103,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/deploy-cf/workflowExpected.yml b/e2e/actions/deploy-cf/workflowExpected.yml index eb854307..3dfa1bd3 100644 --- a/e2e/actions/deploy-cf/workflowExpected.yml +++ b/e2e/actions/deploy-cf/workflowExpected.yml @@ -57,7 +57,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -104,6 +104,20 @@ jobs: CF_ENV_VAR_ENV2: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_something }} CF_ENV_VAR_ENV3: '{"a": "b", "c": "d"}' CF_ENV_VAR_ENV4: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_another_secret }} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf/foo.html + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -158,7 +172,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -205,6 +219,20 @@ jobs: CF_ENV_VAR_ENV2: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_something }} CF_ENV_VAR_ENV3: '{"a": "b", "c": "d"}' CF_ENV_VAR_ENV4: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_another_secret }} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf/foo.html + cli_version: cf8 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -259,7 +287,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -305,6 +333,20 @@ jobs: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} CF_ENV_VAR_ENV2: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_something }} CF_ENV_VAR_ENV4: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_another_secret }} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf/foo.html + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -443,7 +485,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -481,6 +523,20 @@ jobs: username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest-docker.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -535,7 +591,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -594,6 +650,20 @@ jobs: username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf/foo.html + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: @@ -648,7 +718,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -700,6 +770,20 @@ jobs: username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} env: CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: cf logs --recent + if: failure() + uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable + with: + api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }} + appPath: e2e/actions/deploy-cf + cli_version: cf7 + command: halfpipe-logs + manifestPath: e2e/actions/deploy-cf/manifest.yml + org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }} + password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }} + space: dev + testDomain: springernature.app + username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }} - name: Check uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable with: diff --git a/e2e/actions/deploy-katee/workflowExpected.yml b/e2e/actions/deploy-katee/workflowExpected.yml index 74a43023..982decd7 100644 --- a/e2e/actions/deploy-katee/workflowExpected.yml +++ b/e2e/actions/deploy-katee/workflowExpected.yml @@ -35,22 +35,24 @@ jobs: ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }} submodules: recursive - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/deploy-katee file: e2e/actions/deploy-katee/Dockerfile platforms: linux/amd64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -86,7 +88,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -139,7 +141,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/deploy-ml/workflowExpected.yml b/e2e/actions/deploy-ml/workflowExpected.yml index c4f03fa8..baec31cd 100644 --- a/e2e/actions/deploy-ml/workflowExpected.yml +++ b/e2e/actions/deploy-ml/workflowExpected.yml @@ -87,7 +87,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/docker-compose/workflowExpected.yml b/e2e/actions/docker-compose/workflowExpected.yml index c6a39224..63ac4c34 100644 --- a/e2e/actions/docker-compose/workflowExpected.yml +++ b/e2e/actions/docker-compose/workflowExpected.yml @@ -83,7 +83,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/docker-push/.halfpipe.io b/e2e/actions/docker-push/.halfpipe.io index 844dfc3d..730fce8f 100644 --- a/e2e/actions/docker-push/.halfpipe.io +++ b/e2e/actions/docker-push/.halfpipe.io @@ -19,7 +19,6 @@ tasks: name: Push default image: eu.gcr.io/halfpipe-io/someImage restore_artifacts: true - tag: version - type: docker-push name: Push custom @@ -39,7 +38,6 @@ tasks: name: Push multiple platforms image: eu.gcr.io/halfpipe-io/someImage restore_artifacts: true - tag: version platforms: - "linux/amd64" - "linux/arm64" @@ -48,8 +46,17 @@ tasks: name: Push multiple platforms and use cache image: eu.gcr.io/halfpipe-io/someImage restore_artifacts: true - tag: version use_cache: true platforms: - "linux/amd64" - "linux/arm64" + +- type: docker-push + name: Push with secrets + image: eu.gcr.io/halfpipe-io/someImage + vars: + A: a + B: b + secrets: + C: ((secret.c)) + D: d diff --git a/e2e/actions/docker-push/workflowExpected.yml b/e2e/actions/docker-push/workflowExpected.yml index cea79dbb..8ac5485b 100644 --- a/e2e/actions/docker-push/workflowExpected.yml +++ b/e2e/actions/docker-push/workflowExpected.yml @@ -70,22 +70,24 @@ jobs: run: tar -xvf halfpipe-artifacts.tar; rm halfpipe-artifacts.tar working-directory: ${{ github.workspace }} - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -121,7 +123,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -151,25 +153,27 @@ jobs: password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_foo_bar }} username: user - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME BAR=bar BLAH=${{ steps.secrets.outputs.springernature_data_halfpipe-team_very_secret }} - BUILD_VERSION=2.${{ github.run_number }}.0 + BUILD_VERSION FOO=foo - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile2 platforms: linux/amd64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/dockerhubusername/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -218,22 +222,24 @@ jobs: run: tar -xvf halfpipe-artifacts.tar; rm halfpipe-artifacts.tar working-directory: ${{ github.workspace }} - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | - ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} - ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} - ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile platforms: linux/amd64,linux/arm64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy @@ -282,24 +288,102 @@ jobs: run: tar -xvf halfpipe-artifacts.tar; rm halfpipe-artifacts.tar working-directory: ${{ github.workspace }} - name: Build Image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI + cache-from: type=registry,ref=eu.gcr.io/halfpipe-io/cache/someImage:buildcache + cache-to: type=inline + context: e2e/actions/docker-push + file: e2e/actions/docker-push/Dockerfile + platforms: linux/amd64,linux/arm64 + provenance: false + push: true + secrets: | ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} - BUILD_VERSION=2.${{ github.run_number }}.0 - GIT_REVISION=${{ github.sha }} - RUNNING_IN_CI=true - VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }} - VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }} - cache-from: type=registry,ref=eu.gcr.io/halfpipe-io/cache/someImage - cache-to: type=inline + tags: |- + eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} + eu.gcr.io/halfpipe-io/cache/someImage:buildcache + - name: Run Trivy vulnerability scanner + uses: docker://aquasec/trivy + with: + args: -c "cd e2e/actions/docker-push; [ -f .trivyignore ] && echo \"Ignoring the following CVE's due to .trivyignore\" || true; [ -f .trivyignore ] && cat .trivyignore; echo || true; trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 1 eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }}" + entrypoint: /bin/sh + - name: Push Image + run: |- + docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} --tag eu.gcr.io/halfpipe-io/someImage:latest + docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} --tag eu.gcr.io/halfpipe-io/someImage:${{ env.BUILD_VERSION }} + docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} --tag eu.gcr.io/halfpipe-io/someImage:${{ env.GIT_REVISION }} + - name: Repository dispatch + uses: peter-evans/repository-dispatch@v2 + with: + event-type: docker-push:eu.gcr.io/halfpipe-io/someImage + token: ${{ secrets.EE_REPOSITORY_DISPATCH_TOKEN }} + - name: Summary + run: |- + echo ":ship: **Image Pushed Successfully**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "[eu.gcr.io/halfpipe-io/someImage](https://eu.gcr.io/halfpipe-io/someImage)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Tags:" >> $GITHUB_STEP_SUMMARY + echo "- eu.gcr.io/halfpipe-io/someImage:latest" >> $GITHUB_STEP_SUMMARY + echo "- eu.gcr.io/halfpipe-io/someImage:${{ env.BUILD_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- eu.gcr.io/halfpipe-io/someImage:${{ env.GIT_REVISION }}" >> $GITHUB_STEP_SUMMARY + push_with_secrets: + name: Push with secrets + needs: + - push_multiple_platforms_and_use_cache + runs-on: ee-runner + timeout-minutes: 60 + steps: + - name: Vault secrets + id: secrets + uses: hashicorp/vault-action@v2.7.4 + with: + exportEnv: false + method: approle + roleId: ${{ env.VAULT_ROLE_ID }} + secretId: ${{ env.VAULT_SECRET_ID }} + secrets: | + /springernature/data/halfpipe-team/secret c | springernature_data_halfpipe-team_secret_c ; + url: https://vault.halfpipe.io + - name: Checkout code + uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }} + submodules: recursive + - name: Build Image + uses: docker/build-push-action@v5 + with: + build-args: | + A=a + ARTIFACTORY_PASSWORD + ARTIFACTORY_URL + ARTIFACTORY_USERNAME + B=b + BUILD_VERSION + GIT_REVISION + RUNNING_IN_CI context: e2e/actions/docker-push file: e2e/actions/docker-push/Dockerfile - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 provenance: false push: true + secrets: | + ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }} + ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }} + ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }} + C=${{ steps.secrets.outputs.springernature_data_halfpipe-team_secret_c }} + D=d tags: eu.gcr.io/halfpipe-io/cache/someImage:${{ env.GIT_REVISION }} - name: Run Trivy vulnerability scanner uses: docker://aquasec/trivy diff --git a/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml b/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml index 8405de0c..32f10560 100644 --- a/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml +++ b/e2e/actions/feature-update-pipeline-and-tag/workflowExpected.yml @@ -35,10 +35,7 @@ jobs: submodules: recursive - name: Sync workflow with halfpipe manifest id: sync - uses: docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update - with: - args: -c "cd e2e/actions/feature-update-pipeline-and-tag; update-actions-workflow" - entrypoint: /bin/bash + run: halfpipe-update-workflow env: HALFPIPE_FILE_PATH: .halfpipe.io.yml - name: Commit and push changes to workflow diff --git a/e2e/actions/feature-update-pipeline/workflowExpected.yml b/e2e/actions/feature-update-pipeline/workflowExpected.yml index f26f9131..060404c9 100644 --- a/e2e/actions/feature-update-pipeline/workflowExpected.yml +++ b/e2e/actions/feature-update-pipeline/workflowExpected.yml @@ -35,10 +35,7 @@ jobs: submodules: recursive - name: Sync workflow with halfpipe manifest id: sync - uses: docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update - with: - args: -c "cd e2e/actions/feature-update-pipeline; update-actions-workflow" - entrypoint: /bin/bash + run: halfpipe-update-workflow env: HALFPIPE_FILE_PATH: .halfpipe.io - name: Commit and push changes to workflow diff --git a/e2e/actions/run/workflowExpected.yml b/e2e/actions/run/workflowExpected.yml index 6d6046b6..0673712b 100644 --- a/e2e/actions/run/workflowExpected.yml +++ b/e2e/actions/run/workflowExpected.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/actions/trigger-git-options/workflowExpected.yml b/e2e/actions/trigger-git-options/workflowExpected.yml index a9f39c52..63de160c 100644 --- a/e2e/actions/trigger-git-options/workflowExpected.yml +++ b/e2e/actions/trigger-git-options/workflowExpected.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle @@ -63,7 +63,7 @@ jobs: steps: - name: Vault secrets id: secrets - uses: hashicorp/vault-action@v2.7.3 + uses: hashicorp/vault-action@v2.7.4 with: exportEnv: false method: approle diff --git a/e2e/concourse/artifacts/pipelineExpected.yml b/e2e/concourse/artifacts/pipelineExpected.yml index dea44ff7..51e0393e 100644 --- a/e2e/concourse/artifacts/pipelineExpected.yml +++ b/e2e/concourse/artifacts/pipelineExpected.yml @@ -252,6 +252,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/artifacts/manifest.yml + put: cf logs --recent + resource: cf-snpaas-pe-staging params: appPath: git/e2e/concourse/artifacts cliVersion: cf7 @@ -436,31 +444,61 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git + - name: tagList - name: docker_build - outputs: - - name: image params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/artifacts + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/artifacts/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f docker_build/e2e/concourse/artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + docker_build/e2e/concourse/artifacts + docker buildx build \ + -f docker_build/e2e/concourse/artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + docker_build/e2e/concourse/artifacts path: /bin/sh privileged: true task: build @@ -473,8 +511,9 @@ jobs: type: docker-image inputs: - name: git - - name: image - name: docker_build + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -482,16 +521,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: docker_build/e2e/concourse/artifacts path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -747,13 +805,6 @@ resources: folder: halfpipe-team/halfpipe-e2e-artifacts json_key: ((halfpipe-artifacts.private_key)) type: gcp-resource -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image - check_every: 24h0m0s name: cf-snpaas-pe-staging source: diff --git a/e2e/concourse/deploy-cf-docker-image/pipelineExpected.yml b/e2e/concourse/deploy-cf-docker-image/pipelineExpected.yml index 29b5a889..2459ac16 100644 --- a/e2e/concourse/deploy-cf-docker-image/pipelineExpected.yml +++ b/e2e/concourse/deploy-cf-docker-image/pipelineExpected.yml @@ -62,6 +62,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 @@ -105,6 +113,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 @@ -221,6 +237,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: rolling-cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 @@ -254,6 +278,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: rolling-cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 @@ -333,6 +365,14 @@ jobs: timeout: 1h - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-docker-image/manifest.yml + put: cf logs --recent + resource: rolling-cf-snpaas-dev params: buildVersionPath: version/version cliVersion: cf7 diff --git a/e2e/concourse/deploy-cf-rolling/pipelineExpected.yml b/e2e/concourse/deploy-cf-rolling/pipelineExpected.yml index c37216a4..8e0a35c0 100644 --- a/e2e/concourse/deploy-cf-rolling/pipelineExpected.yml +++ b/e2e/concourse/deploy-cf-rolling/pipelineExpected.yml @@ -10,6 +10,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-rolling/manifest.yml + put: cf logs --recent + resource: rolling-cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf-rolling cliVersion: cf7 @@ -34,6 +42,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-rolling/manifest.yml + put: cf logs --recent + resource: rolling-cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf-rolling cliVersion: cf7 @@ -108,6 +124,14 @@ jobs: timeout: 1h - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-rolling/manifest.yml + put: cf logs --recent + resource: rolling-cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf-rolling cliVersion: cf7 diff --git a/e2e/concourse/deploy-cf-with-artefact/pipelineExpected.yml b/e2e/concourse/deploy-cf-with-artefact/pipelineExpected.yml index f2520428..929d8d66 100644 --- a/e2e/concourse/deploy-cf-with-artefact/pipelineExpected.yml +++ b/e2e/concourse/deploy-cf-with-artefact/pipelineExpected.yml @@ -129,6 +129,14 @@ jobs: timeout: 15m - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf-with-artefact/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: artifacts/e2e/concourse/deploy-cf-with-artefact/build/linux/binary cliVersion: cf7 diff --git a/e2e/concourse/deploy-cf/pipelineExpected.yml b/e2e/concourse/deploy-cf/pipelineExpected.yml index c6bbb7f2..5be34d4c 100644 --- a/e2e/concourse/deploy-cf/pipelineExpected.yml +++ b/e2e/concourse/deploy-cf/pipelineExpected.yml @@ -10,6 +10,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf cliVersion: cf7 @@ -34,6 +42,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf8 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf cliVersion: cf8 @@ -69,6 +85,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf cliVersion: cf7 @@ -502,6 +526,14 @@ jobs: timeout: 1h - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/deploy-cf/manifest.yml + put: cf logs --recent + resource: cf-dev-api-halfpipe-team-dev params: appPath: git/e2e/concourse/deploy-cf cliVersion: cf7 diff --git a/e2e/concourse/deploy-katee/pipelineExpected.yml b/e2e/concourse/deploy-katee/pipelineExpected.yml index b4d6bb32..45d59b8a 100644 --- a/e2e/concourse/deploy-katee/pipelineExpected.yml +++ b/e2e/concourse/deploy-katee/pipelineExpected.yml @@ -87,28 +87,54 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/deploy-katee + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/deploy-katee/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/deploy-katee/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/deploy-katee + docker buildx build \ + -f git/e2e/concourse/deploy-katee/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/deploy-katee path: /bin/sh privileged: true task: build @@ -121,7 +147,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -129,16 +156,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/deploy-katee path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: someimage + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/halfpipe-team/someImage:$(cat git/.git/ref) --tag eu.gcr.io/halfpipe-io/halfpipe-team/someImage:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -379,10 +425,3 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-pipeline-name type: semver -- check_every: 24h0m0s - name: someimage - source: - password: ((halfpipe-gcr.private_key)) - repository: eu.gcr.io/halfpipe-io/halfpipe-team/someImage - username: _json_key - type: registry-image diff --git a/e2e/concourse/docker-push-paths/pipelineExpected.yml b/e2e/concourse/docker-push-paths/pipelineExpected.yml index bcd4794f..59a77f02 100644 --- a/e2e/concourse/docker-push-paths/pipelineExpected.yml +++ b/e2e/concourse/docker-push-paths/pipelineExpected.yml @@ -34,28 +34,54 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-paths/some/build/dir + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-paths/some/build/dir + docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-paths/some/build/dir path: /bin/sh privileged: true task: build @@ -68,7 +94,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -76,16 +103,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-paths path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -124,28 +170,54 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-paths + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-paths + docker buildx build \ + -f git/e2e/concourse/docker-push-paths/dockerfiles/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-paths path: /bin/sh privileged: true task: build @@ -158,7 +230,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -166,16 +239,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-paths path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resources: @@ -188,17 +280,3 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image -- check_every: 24h0m0s - name: halfpipe - source: - password: verysecret - repository: springerplatformengineering/halfpipe - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml index 202fc7d0..4b3f62c4 100644 --- a/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-docker-trigger/pipelineExpected.yml @@ -41,30 +41,60 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-docker-trigger + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-with-docker-trigger + docker buildx build \ + -f git/e2e/concourse/docker-push-with-docker-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh privileged: true task: build @@ -77,7 +107,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -85,16 +116,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-docker-trigger path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resources: @@ -111,10 +161,3 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml index 281e223a..22278140 100644 --- a/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-pipeline-trigger/pipelineExpected.yml @@ -39,30 +39,60 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-pipeline-trigger + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-with-pipeline-trigger + docker buildx build \ + -f git/e2e/concourse/docker-push-with-pipeline-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh privileged: true task: build @@ -75,7 +105,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -83,16 +114,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-pipeline-trigger path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resource_types: @@ -113,13 +163,6 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image - name: halfpipe-example-docker-push.docker-push source: concourse_url: ((concourse.url)) diff --git a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml index c9d96dbc..b0d47756 100644 --- a/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-restore-artifacts/pipelineExpected.yml @@ -235,31 +235,61 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git + - name: tagList - name: docker_build - outputs: - - name: image params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts + docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true task: build @@ -272,8 +302,9 @@ jobs: type: docker-image inputs: - name: git - - name: image - name: docker_build + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -281,16 +312,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat ../../../.git/ref) || true dir: docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image1 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --tag springerplatformengineering/image1:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -384,31 +434,61 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git + - name: tagList - name: docker_build - outputs: - - name: image params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: docker_build/e2e/concourse/docker-push-with-restore-artifacts + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts + docker buildx build \ + -f docker_build/e2e/concourse/docker-push-with-restore-artifacts/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh privileged: true task: build @@ -421,8 +501,9 @@ jobs: type: docker-image inputs: - name: git - - name: image - name: docker_build + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -430,16 +511,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat ../../../.git/ref) || true dir: docker_build/e2e/concourse/docker-push-with-restore-artifacts path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image2 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --tag springerplatformengineering/image2:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resource_types: @@ -476,17 +576,3 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-docker-push-with-update-pipeline type: semver -- check_every: 24h0m0s - name: image1 - source: - password: verysecret - repository: springerplatformengineering/image1 - username: rob - type: registry-image -- check_every: 24h0m0s - name: image2 - source: - password: verysecret - repository: springerplatformengineering/image2 - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml index b0f327bd..d7bb9783 100644 --- a/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml +++ b/e2e/concourse/docker-push-with-update-pipeline/pipelineExpected.yml @@ -87,30 +87,60 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-with-update-pipeline + docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true task: build @@ -123,7 +153,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -131,16 +162,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image1 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image1:$(cat git/.git/ref) --tag springerplatformengineering/image1:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -186,30 +236,60 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push-with-update-pipeline + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-with-update-pipeline + docker buildx build \ + -f git/e2e/concourse/docker-push-with-update-pipeline/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh privileged: true task: build @@ -222,7 +302,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -230,16 +311,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push-with-update-pipeline path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: image2 + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/image2:$(cat git/.git/ref) --tag springerplatformengineering/image2:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resources: @@ -260,17 +360,3 @@ resources: json_key: ((halfpipe-semver.private_key)) key: halfpipe-team-docker-push-with-update-pipeline type: semver -- check_every: 24h0m0s - name: image1 - source: - password: verysecret - repository: springerplatformengineering/image1 - username: rob - type: registry-image -- check_every: 24h0m0s - name: image2 - source: - password: verysecret - repository: springerplatformengineering/image2 - username: rob - type: registry-image diff --git a/e2e/concourse/docker-push/.halfpipe.io b/e2e/concourse/docker-push/.halfpipe.io index 03657f2f..9782a497 100644 --- a/e2e/concourse/docker-push/.halfpipe.io +++ b/e2e/concourse/docker-push/.halfpipe.io @@ -15,6 +15,10 @@ tasks: vars: A: a B: b + secrets: + C: c + D: d + - type: docker-push username: rob password: verysecret @@ -28,7 +32,7 @@ tasks: retries: 1 ignore_vulnerabilities: true scan_timeout: 30 + use_cache: true platforms: - "linux/amd64" - "linux/arm64" - diff --git a/e2e/concourse/docker-push/pipelineExpected.yml b/e2e/concourse/docker-push/pipelineExpected.yml index c6c2f4ef..505aa798 100644 --- a/e2e/concourse/docker-push/pipelineExpected.yml +++ b/e2e/concourse/docker-push/pipelineExpected.yml @@ -34,30 +34,66 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b + C: c + D: d DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + --secret id=C \ + --secret id=D \ + git/e2e/concourse/docker-push + docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + --secret id=C \ + --secret id=D \ + git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -70,7 +106,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -78,16 +115,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly.thisismy-tag + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) thisIsMy_Tag; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe_fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -128,28 +184,54 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push + docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -163,7 +245,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -171,17 +254,36 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h - attempts: 2 - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly.thisismy-tag2 + config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) thisIsMy_Tag2; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe_fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true - build_log_retention: @@ -230,23 +332,52 @@ jobs: inputs: - name: git - name: tagList - outputs: - - name: image params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) --push --provenance=false $CONTEXT + echo $ docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64,linux/arm64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-to type=inline \ + git/e2e/concourse/docker-push + docker buildx build \ + -f git/e2e/concourse/docker-push/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64,linux/arm64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-from type=registry,ref=eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:buildcache \ + --cache-to type=inline \ + git/e2e/concourse/docker-push path: /bin/sh privileged: true task: build @@ -260,7 +391,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -268,7 +400,8 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat git/.git/ref) || true + trivy image --timeout 30m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe_fly:$(cat ../../../.git/ref) || true + dir: git/e2e/concourse/docker-push path: /bin/sh task: trivy timeout: 1h @@ -285,16 +418,8 @@ jobs: inputs: - name: git - name: tagList - outputs: - - name: image params: - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/docker-push DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/docker-push/Dockerfile platform: linux run: args: @@ -317,17 +442,3 @@ resources: private_key: ((halfpipe-github.private_key)) uri: git@github.com:springernature/halfpipe.git type: git -- check_every: 24h0m0s - name: halfpipe-fly.thisismy-tag - source: - password: verysecret - repository: springerplatformengineering/halfpipe_fly:thisIsMy_Tag - username: rob - type: registry-image -- check_every: 24h0m0s - name: halfpipe-fly.thisismy-tag2 - source: - password: verysecret - repository: springerplatformengineering/halfpipe_fly:thisIsMy_Tag2 - username: rob - type: registry-image diff --git a/e2e/concourse/notifications/pipelineExpected.yml b/e2e/concourse/notifications/pipelineExpected.yml index db8515c6..9318f7e6 100644 --- a/e2e/concourse/notifications/pipelineExpected.yml +++ b/e2e/concourse/notifications/pipelineExpected.yml @@ -239,6 +239,14 @@ jobs: trigger: true - attempts: 2 no_get: true + on_failure: + no_get: true + params: + cliVersion: cf7 + command: halfpipe-logs + manifestPath: git/e2e/concourse/notifications/manifest.yml + put: cf logs --recent + resource: cf-snpaas-pe-staging params: appPath: git/e2e/concourse/notifications cliVersion: cf7 diff --git a/e2e/concourse/timer-trigger/pipelineExpected.yml b/e2e/concourse/timer-trigger/pipelineExpected.yml index 167c3e62..9c5ccd25 100644 --- a/e2e/concourse/timer-trigger/pipelineExpected.yml +++ b/e2e/concourse/timer-trigger/pipelineExpected.yml @@ -39,30 +39,60 @@ jobs: image_resource: name: "" source: - repository: concourse/oci-build-task + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key type: registry-image inputs: - name: git - outputs: - - name: image + - name: tagList params: - BUILD_ARG_A: a - BUILD_ARG_ARTIFACTORY_PASSWORD: ((artifactory.password)) - BUILD_ARG_ARTIFACTORY_URL: ((artifactory.url)) - BUILD_ARG_ARTIFACTORY_USERNAME: ((artifactory.username)) - BUILD_ARG_B: b - BUILD_ARG_RUNNING_IN_CI: "true" - CONTEXT: git/e2e/concourse/timer-trigger + A: a + ARTIFACTORY_PASSWORD: ((artifactory.password)) + ARTIFACTORY_URL: ((artifactory.url)) + ARTIFACTORY_USERNAME: ((artifactory.username)) + B: b DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) - DOCKERFILE: git/e2e/concourse/timer-trigger/Dockerfile + RUNNING_IN_CI: "true" platform: linux run: args: - -c - |- - mkdir ~/.docker echo $DOCKER_CONFIG_JSON > ~/.docker/config.json - build + echo $ docker buildx build \ + -f git/e2e/concourse/timer-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/timer-trigger + docker buildx build \ + -f git/e2e/concourse/timer-trigger/Dockerfile \ + --push \ + --provenance false \ + --platform linux/amd64 \ + --tag eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) \ + --build-arg A \ + --build-arg ARTIFACTORY_PASSWORD \ + --build-arg ARTIFACTORY_URL \ + --build-arg ARTIFACTORY_USERNAME \ + --build-arg B \ + --build-arg RUNNING_IN_CI \ + --secret id=ARTIFACTORY_PASSWORD \ + --secret id=ARTIFACTORY_URL \ + --secret id=ARTIFACTORY_USERNAME \ + git/e2e/concourse/timer-trigger path: /bin/sh privileged: true task: build @@ -75,7 +105,8 @@ jobs: type: docker-image inputs: - name: git - - name: image + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) platform: linux run: args: @@ -83,16 +114,35 @@ jobs: - |- [ -f .trivyignore ] && echo "Ignoring the following CVE's due to .trivyignore" || true [ -f .trivyignore ] && cat .trivyignore; echo || true - trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 --input ../../../../image/image.tar || true + trivy image --timeout 15m --ignore-unfixed --severity CRITICAL --scanners vuln --exit-code 0 eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat ../../../.git/ref) || true dir: git/e2e/concourse/timer-trigger path: /bin/sh task: trivy timeout: 1h - - no_get: true - params: - additional_tags: tagList/tagList - image: image/image.tar - put: halfpipe-fly + - config: + image_resource: + name: "" + source: + password: ((halfpipe-gcr.private_key)) + repository: eu.gcr.io/halfpipe-io/halfpipe-buildx + tag: latest + username: _json_key + type: registry-image + inputs: + - name: git + - name: tagList + params: + DOCKER_CONFIG_JSON: ((halfpipe-gcr.docker_config)) + platform: linux + run: + args: + - -c + - |- + echo $DOCKER_CONFIG_JSON > ~/.docker/config.json + for tag in $(cat tagList/tagList) ; do docker buildx imagetools create eu.gcr.io/halfpipe-io/cache/springerplatformengineering/halfpipe-fly:$(cat git/.git/ref) --tag springerplatformengineering/halfpipe-fly:$tag; done + path: /bin/sh + privileged: true + task: publish-final-image timeout: 1h serial: true resource_types: @@ -119,10 +169,3 @@ resources: fire_immediately: true location: UTC type: halfpipe-cron-resource -- check_every: 24h0m0s - name: halfpipe-fly - source: - password: verysecret - repository: springerplatformengineering/halfpipe-fly - username: rob - type: registry-image diff --git a/e2e/shell/all/.halfpipe.io b/e2e/shell/all/.halfpipe.io new file mode 100644 index 00000000..fd3de502 --- /dev/null +++ b/e2e/shell/all/.halfpipe.io @@ -0,0 +1,33 @@ +team: halfpipe-team +pipeline: pipeline-name + +feature_toggles: +- update-pipeline + +tasks: +- type: run + name: run + script: ./test.sh + docker: + image: alpine:test + vars: + ENV1: 1234 + ENV2: ((secret.something)) + ENV3: '{"a": "b", "c": "d"}' + ENV4: ((another.secret)) + VERY_SECRET: blah + +- type: docker-compose + name: docker-compose-simple + +- type: docker-compose + name: docker-compose-complex + command: \echo hello + compose_file: custom-docker-compose.yml + service: customservice + vars: + ENV1: 1234 + ENV2: ((secret.something)) + ENV3: '{"a": "b", "c": "d"}' + ENV4: ((another.secret)) + VERY_SECRET: blah diff --git a/e2e/shell/all/Dockerfile b/e2e/shell/all/Dockerfile new file mode 100644 index 00000000..76737680 --- /dev/null +++ b/e2e/shell/all/Dockerfile @@ -0,0 +1 @@ +from alpine diff --git a/e2e/shell/all/custom-docker-compose.yml b/e2e/shell/all/custom-docker-compose.yml new file mode 100644 index 00000000..88f55fec --- /dev/null +++ b/e2e/shell/all/custom-docker-compose.yml @@ -0,0 +1,5 @@ +version: '3' + +services: + customservice: + image: appropriate/curl diff --git a/e2e/shell/all/docker-compose-complex_expected.txt b/e2e/shell/all/docker-compose-complex_expected.txt new file mode 100644 index 00000000..f52f444a --- /dev/null +++ b/e2e/shell/all/docker-compose-complex_expected.txt @@ -0,0 +1,17 @@ +docker compose \ + -f custom-docker-compose.yml \ + run \ + -v "$PWD":/app \ + -w /app \ + -e ARTIFACTORY_PASSWORD="$(vault kv get -field=password /springernature/shared/artifactory)" \ + -e ARTIFACTORY_URL="$(vault kv get -field=url /springernature/shared/artifactory)" \ + -e ARTIFACTORY_USERNAME="$(vault kv get -field=username /springernature/shared/artifactory)" \ + -e ENV1="1234" \ + -e ENV2="$(vault kv get -field=something /springernature/halfpipe-team/secret)" \ + -e ENV3="{"a": "b", "c": "d"}" \ + -e ENV4="$(vault kv get -field=secret /springernature/halfpipe-team/another)" \ + -e RUNNING_IN_CI="true" \ + -e VERY_SECRET="blah" \ + --use-aliases \ + customservice \ + \echo hello diff --git a/e2e/shell/all/docker-compose-simple_expected.txt b/e2e/shell/all/docker-compose-simple_expected.txt new file mode 100644 index 00000000..bd5995fc --- /dev/null +++ b/e2e/shell/all/docker-compose-simple_expected.txt @@ -0,0 +1,11 @@ +docker compose \ + -f docker-compose.yml \ + run \ + -v "$PWD":/app \ + -w /app \ + -e ARTIFACTORY_PASSWORD="$(vault kv get -field=password /springernature/shared/artifactory)" \ + -e ARTIFACTORY_URL="$(vault kv get -field=url /springernature/shared/artifactory)" \ + -e ARTIFACTORY_USERNAME="$(vault kv get -field=username /springernature/shared/artifactory)" \ + -e RUNNING_IN_CI="true" \ + --use-aliases \ + app diff --git a/e2e/shell/all/docker-compose.yml b/e2e/shell/all/docker-compose.yml new file mode 100644 index 00000000..e8c0c0f0 --- /dev/null +++ b/e2e/shell/all/docker-compose.yml @@ -0,0 +1,5 @@ +version: '3' + +services: + app: + image: appropriate/curl diff --git a/e2e/shell/all/run_expected.txt b/e2e/shell/all/run_expected.txt new file mode 100644 index 00000000..272878d4 --- /dev/null +++ b/e2e/shell/all/run_expected.txt @@ -0,0 +1,14 @@ +docker run -it \ + -v "$PWD":/app \ + -w /app \ + -e ARTIFACTORY_PASSWORD="$(vault kv get -field=password /springernature/shared/artifactory)" \ + -e ARTIFACTORY_URL="$(vault kv get -field=url /springernature/shared/artifactory)" \ + -e ARTIFACTORY_USERNAME="$(vault kv get -field=username /springernature/shared/artifactory)" \ + -e ENV1="1234" \ + -e ENV2="$(vault kv get -field=something /springernature/halfpipe-team/secret)" \ + -e ENV3="{"a": "b", "c": "d"}" \ + -e ENV4="$(vault kv get -field=secret /springernature/halfpipe-team/another)" \ + -e RUNNING_IN_CI="true" \ + -e VERY_SECRET="blah" \ + alpine:test \ + ./test.sh diff --git a/e2e/shell/all/test.sh b/e2e/shell/all/test.sh new file mode 100755 index 00000000..b9e78644 --- /dev/null +++ b/e2e/shell/all/test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +for f in `find . -name '*_expected.txt'`; do + taskName="${f:2:(-13)}" + echo " task name: $taskName" + ../../../halfpipe -q exec "$taskName" > "${f/expected/actual}" + diff -w "$f" "${f/expected/actual}" +done diff --git a/e2e/test.sh b/e2e/test.sh index ebda5859..724ccd21 100755 --- a/e2e/test.sh +++ b/e2e/test.sh @@ -1,36 +1,36 @@ #!/usr/bin/env bash runTest() { dir=${1%*/} - yml="/tmp/halfpipe-e2e/$dir.yml" - log="/tmp/halfpipe-e2e/$dir.log" - ( - set -e - echo "* Running ${dir}" - cd ${dir} - if [[ -f test.sh ]]; then - ./test.sh - elif [[ -f workflowExpected.yml ]]; then - # actions - ../../../halfpipe -q -o workflowActual.yml - sed '6s/\"\"/main/' workflowActual.yml > $yml - diff --ignore-blank-lines $yml workflowExpected.yml - else - # concourse - ../../../halfpipe -q -o pipelineActual.yml - sed 's/ branch: ""/ branch: main/g' pipelineActual.yml | sed -E 's/(key:.+)\-$/\1/g' > $yml - diff --ignore-blank-lines $yml pipelineExpected.yml - if command -v fly > /dev/null; then - fly validate-pipeline -c pipelineActual.yml &> /dev/null - fi + echo "* Running ${dir}" + mkdir -p /tmp/halfpipe-e2e/$dir + tmpYml="/tmp/halfpipe-e2e/$dir.yml" + + cd ${dir} + if [[ -f test.sh ]]; then + ./test.sh + elif [[ -f workflowExpected.yml ]]; then + # actions + ../../../halfpipe -q -o workflowActual.yml + sed '6s/\"\"/main/' workflowActual.yml > $tmpYml + diff --ignore-blank-lines $tmpYml workflowExpected.yml + elif [[ -f pipelineExpected.yml ]]; then + # concourse + ../../../halfpipe -q -o pipelineActual.yml + sed 's/ branch: ""/ branch: main/g' pipelineActual.yml | sed -E 's/(key:.+)\-$/\1/g' > $tmpYml + diff --ignore-blank-lines $tmpYml pipelineExpected.yml + if command -v fly > /dev/null; then + fly validate-pipeline -c pipelineActual.yml &> /dev/null fi - ) &>> $log + fi } -export -f runTest rm -rf /tmp/halfpipe-e2e -mkdir -p /tmp/halfpipe-e2e/{actions,concourse} -ls -d */*/ | xargs -ID -P16 bash -c "runTest D" -RET_CODE=$? -cat /tmp/halfpipe-e2e/*/*.log -exit $RET_CODE +export -f runTest + +if command -v parallel > /dev/null; then + ls -d */*/ | parallel -j16 runTest +else + # xargs doesn't return exit code reliably with parallel + ls -d */*/ | xargs -ID -P1 bash -c "runTest D" +fi diff --git a/go.mod b/go.mod index b0bbeed5..f6281627 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( code.cloudfoundry.org/cli v7.1.0+incompatible github.com/blang/semver v3.5.1+incompatible github.com/cloudfoundry/bosh-cli v6.4.1+incompatible - github.com/cloudfoundry/bosh-utils v0.0.388 // indirect + github.com/cloudfoundry/bosh-utils v0.0.402 // indirect github.com/concourse/concourse v1.6.1-0.20230204041300-d289c02f878d github.com/cppforlife/go-patch v0.2.0 // indirect github.com/gookit/color v1.5.4 @@ -17,9 +17,9 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mbrevoort/cronexpr v0.0.0-20170805223836-93a834a6c23a github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/onsi/gomega v1.27.10 + github.com/onsi/gomega v1.28.0 github.com/pkg/errors v0.9.1 - github.com/spf13/afero v1.9.5 + github.com/spf13/afero v1.10.0 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 github.com/tcnksm/go-gitconfig v0.1.2 @@ -30,7 +30,7 @@ require ( require ( github.com/sirupsen/logrus v1.9.3 - golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d gopkg.in/yaml.v3 v3.0.1 ) @@ -50,9 +50,9 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/tedsuo/rata v1.0.1-0.20170830210128-07d200713958 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/crypto v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.16.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 56ad325e..e52c46a2 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudfoundry/bosh-cli v6.4.1+incompatible h1:n5/+NIF9QxvGINOrjh6DmO+GTen78MoCj5+LU9L8bR4= github.com/cloudfoundry/bosh-cli v6.4.1+incompatible/go.mod h1:rzIB+e1sn7wQL/TJ54bl/FemPKRhXby5BIMS3tLuWFM= -github.com/cloudfoundry/bosh-utils v0.0.388 h1:5UwnFgDDioddiWzRuQbxeP+ERcJwFQ1FM0x+udN97nM= -github.com/cloudfoundry/bosh-utils v0.0.388/go.mod h1:bKQU0etIkZD0cTeqvv+im2K6yWzQIv0kx3IXnkKAxzU= +github.com/cloudfoundry/bosh-utils v0.0.402 h1:uvVgTcbaF0x04sVG6kyK+OgeVP+PZzv5F/R7dJFKBx8= +github.com/cloudfoundry/bosh-utils v0.0.402/go.mod h1:TiXvpLZjeUXGEdUGhY8Sdnv51h8/g5X0EJ+8HLqApCM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -140,8 +140,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -194,10 +194,10 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= -github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= +github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c= +github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= @@ -213,8 +213,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= -github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -248,8 +248,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -260,8 +260,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -317,8 +317,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -377,11 +377,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -390,8 +390,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -442,8 +442,8 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/linters/docker-push.go b/linters/docker-push.go index 8b188df8..2c3b226c 100644 --- a/linters/docker-push.go +++ b/linters/docker-push.go @@ -2,6 +2,7 @@ package linters import ( "fmt" + "golang.org/x/exp/slices" "os" "regexp" "strings" @@ -10,7 +11,7 @@ import ( "github.com/springernature/halfpipe/manifest" ) -func LintDockerPushTask(docker manifest.DockerPush, manifest manifest.Manifest, fs afero.Afero) (errs []error) { +func LintDockerPushTask(docker manifest.DockerPush, fs afero.Afero) (errs []error) { if docker.Image == "" { errs = append(errs, NewErrMissingField("image")) } else { @@ -61,5 +62,17 @@ func LintDockerPushTask(docker manifest.DockerPush, manifest manifest.Manifest, errs = append(errs, ErrDockerPushTag.AsWarning()) } + for _, platform := range docker.Platforms { + if !slices.Contains([]string{"linux/amd64", "linux/arm64"}, platform) { + errs = append(errs, ErrDockerPlatformUnknown) + } + } + + for k, v := range docker.Vars { + if strings.HasPrefix(v, "((") && strings.HasSuffix(v, "))") && !strings.HasPrefix(k, "ARTIFACTORY_") { + errs = append(errs, ErrDockerVarSecret.WithValue(k).AsWarning()) + } + } + return errs } diff --git a/linters/docker-push_test.go b/linters/docker-push_test.go index 30b298ab..013f060f 100644 --- a/linters/docker-push_test.go +++ b/linters/docker-push_test.go @@ -13,7 +13,7 @@ var emptyManifest = manifest.Manifest{} func TestDockerPushTaskWithEmptyTask(t *testing.T) { fs := afero.Afero{Fs: afero.NewMemMapFs()} - errors := LintDockerPushTask(manifest.DockerPush{}, emptyManifest, fs) + errors := LintDockerPushTask(manifest.DockerPush{}, fs) assertContainsError(t, errors, NewErrMissingField("image")) } @@ -25,7 +25,7 @@ func TestDockerPushTaskWithBadRepo(t *testing.T) { Image: "asd", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assertContainsError(t, errors, ErrInvalidField.WithValue("image")) } @@ -40,7 +40,7 @@ func TestDockerPushTaskWithoutTeamDirectoryInHalfpipeRepo(t *testing.T) { DockerfilePath: "Dockerfile", } - errs := LintDockerPushTask(task, emptyManifest, fs) + errs := LintDockerPushTask(task, fs) assertContainsError(t, errs, ErrInvalidField.WithValue("image")) } @@ -55,7 +55,7 @@ func TestDockerPushTaskWithTeamDirectoryInHalfpipeRepo(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) } @@ -70,7 +70,7 @@ func TestDockerPushTaskWithoutTeamDirectoryInGCRRepo(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) } @@ -85,7 +85,7 @@ func TestDockerPushTaskWhenDockerfileIsMissing(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assertContainsError(t, errors, ErrFileNotFound) }) @@ -100,7 +100,7 @@ func TestDockerPushTaskWhenDockerfileIsMissing(t *testing.T) { RestoreArtifacts: true, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assertNotContainsError(t, errors, ErrFileNotFound) }) } @@ -122,7 +122,7 @@ func TestDockerPushTaskWithCorrectData(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -141,7 +141,7 @@ func TestDockerPushTaskWithCorrectData(t *testing.T) { DockerfilePath: "dockerfile/Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -160,7 +160,7 @@ func TestDockerPushTaskWithCorrectData(t *testing.T) { DockerfilePath: "../dockerfile/Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -182,7 +182,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { BuildPath: "buildPathDoesntExist", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 1) assertContainsError(t, errors, ErrInvalidField.WithValue("build_path")) }) @@ -206,7 +206,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { BuildPath: buildPath, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 1) assertContainsError(t, errors, ErrInvalidField.WithValue("build_path")) }) @@ -230,7 +230,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { BuildPath: buildPath, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -253,7 +253,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { BuildPath: buildPath, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Len(t, errors, 0) }) @@ -269,7 +269,7 @@ func TestDockerPushWithBuildPath(t *testing.T) { RestoreArtifacts: true, } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Empty(t, errors) }) @@ -291,15 +291,15 @@ func TestDockerPushRetries(t *testing.T) { } task.Retries = -1 - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assertContainsError(t, errors, ErrInvalidField.WithValue("retries")) task.Retries = 6 - errors = LintDockerPushTask(task, emptyManifest, fs) + errors = LintDockerPushTask(task, fs) assertContainsError(t, errors, ErrInvalidField.WithValue("retries")) task.Retries = 4 - errors = LintDockerPushTask(task, emptyManifest, fs) + errors = LintDockerPushTask(task, fs) assert.Len(t, errors, 0) } @@ -315,7 +315,7 @@ func TestDockerPushTag(t *testing.T) { DockerfilePath: "Dockerfile", } - errors := LintDockerPushTask(task, emptyManifest, fs) + errors := LintDockerPushTask(task, fs) assert.Empty(t, errors) }) @@ -331,7 +331,7 @@ func TestDockerPushTag(t *testing.T) { Tag: "yolo", } - errs := LintDockerPushTask(task, emptyManifest, fs) + errs := LintDockerPushTask(task, fs) assertContainsError(t, errs, ErrDockerPushTag) }) } @@ -349,13 +349,11 @@ func TestMultiplePlatforms(t *testing.T) { Platforms: []string{"linux/arm64", "linux/amd64"}, } - m := manifest.Manifest{Platform: "actions"} - - errors := LintDockerPushTask(task, m, fs) + errors := LintDockerPushTask(task, fs) assert.Empty(t, errors) }) - t.Run("only linux/amd64 for concourse is fine", func(t *testing.T) { + t.Run("errors when unknown platform in docker push", func(t *testing.T) { fs := afero.Afero{Fs: afero.NewMemMapFs()} fs.WriteFile("Dockerfile", []byte("FROM ubuntu"), 0777) @@ -364,12 +362,30 @@ func TestMultiplePlatforms(t *testing.T) { Username: "asd", Password: "asdf", DockerfilePath: "Dockerfile", - Platforms: []string{"linux/amd64"}, + Platforms: []string{"linux/ad64"}, } - m := manifest.Manifest{Platform: "actions"} + errors := LintDockerPushTask(task, fs) + assertContainsError(t, errors, ErrDockerPlatformUnknown) + }) +} - errors := LintDockerPushTask(task, m, fs) - assert.Empty(t, errors) +func TestSecrets(t *testing.T) { + t.Run("no secrets in vars please", func(t *testing.T) { + fs := afero.Afero{Fs: afero.NewMemMapFs()} + fs.WriteFile("Dockerfile", []byte("FROM ubuntu"), 0777) + + task := manifest.DockerPush{ + Image: "asd/asd", + Username: "asd", + Password: "asdf", + DockerfilePath: "Dockerfile", + Platforms: []string{"linux/arm64", "linux/amd64"}, + Vars: manifest.Vars{"var1": "((a.secret))"}, + } + + errors := LintDockerPushTask(task, fs) + assertContainsError(t, errors, ErrDockerVarSecret.WithValue("var1").AsWarning()) }) + } diff --git a/linters/errors.go b/linters/errors.go index 8092d33e..95ebd478 100644 --- a/linters/errors.go +++ b/linters/errors.go @@ -33,10 +33,14 @@ var ( ErrCFFromArtifact = newError("this file must be saved as an artifact in a previous task") ErrCFPrePromoteArtifact = newError("cannot have pre promote tasks with CF manifest restored from artifact") - ErrUnsupportedRegistry = newError("image must be from halfpipe registry. Please see ") - ErrDockerPushTag = newError("the field 'tag' is no longer used and is safe to delete") - ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") - ErrMultipleTriggers = newError("cannot have multiple triggers of this type") + ErrUnsupportedRegistry = newError("image must be from halfpipe registry. Please see ") + ErrDockerPushTag = newError("the field 'tag' is no longer used and is safe to delete") + + ErrDockerPlatformUnknown = newError("only linux/amd64 and/or linux/arm64 are supported") + ErrDockerComposeVersion = newError("the docker-compose file version used is deprecated. All services must be under the 'services' key and 'Version' must be '2' or higher. Please see ") + ErrDockerVarSecret = newError("using a secret in docker build vars is not secure. See the 'secrets' option of the docker-push task") + + ErrMultipleTriggers = newError("cannot have multiple triggers of this type") ErrVelaVariableMissing = newError("vela manifest variable is not specified in halfpipe manifest") ErrVelaNamespace = newError("vela namespace must start with 'katee-'") @@ -73,8 +77,7 @@ func (e Error) Error() string { } func (e Error) AsWarning() Error { - e.level = "warning" - return e + return Error{err: e.err, level: "warning", value: e.value} } func (e Error) IsWarning() bool { diff --git a/linters/task.go b/linters/task.go index 14ce294a..757d1051 100644 --- a/linters/task.go +++ b/linters/task.go @@ -17,7 +17,7 @@ type taskLinter struct { lintDeployCFTask func(task manifest.DeployCF, readCfManifest cf.ManifestReader, fs afero.Afero) []error lintDeployKateeTask func(task manifest.DeployKatee, man manifest.Manifest, fs afero.Afero) []error LintPrePromoteTask func(task manifest.Task) []error - lintDockerPushTask func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) []error + lintDockerPushTask func(task manifest.DockerPush, fs afero.Afero) []error lintDockerComposeTask func(task manifest.DockerCompose, fs afero.Afero) []error lintConsumerIntegrationTestTask func(task manifest.ConsumerIntegrationTest, providerHostRequired bool) []error lintDeployMLZipTask func(task manifest.DeployMLZip) []error @@ -97,7 +97,7 @@ func (linter taskLinter) lintTasks(listName string, ts []manifest.Task, man mani case manifest.DeployKatee: errs = linter.lintDeployKateeTask(task, man, linter.Fs) case manifest.DockerPush: - errs = linter.lintDockerPushTask(task, man, linter.Fs) + errs = linter.lintDockerPushTask(task, linter.Fs) case manifest.DockerCompose: errs = linter.lintDockerComposeTask(task, linter.Fs) case manifest.ConsumerIntegrationTest: diff --git a/linters/task_test.go b/linters/task_test.go index 454e69d0..5aa0aa5f 100644 --- a/linters/task_test.go +++ b/linters/task_test.go @@ -113,7 +113,7 @@ func TestCallsOutToTheLintersCorrectly(t *testing.T) { calledLintPrePromoteTasksNum++ return }, - lintDockerPushTask: func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) (errs []error) { + lintDockerPushTask: func(task manifest.DockerPush, fs afero.Afero) (errs []error) { calledLintDockerPushTask = true calledLintDockerPushTaskNum++ return @@ -232,7 +232,7 @@ func TestMergesTheErrorsAndWarningsCorrectlyWithPrePromote(t *testing.T) { LintPrePromoteTask: func(tasks manifest.Task) (errs []error) { return []error{prePromoteErr, prePromoteWarn} }, - lintDockerPushTask: func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) (errs []error) { + lintDockerPushTask: func(task manifest.DockerPush, fs afero.Afero) (errs []error) { return []error{dockerPushErr, dockerPushWarn} }, lintDeployMLZipTask: func(task manifest.DeployMLZip) (errs []error) { @@ -323,7 +323,7 @@ func TestMergesTheErrorsAndWarningsCorrectlyWithParallel(t *testing.T) { LintPrePromoteTask: func(tasks manifest.Task) (errs []error) { return []error{prePromoteErr, prePromoteWarn} }, - lintDockerPushTask: func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) (errs []error) { + lintDockerPushTask: func(task manifest.DockerPush, fs afero.Afero) (errs []error) { return []error{dockerPushErr, dockerPushWarn} }, lintDeployMLZipTask: func(task manifest.DeployMLZip) (errs []error) { @@ -557,7 +557,7 @@ func TestLintTimeout(t *testing.T) { return }, LintPrePromoteTask: func(task manifest.Task) (errs []error) { return }, - lintDockerPushTask: func(task manifest.DockerPush, man manifest.Manifest, fs afero.Afero) (errs []error) { + lintDockerPushTask: func(task manifest.DockerPush, fs afero.Afero) (errs []error) { return }, lintDockerComposeTask: func(task manifest.DockerCompose, fs afero.Afero) (errs []error) { diff --git a/manifest/docker_push.go b/manifest/docker_push.go index d9274138..c6af13e0 100644 --- a/manifest/docker_push.go +++ b/manifest/docker_push.go @@ -10,6 +10,7 @@ type DockerPush struct { IgnoreVulnerabilities bool `json:"ignore_vulnerabilities,omitempty" yaml:"ignore_vulnerabilities,omitempty"` ScanTimeout int `json:"scan_timeout,omitempty" yaml:"scan_timeout,omitempty"` Vars Vars `yaml:"vars,omitempty" secretAllowed:"true"` + Secrets Vars `yaml:"secrets,omitempty" secretAllowed:"true"` RestoreArtifacts bool `json:"restore_artifacts" yaml:"restore_artifacts,omitempty"` Retries int `yaml:"retries,omitempty"` NotifyOnSuccess bool `json:"notify_on_success,omitempty" yaml:"notify_on_success,omitempty"` diff --git a/manifest/manifest.go b/manifest/manifest.go index 8b855794..772dfdd0 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -69,6 +69,15 @@ func (tl TaskList) Flatten() (updated TaskList) { return } +func (tl TaskList) GetTask(name string) Task { + for _, t := range tl.Flatten() { + if t.GetName() == name { + return t + } + } + return nil +} + func (tl TaskList) PreviousTaskNames(currentIndex int) []string { if currentIndex == 0 { return []string{} diff --git a/renderers/actions/actions.go b/renderers/actions/actions.go index 1fe796a4..00327712 100644 --- a/renderers/actions/actions.go +++ b/renderers/actions/actions.go @@ -13,8 +13,6 @@ import ( "github.com/springernature/halfpipe/manifest" ) -const eeRunner = "ee-runner" - var globalEnv = Env{ "ARTIFACTORY_PASSWORD": githubSecrets.ArtifactoryPassword, "ARTIFACTORY_URL": githubSecrets.ArtifactoryURL, @@ -81,7 +79,7 @@ func (a *Actions) jobs(tasks manifest.TaskList, man manifest.Manifest, parent *p job := Job{ Name: task.GetName(), - RunsOn: eeRunner, + RunsOn: config.ActionsRunnerName, Steps: convertSecrets(steps, man.Team), TimeoutMinutes: timeoutInMinutes(task.GetTimeout()), Needs: needs, diff --git a/renderers/actions/deploy_cf.go b/renderers/actions/deploy_cf.go index 0f47917b..4cc7f4fd 100644 --- a/renderers/actions/deploy_cf.go +++ b/renderers/actions/deploy_cf.go @@ -75,6 +75,15 @@ func (a *Actions) deployCFSteps(task manifest.DeployCF, man manifest.Manifest) ( } deploySteps = append(deploySteps, push) + deploySteps = append(deploySteps, Step{ + Name: "cf logs --recent", + If: "failure()", + Uses: uses, + With: addCommonParams(With{ + "command": "halfpipe-logs", + }), + }) + deploySteps = append(deploySteps, Step{ Name: "Check", Uses: uses, diff --git a/renderers/actions/docker_push.go b/renderers/actions/docker_push.go index 523abc23..c3637002 100644 --- a/renderers/actions/docker_push.go +++ b/renderers/actions/docker_push.go @@ -2,24 +2,17 @@ package actions import ( "fmt" - "github.com/springernature/halfpipe/renderers/shared" "path" "strings" + "github.com/springernature/halfpipe/renderers/shared" + "github.com/springernature/halfpipe/manifest" ) func (a *Actions) dockerPushSteps(task manifest.DockerPush) (steps Steps) { steps = dockerLogin(task.Image, task.Username, task.Password) - buildArgs := map[string]string{} - for k, v := range globalEnv { - buildArgs[k] = v - } - for k, v := range task.Vars { - buildArgs[k] = v - } - - steps = append(steps, buildImage(a, task, buildArgs)) + steps = append(steps, buildImage(a, task)) steps = append(steps, scanImage(a, task)) steps = append(steps, pushImage(task)) steps = append(steps, repositoryDispatch(task.Image)) @@ -46,23 +39,37 @@ func repositoryDispatch(eventName string) Step { } } -func buildImage(a *Actions, task manifest.DockerPush, buildArgs map[string]string) Step { +func buildImage(a *Actions, task manifest.DockerPush) Step { + buildArgs := map[string]string{ + "ARTIFACTORY_PASSWORD": "", + "ARTIFACTORY_URL": "", + "ARTIFACTORY_USERNAME": "", + "BUILD_VERSION": "", + "GIT_REVISION": "", + "RUNNING_IN_CI": "", + } + for k, v := range task.Vars { + buildArgs[k] = v + } + step := Step{ Name: "Build Image", - Uses: "docker/build-push-action@v4", + Uses: "docker/build-push-action@v5", With: With{ "context": path.Join(a.workingDir, task.BuildPath), "file": path.Join(a.workingDir, task.DockerfilePath), "push": true, - "tags": shared.CachePath(task, ":${{ env.GIT_REVISION }}"), + "tags": shared.CachePath(task, "${{ env.GIT_REVISION }}"), "build-args": MultiLine{buildArgs}, "platforms": strings.Join(task.Platforms, ","), "provenance": false, + "secrets": MultiLine{task.Secrets}, }, } if task.UseCache { - step.With["cache-from"] = fmt.Sprintf("type=registry,ref=%s", shared.CachePath(task, "")) + step.With["tags"] = fmt.Sprintf("%s\n%s", step.With["tags"], shared.CachePath(task, "buildcache")) + step.With["cache-from"] = fmt.Sprintf("type=registry,ref=%s", shared.CachePath(task, "buildcache")) step.With["cache-to"] = "type=inline" } diff --git a/renderers/actions/secrets.go b/renderers/actions/secrets.go index f670bb7b..935d9841 100644 --- a/renderers/actions/secrets.go +++ b/renderers/actions/secrets.go @@ -2,6 +2,7 @@ package actions import ( "fmt" + "github.com/springernature/halfpipe/renderers/shared/secrets" "sort" "strings" ) @@ -28,92 +29,25 @@ var githubSecrets = struct { VaultSecretID: "${{ secrets.VAULT_SECRET_ID }}", } -type Secret struct { - vaultPath string +func secretVar(s *secrets.Secret) string { + return fmt.Sprintf("${{ steps.secrets.outputs.%s }}", secretOutputVar(s)) } -func (s *Secret) actionsVar() string { - return fmt.Sprintf("${{ steps.secrets.outputs.%s }}", s.outputVar()) +func secretVaultPath(s *secrets.Secret) string { + return fmt.Sprintf("/springernature/data/%s %s", s.MapPath, s.Key) } - -func (s *Secret) outputVar() string { - ov := strings.ReplaceAll(s.vaultPath, "/", "_") +func secretOutputVar(s *secrets.Secret) string { + ov := strings.ReplaceAll(secretVaultPath(s), "/", "_") ov = strings.ReplaceAll(ov, " ", "_") ov = strings.TrimPrefix(ov, "_") return ov } -// check if a secret matches one of the shared secrets -// vault kv list /springernature/shared -func isShared(s string) bool { - return map[string]bool{ - "PPG-gradle-version-reporter": true, - "PPG-owasp-dependency-reporter": true, - "artifactory": true, - "artifactory-support": true, - "artifactory_test": true, - "bla": true, - "burpsuiteenterprise": true, - "contrastsecurity": true, - "ee-sso-route-service": true, - "fastly": true, - "grafana": true, - "halfpipe-artifacts": true, - "halfpipe-docker-config": true, - "halfpipe-gcr": true, - "halfpipe-github": true, - "halfpipe-ml-deploy": true, - "halfpipe-semver": true, - "halfpipe-slack": true, - "katee-tls-dev": true, - "katee-tls-prod": true, - "sentry-release-integration": true, - }[s] -} - -func toSecret(s string, team string) *Secret { - if !isSecret(s) { - return nil - } - - secretValue := s[2 : len(s)-2] - - if isKeyValueSecret(secretValue) { - parts := strings.Split(secretValue, ".") - if isShared(parts[0]) { - team = "shared" - } - return &Secret{ - vaultPath: fmt.Sprintf("/springernature/data/%s/%s %s", team, parts[0], parts[1]), - } - } - - if isAbsolutePathSecret(secretValue) { - return &Secret{ - vaultPath: secretValue, - } - } - - return nil -} - -func isSecret(s string) bool { - return strings.HasPrefix(s, "((") && strings.HasSuffix(s, "))") -} - -func isAbsolutePathSecret(s string) bool { - return len(strings.Split(s, " ")) == 2 -} - -func isKeyValueSecret(s string) bool { - return len(strings.Split(s, ".")) == 2 -} - -func secretsToActionsSecret(secrets []*Secret) string { +func secretsToActionsSecret(secrets []*secrets.Secret) string { uniqueSecrets := map[string]string{} for _, s := range secrets { - x := fmt.Sprintf("%s | %s ;\n", s.vaultPath, s.outputVar()) - uniqueSecrets[s.outputVar()] = x + x := fmt.Sprintf("%s | %s ;\n", secretVaultPath(s), secretOutputVar(s)) + uniqueSecrets[secretOutputVar(s)] = x } var secs []string @@ -125,11 +59,11 @@ func secretsToActionsSecret(secrets []*Secret) string { return strings.Join(secs, "") } -func fetchSecrets(secrets []*Secret, team string) Step { +func fetchSecrets(secrets []*secrets.Secret) Step { return Step{ Name: "Vault secrets", ID: "secrets", - Uses: "hashicorp/vault-action@v2.7.3", + Uses: "hashicorp/vault-action@v2.7.4", With: With{ "url": "https://vault.halfpipe.io", "method": "approle", @@ -142,7 +76,7 @@ func fetchSecrets(secrets []*Secret, team string) Step { } func convertSecrets(steps Steps, team string) (newSteps Steps) { - secrets := []*Secret{} + allSecrets := []*secrets.Secret{} for _, step := range steps { newWith := With{} @@ -150,39 +84,39 @@ func convertSecrets(steps Steps, team string) (newSteps Steps) { switch v := value.(type) { case MultiLine: secretList, multiLineStringWithActionSecret := multiLineStringToSecret(v.m, team) - secrets = append(secrets, secretList...) + allSecrets = append(allSecrets, secretList...) value = MultiLine{multiLineStringWithActionSecret} default: - if s := toSecret(fmt.Sprintf("%v", value), team); s != nil { - secrets = append(secrets, s) - value = s.actionsVar() + if s := secrets.New(fmt.Sprintf("%v", value), team); s != nil { + allSecrets = append(allSecrets, s) + value = secretVar(s) } } newWith[key] = value } step.With = newWith for k, v := range step.Env { - if s := toSecret(v, team); s != nil { - secrets = append(secrets, s) - step.Env[k] = s.actionsVar() + if s := secrets.New(v, team); s != nil { + allSecrets = append(allSecrets, s) + step.Env[k] = secretVar(s) } } newSteps = append(newSteps, step) } - if len(secrets) > 0 { - newSteps = append(Steps{fetchSecrets(secrets, team)}, newSteps...) + if len(allSecrets) > 0 { + newSteps = append(Steps{fetchSecrets(allSecrets)}, newSteps...) } return newSteps } -func multiLineStringToSecret(ml map[string]string, team string) ([]*Secret, map[string]string) { +func multiLineStringToSecret(ml map[string]string, team string) ([]*secrets.Secret, map[string]string) { m := make(map[string]string) - var sec []*Secret + var sec []*secrets.Secret for k, v := range ml { - if a := toSecret(v, team); a != nil { + if a := secrets.New(v, team); a != nil { sec = append(sec, a) - m[k] = a.actionsVar() + m[k] = secretVar(a) } else { m[k] = v } diff --git a/renderers/actions/update.go b/renderers/actions/update.go index 2f0117d2..b36dd334 100644 --- a/renderers/actions/update.go +++ b/renderers/actions/update.go @@ -6,19 +6,10 @@ import ( ) func (a *Actions) updateSteps(task manifest.Update, man manifest.Manifest) Steps { - cdPrefix := "" - if a.workingDir != "" { - cdPrefix = fmt.Sprintf("cd %s; ", a.workingDir) - } - update := Step{ Name: "Sync workflow with halfpipe manifest", ID: "sync", - Uses: "docker://eu.gcr.io/halfpipe-io/halfpipe-auto-update", - With: With{ - "args": fmt.Sprintf(`-c "%supdate-actions-workflow"`, cdPrefix), - "entrypoint": "/bin/bash", - }, + Run: "halfpipe-update-workflow", Env: Env{ "HALFPIPE_FILE_PATH": a.halfpipeFilePath, }, diff --git a/renderers/actions/workflow.go b/renderers/actions/workflow.go index a97fd351..4b1e9e1f 100644 --- a/renderers/actions/workflow.go +++ b/renderers/actions/workflow.go @@ -92,7 +92,11 @@ type MultiLine struct { func (ml MultiLine) MarshalYAML() (interface{}, error) { var out []string for k, v := range ml.m { - out = append(out, fmt.Sprintf("%s=%s\n", k, v)) + if v == "" { + out = append(out, fmt.Sprintf("%s\n", k)) + } else { + out = append(out, fmt.Sprintf("%s=%s\n", k, v)) + } } sort.Strings(out) diff --git a/renderers/concourse/deploy_cf.go b/renderers/concourse/deploy_cf.go index 237c7930..46319ade 100644 --- a/renderers/concourse/deploy_cf.go +++ b/renderers/concourse/deploy_cf.go @@ -43,12 +43,12 @@ func (c Concourse) deployCFJob(task manifest.DeployCF, man manifest.Manifest, ba if len(task.PrePromote) == 0 { steps = append(steps, deploy.pushApp()) } else if task.Rolling { - steps = append(steps, deploy.pushCandidateApp()) + steps = append(steps, deploy.logsOnFailure(deploy.pushCandidateApp())) steps = append(steps, c.prePromoteTasks(deploy)...) steps = append(steps, deploy.pushApp()) steps = append(steps, deploy.removeTestApp()) } else { - steps = append(steps, deploy.pushCandidateApp()) + steps = append(steps, deploy.logsOnFailure(deploy.pushCandidateApp())) steps = append(steps, deploy.checkApp()) steps = append(steps, c.prePromoteTasks(deploy)...) steps = append(steps, deploy.promoteCandidateAppToLive()) @@ -232,7 +232,27 @@ func (d deployCF) pushApp() atc.Step { push.Params["buildVersionPath"] = path.Join("version", "version") } - return stepWithAttemptsAndTimeout(&push, d.task.GetAttempts(), d.task.GetTimeout()) + return d.logsOnFailure(stepWithAttemptsAndTimeout(&push, d.task.GetAttempts(), d.task.GetTimeout())) +} + +func (d deployCF) logsOnFailure(stepConfig atc.Step) atc.Step { + return atc.Step{ + Config: &atc.OnFailureStep{ + Step: stepConfig.Config, + Hook: atc.Step{ + Config: &atc.PutStep{ + Name: "cf logs --recent", + Resource: d.resourceName, + Params: atc.Params{ + "command": "halfpipe-logs", + "cliVersion": d.task.CliVersion, + "manifestPath": d.manifestPath, + }, + NoGet: true, + }, + }, + }, + } } func (d deployCF) configureSSO() atc.Step { diff --git a/renderers/concourse/docker_push.go b/renderers/concourse/docker_push.go index f15ef681..332f2187 100644 --- a/renderers/concourse/docker_push.go +++ b/renderers/concourse/docker_push.go @@ -6,6 +6,7 @@ import ( "github.com/springernature/halfpipe/config" "github.com/springernature/halfpipe/manifest" "github.com/springernature/halfpipe/renderers/shared" + "golang.org/x/exp/slices" "path" "strings" ) @@ -16,16 +17,10 @@ var tagListFile = path.Join(tagList_Dir, "tagList") func (c Concourse) dockerPushJob(task manifest.DockerPush, basePath string, man manifest.Manifest) atc.JobConfig { var steps []atc.Step - resourceName := manifest.DockerTrigger{Image: task.Image}.GetTriggerName() - - fullBasePath := path.Join(gitDir, basePath) - if task.RestoreArtifacts { - fullBasePath = path.Join(dockerBuildTmpDir, basePath) - } steps = append(steps, restoreArtifacts(task)...) steps = append(steps, createTagList(task, man.FeatureToggles.UpdatePipeline())...) - steps = append(steps, buildAndPush(task, resourceName, fullBasePath)...) + steps = append(steps, buildAndPush(task, basePath)...) return atc.JobConfig{ Name: task.GetName(), @@ -104,21 +99,9 @@ func createTagList(task manifest.DockerPush, updatePipeline bool) []atc.Step { return append([]atc.Step{}, stepWithAttemptsAndTimeout(createTagList, task.GetAttempts(), task.Timeout)) } -func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { - var imageFile string - var gitRef string - if !multiPlatform(task) { - imageFile = fmt.Sprintf("--input %s", path.Join(relativePathToRepoRoot(gitDir, fullBasePath), "image/image.tar")) - } else { - imageFile = shared.CachePath(task, "") - gitRef = fmt.Sprintf(":$(cat %s)", path.Join(gitDir, ".git", "ref")) - fullBasePath = "" - } - - //exitCode := 1 - //if task.IgnoreVulnerabilities { - // exitCode = 0 - //} +func trivyTask(task manifest.DockerPush, fullBasePath string, basePath string) atc.StepConfig { + imageFile := shared.CachePath(task, "") + gitRef := fmt.Sprintf(":$(cat %s)", pathToGitRef(gitDir, basePath)) // temporary: always exit 0 until we have communicated the ignoreVulnerabilites opt-in exitCode := 0 @@ -142,9 +125,11 @@ func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { }, "\n")}, Dir: fullBasePath, }, + Params: atc.TaskEnv{ + "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", + }, Inputs: []atc.TaskInputConfig{ {Name: gitDir}, - {Name: "image"}, }, }, } @@ -156,85 +141,86 @@ func trivyTask(task manifest.DockerPush, fullBasePath string) atc.StepConfig { return step } -func buildAndPush(task manifest.DockerPush, resourceName string, fullBasePath string) []atc.Step { +func buildAndPush(task manifest.DockerPush, basePath string) []atc.Step { var steps []atc.Step image, tag := shared.SplitTag(task.Image) dockerImageWithCachePath := shared.CachePath(task, "") + fullBasePath := path.Join(gitDir, basePath) + if task.RestoreArtifacts { + fullBasePath = path.Join(dockerBuildTmpDir, basePath) + } + params := atc.TaskEnv{ - "CONTEXT": path.Join(fullBasePath, task.BuildPath), - "DOCKERFILE": path.Join(fullBasePath, task.DockerfilePath), "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", } + var buildStep *atc.TaskStep + + buildCommand := []string{ + "docker buildx build", + fmt.Sprintf("-f %s", path.Join(fullBasePath, task.DockerfilePath)), + "--push", + "--provenance false", + fmt.Sprintf("--platform %s", strings.Join(task.Platforms, ",")), + fmt.Sprintf("--tag %s", shared.CachePath(task, "$(cat git/.git/ref)")), + } + + buildArgs := []string{} for k, v := range convertVars(task.Vars) { - params[fmt.Sprintf("BUILD_ARG_%s", k)] = fmt.Sprintf("%s", v) + params[k] = v.(string) + buildArgs = append(buildArgs, fmt.Sprintf("--build-arg %s", k)) } + slices.Sort(buildArgs) + buildCommand = append(buildCommand, buildArgs...) - var buildStep *atc.TaskStep + secrets := []string{} + for k, v := range convertVars(task.Secrets) { + params[k] = v.(string) + secrets = append(secrets, fmt.Sprintf("--secret id=%s", k)) - if !multiPlatform(task) { - buildStep = &atc.TaskStep{ - Name: "build", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": "concourse/oci-build-task", - }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{ - "-c", - fmt.Sprintf("%s\n%s\n%s", "mkdir ~/.docker", "echo $DOCKER_CONFIG_JSON > ~/.docker/config.json", "build"), - }, - }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - }, - Outputs: []atc.TaskOutputConfig{ - {Name: "image"}, - }, - }, - } - } else { - gitRef := fmt.Sprintf("$(cat %s)", path.Join(gitDir, ".git", "ref")) + } + slices.Sort(secrets) + buildCommand = append(buildCommand, secrets...) - buildStep = &atc.TaskStep{ - Name: "build", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": config.DockerRegistry + "halfpipe-buildx", - "tag": "latest", - "password": "((halfpipe-gcr.private_key))", - "username": "_json_key", - }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{"-c", strings.Join([]string{ - `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - fmt.Sprintf(`docker buildx build -f $DOCKERFILE --platform linux/amd64,linux/arm64 -t %s:%s --push --provenance=false $CONTEXT`, dockerImageWithCachePath, gitRef)}, "\n"), - }, - }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - {Name: tagList_Dir}, + if task.UseCache { + buildCommand = append(buildCommand, fmt.Sprintf("--tag %s", shared.CachePath(task, "buildcache"))) + buildCommand = append(buildCommand, fmt.Sprintf("--cache-from type=registry,ref=%s", shared.CachePath(task, "buildcache"))) + buildCommand = append(buildCommand, "--cache-to type=inline") + } + + buildCommand = append(buildCommand, path.Join(fullBasePath, task.BuildPath)) + + buildCommandStr := strings.Join(buildCommand, ` \ + `) + buildStep = &atc.TaskStep{ + Name: "build", + Privileged: true, + Config: &atc.TaskConfig{ + Platform: "linux", + ImageResource: &atc.ImageResource{ + Type: "registry-image", + Source: atc.Source{ + "repository": config.DockerRegistry + "halfpipe-buildx", + "tag": "latest", + "password": "((halfpipe-gcr.private_key))", + "username": "_json_key", }, - Outputs: []atc.TaskOutputConfig{ - {Name: "image"}, + }, + Params: params, + Run: atc.TaskRunConfig{ + Path: "/bin/sh", + Args: []string{"-c", strings.Join([]string{ + `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, + fmt.Sprintf(`echo $ %s`, buildCommandStr), + buildCommandStr}, "\n"), }, }, - } + Inputs: []atc.TaskInputConfig{ + {Name: gitDir}, + {Name: tagList_Dir}, + }, + }, } if task.ReadsFromArtifacts() { @@ -242,59 +228,42 @@ func buildAndPush(task manifest.DockerPush, resourceName string, fullBasePath st } steps = append(steps, stepWithAttemptsAndTimeout(buildStep, task.GetAttempts(), task.GetTimeout())) - steps = append(steps, stepWithAttemptsAndTimeout(trivyTask(task, fullBasePath), task.GetAttempts(), task.GetTimeout())) + steps = append(steps, stepWithAttemptsAndTimeout(trivyTask(task, fullBasePath, basePath), task.GetAttempts(), task.GetTimeout())) - if !multiPlatform(task) { - putStep := &atc.PutStep{ - Name: resourceName, - Params: atc.Params{ - "image": "image/image.tar", - "additional_tags": tagListFile, - }, - NoGet: true, - } - steps = append(steps, stepWithAttemptsAndTimeout(putStep, task.GetAttempts(), task.GetTimeout())) - } else { - gitRef := fmt.Sprintf("$(cat %s)", path.Join(gitDir, ".git", "ref")) - publishCommand := fmt.Sprintf(`for tag in $(cat %s) %s; do docker buildx imagetools create %s:%s --tag %s:$tag; done`, tagListFile, tag, dockerImageWithCachePath, gitRef, image) + publishCommand := fmt.Sprintf(`for tag in $(cat %s) %s; do docker buildx imagetools create %s:$(cat git/.git/ref) --tag %s:$tag; done`, tagListFile, tag, dockerImageWithCachePath, image) - pushStep := &atc.TaskStep{ - Name: "publish-final-image", - Privileged: true, - Config: &atc.TaskConfig{ - Platform: "linux", - ImageResource: &atc.ImageResource{ - Type: "registry-image", - Source: atc.Source{ - "repository": config.DockerRegistry + "halfpipe-buildx", - "tag": "latest", - "password": "((halfpipe-gcr.private_key))", - "username": "_json_key", - }, - }, - Params: params, - Run: atc.TaskRunConfig{ - Path: "/bin/sh", - Args: []string{"-c", strings.Join([]string{ - `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, - publishCommand, - }, "\n"), - }, - }, - Inputs: []atc.TaskInputConfig{ - {Name: gitDir}, - {Name: tagList_Dir}, + pushStep := &atc.TaskStep{ + Name: "publish-final-image", + Privileged: true, + Config: &atc.TaskConfig{ + Platform: "linux", + ImageResource: &atc.ImageResource{ + Type: "registry-image", + Source: atc.Source{ + "repository": config.DockerRegistry + "halfpipe-buildx", + "tag": "latest", + "password": "((halfpipe-gcr.private_key))", + "username": "_json_key", }, - Outputs: []atc.TaskOutputConfig{ - {Name: "image"}, + }, + Params: atc.TaskEnv{ + "DOCKER_CONFIG_JSON": "((halfpipe-gcr.docker_config))", + }, + Run: atc.TaskRunConfig{ + Path: "/bin/sh", + Args: []string{"-c", strings.Join([]string{ + `echo $DOCKER_CONFIG_JSON > ~/.docker/config.json`, + publishCommand, + }, "\n"), }, }, - } - steps = append(steps, stepWithAttemptsAndTimeout(pushStep, task.GetAttempts(), task.GetTimeout())) + Inputs: []atc.TaskInputConfig{ + {Name: gitDir}, + {Name: tagList_Dir}, + }, + }, } - return steps -} + steps = append(steps, stepWithAttemptsAndTimeout(pushStep, task.GetAttempts(), task.GetTimeout())) -func multiPlatform(task manifest.DockerPush) bool { - return !(len(task.Platforms) == 1 && task.Platforms[0] == "linux/amd64") + return steps } diff --git a/renderers/concourse/pipeline.go b/renderers/concourse/pipeline.go index aa2a8260..630c3c6b 100644 --- a/renderers/concourse/pipeline.go +++ b/renderers/concourse/pipeline.go @@ -184,22 +184,6 @@ func (c Concourse) initialPlan(man manifest.Manifest, task manifest.Task, previo return steps } -func (c Concourse) dockerPushResources(tasks manifest.TaskList) (resourceConfigs atc.ResourceConfigs) { - for _, task := range tasks { - switch task := task.(type) { - case manifest.DockerPush: - if len(task.Platforms) == 1 { - resourceConfigs = append(resourceConfigs, c.dockerPushResource(task)) - } - case manifest.Parallel: - resourceConfigs = append(resourceConfigs, c.dockerPushResources(task.Tasks)...) - case manifest.Sequence: - resourceConfigs = append(resourceConfigs, c.dockerPushResources(task.Tasks)...) - } - } - - return resourceConfigs -} func (c Concourse) pipelineResources(triggers manifest.TriggerList) (resourceTypes atc.ResourceTypes, resourceConfigs atc.ResourceConfigs) { for _, trigger := range triggers { @@ -268,8 +252,6 @@ func (c Concourse) resourceConfigs(man manifest.Manifest) (resourceTypes atc.Res resourceConfigs = append(resourceConfigs, c.versionResource(man)) } - resourceConfigs = append(resourceConfigs, c.dockerPushResources(man.Tasks)...) - cfResourceTypes, cfResources := c.cfPushResources(man) resourceTypes = append(resourceTypes, cfResourceTypes...) resourceConfigs = append(resourceConfigs, cfResources...) @@ -569,5 +551,4 @@ func stepWithAttemptsAndTimeout(stepConfig atc.StepConfig, attempts int, timeout Attempts: attempts, }, } - } diff --git a/renderers/concourse/pipeline_resources.go b/renderers/concourse/pipeline_resources.go index 69dca598..86ec60a1 100644 --- a/renderers/concourse/pipeline_resources.go +++ b/renderers/concourse/pipeline_resources.go @@ -230,19 +230,6 @@ func (c Concourse) deployCFResource(deployCF manifest.DeployCF, resourceName str } } -func (c Concourse) dockerPushResource(docker manifest.DockerPush) atc.ResourceConfig { - return atc.ResourceConfig{ - Name: manifest.DockerTrigger{Image: docker.Image}.GetTriggerName(), - Type: "registry-image", - Source: atc.Source{ - "username": docker.Username, - "password": docker.Password, - "repository": docker.Image, - }, - CheckEvery: &longResourceCheckInterval, - } -} - func (c Concourse) dockerTriggerResource(trigger manifest.DockerTrigger) atc.ResourceConfig { config := atc.ResourceConfig{ Name: trigger.GetTriggerName(), diff --git a/renderers/shared/docker_push.go b/renderers/shared/docker_push.go index 4e10c34b..9f9b545e 100644 --- a/renderers/shared/docker_push.go +++ b/renderers/shared/docker_push.go @@ -9,6 +9,9 @@ import ( func CachePath(task manifest.DockerPush, tag string) string { image, _ := SplitTag(task.Image) + if tag != "" && !strings.HasPrefix(tag, ":") { + tag = fmt.Sprintf(":%s", tag) + } if strings.HasPrefix(task.Image, config.DockerRegistry) { r := strings.Replace(image, config.DockerRegistry, fmt.Sprintf("%scache/", config.DockerRegistry), 1) diff --git a/renderers/shared/secrets/secrets.go b/renderers/shared/secrets/secrets.go new file mode 100644 index 00000000..ed80b01c --- /dev/null +++ b/renderers/shared/secrets/secrets.go @@ -0,0 +1,87 @@ +package secrets + +import ( + "fmt" + "strings" +) + +// Secret models a Vault secret +// MapPath is root-relative path e.g. "/myteam/myproject/mysecretmap" +type Secret struct { + MapPath string + Key string +} + +// New returns a Secret from a string in the "halfpipe" format +// "((map.key))" or "((/path/to/map key))" +func New(s string, team string) *Secret { + if !isSecret(s) { + return nil + } + + secretValue := strings.TrimSpace(s[2 : len(s)-2]) + + if isKeyValueSecret(secretValue) { + parts := strings.Split(secretValue, ".") + if isSharedSecret(parts[0]) { + team = "shared" + } + return &Secret{ + MapPath: fmt.Sprintf("%s/%s", team, parts[0]), + Key: parts[1], + } + } + + if isAbsolutePathSecret(secretValue) { + parts := strings.Split(secretValue, " ") + mapPath := strings.TrimPrefix(parts[0], "/springernature/data/") + mapPath = strings.TrimPrefix(mapPath, "/springernature/") + return &Secret{ + MapPath: mapPath, + Key: parts[1], + } + } + + return nil +} + +func isSecret(s string) bool { + return strings.HasPrefix(s, "((") && strings.HasSuffix(s, "))") +} + +func isAbsolutePathSecret(s string) bool { + return len(strings.Split(s, " ")) == 2 +} + +func isKeyValueSecret(s string) bool { + return len(strings.Split(s, ".")) == 2 +} + +func isSharedSecret(s string) bool { + return map[string]bool{ + "PPG-gradle-version-reporter": true, + "PPG-owasp-dependency-reporter": true, + "artifactory": true, + "artifactory-support": true, + "artifactory_test": true, + "bla": true, + "burpsuiteenterprise": true, + "content_hub-casper-credentials-live": true, + "content_hub-casper-credentials-qa": true, + "contrastsecurity": true, + "eas-sigrid": true, + "ee-sso-route-service": true, + "fastly": true, + "grafana": true, + "halfpipe-artifacts": true, + "halfpipe-docker-config": true, + "halfpipe-gcr": true, + "halfpipe-github": true, + "halfpipe-ml-deploy": true, + "halfpipe-semver": true, + "halfpipe-slack": true, + "katee-tls-dev": true, + "katee-tls-prod": true, + "sentry-release-integration": true, + }[s] +} diff --git a/renderers/shell/shell.go b/renderers/shell/shell.go new file mode 100644 index 00000000..95781d6c --- /dev/null +++ b/renderers/shell/shell.go @@ -0,0 +1,90 @@ +package shell + +import ( + "fmt" + "github.com/springernature/halfpipe" + "github.com/springernature/halfpipe/manifest" + "github.com/springernature/halfpipe/renderers/shared/secrets" + "sort" + "strings" +) + +type shell struct { + taskName string +} + +func New(taskName string) halfpipe.Renderer { + return shell{taskName: taskName} +} + +func (s shell) Render(man manifest.Manifest) (string, error) { + task := man.Tasks.GetTask(s.taskName) + + switch t := task.(type) { + case manifest.Run: + return renderRunCommand(t, man.Team), nil + case manifest.DockerCompose: + return renderDockerComposeCommand(t, man.Team), nil + } + + errMsg := "task not found with name '%s' and type 'run' or 'docker-compose\n\navailable tasks:\n" + for _, t := range man.Tasks.Flatten() { + switch t := t.(type) { + case manifest.Run, manifest.DockerCompose: + errMsg += fmt.Sprintf(" %s", t.GetName()) + } + } + return "", fmt.Errorf(errMsg, s.taskName) +} + +func renderRunCommand(task manifest.Run, team string) string { + s := []string{ + "docker run -it", + `-v "$PWD":/app`, + "-w /app", + } + + vars := []string{} + for k, v := range task.Vars { + vars = append(vars, fmt.Sprintf(`-e %s="%s"`, k, convertSecret(v, team))) + } + sort.Strings(vars) + s = append(s, vars...) + + s = append(s, task.Docker.Image, task.Script) + + return strings.Join(s, " \\ \n ") +} + +func renderDockerComposeCommand(task manifest.DockerCompose, team string) string { + s := []string{ + "docker compose", + fmt.Sprintf("-f %s", task.ComposeFile), + "run", + `-v "$PWD":/app`, + "-w /app", + } + + vars := []string{} + for k, v := range task.Vars { + vars = append(vars, fmt.Sprintf(`-e %s="%s"`, k, convertSecret(v, team))) + } + sort.Strings(vars) + s = append(s, vars...) + + s = append(s, "--use-aliases", task.Service) + + if task.Command != "" { + s = append(s, task.Command) + } + + return strings.Join(s, " \\ \n ") +} + +func convertSecret(s string, team string) string { + secret := secrets.New(s, team) + if secret == nil { + return s + } + return fmt.Sprintf("$(vault kv get -field=%s /springernature/%s)", secret.Key, secret.MapPath) +} diff --git a/renderers/shell/shell_test.go b/renderers/shell/shell_test.go new file mode 100644 index 00000000..918e0cd4 --- /dev/null +++ b/renderers/shell/shell_test.go @@ -0,0 +1,25 @@ +package shell + +import ( + "github.com/springernature/halfpipe/manifest" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestShell_Render_SadPath(t *testing.T) { + + t.Run("task doesn't exist", func(t *testing.T) { + renderer := New("task name that doesn't exist") + actual, err := renderer.Render(manifest.Manifest{Tasks: manifest.TaskList{manifest.Run{Name: "task name"}}}) + assert.Error(t, err) + assert.Empty(t, actual) + }) + + t.Run("task exists but type not supported", func(t *testing.T) { + renderer := New("task name") + actual, err := renderer.Render(manifest.Manifest{Tasks: manifest.TaskList{manifest.DockerPush{Name: "task name"}}}) + assert.Error(t, err) + assert.Empty(t, actual) + }) + +}