Merge branch 'main' into dependabot/github_actions/actions/checkout-4

.github/workflows/codeql.yml

@@ -16,7 +16,6 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
-
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
       with:

.github/workflows/for-dependabot-to-check.yml

@@ -47,7 +47,7 @@ jobs:
     steps:
     - name: Vault secrets
       id: secrets
-      uses: hashicorp/[email protected].3
+      uses: hashicorp/[email protected].4
       with:
         exportEnv: false
         method: approle
@@ -81,6 +81,20 @@ jobs:
         username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }}
       env:
         CF_ENV_VAR_BUILD_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+    - name: cf logs --recent
+      if: failure()
+      uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable
+      with:
+        api: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_api-snpaas }}
+        appPath: .
+        cli_version: cf7
+        command: halfpipe-logs
+        manifestPath: e2e/actions/deploy-cf/manifest.yml
+        org: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_org-snpaas }}
+        password: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_password-snpaas }}
+        space: cf-space
+        testDomain: springernature.app
+        username: ${{ steps.secrets.outputs.springernature_data_halfpipe-team_cloudfoundry_username-snpaas }}
     - name: Check
       uses: docker://eu.gcr.io/halfpipe-io/cf-resource-v2:stable
       with:
@@ -149,22 +163,24 @@ jobs:
         ssh-key: ${{ secrets.EE_GITHUB_PRIVATE_KEY }}
         submodules: recursive
     - name: Build Image
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
-          ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }}
-          ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }}
-          ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }}
-          BUILD_VERSION=2.${{ github.run_number }}.0
-          GIT_REVISION=${{ github.sha }}
-          RUNNING_IN_CI=true
-          VAULT_ROLE_ID=${{ secrets.VAULT_ROLE_ID }}
-          VAULT_SECRET_ID=${{ secrets.VAULT_SECRET_ID }}
+          ARTIFACTORY_PASSWORD
+          ARTIFACTORY_URL
+          ARTIFACTORY_USERNAME
+          BUILD_VERSION
+          GIT_REVISION
+          RUNNING_IN_CI
         context: .
         file: e2e/actions/docker-push/Dockerfile
         platforms: linux/amd64
         provenance: false
         push: true
+        secrets: |
+          ARTIFACTORY_PASSWORD=${{ secrets.EE_ARTIFACTORY_PASSWORD }}
+          ARTIFACTORY_URL=${{ secrets.EE_ARTIFACTORY_URL }}
+          ARTIFACTORY_USERNAME=${{ secrets.EE_ARTIFACTORY_USERNAME }}
         tags: eu.gcr.io/halfpipe-io/cache/blah:${{ env.GIT_REVISION }}
     - name: Run Trivy vulnerability scanner
       uses: docker://aquasec/trivy

.gitignore

@@ -7,3 +7,4 @@ vendor
 pipelineActual.yml
 workflowActual.yml
 e2e/concourse/init/.halfpipe.io
+e2e/shell/all/*_actual.txt

README.md

@@ -44,8 +44,7 @@ We also use [GitHub Actions](https://github.com/springernature/halfpipe/actions)
 dependabot will raise PRs. Alternatively, to manually update all deps:
 
 ```bash
-go get -t -u ./...
-go mod tidy
+go get -t -u ./... && go mod tidy
 ```
 
 ### GitHub actions 

build.sh

@@ -13,6 +13,10 @@ if [ "${1-}" = "ci" ]; then
     go_opts="-mod=readonly"
 fi
 
+
+# https://github.com/golang/go/wiki/LoopvarExperiment
+export GOEXPERIMENT=loopvar
+
 echo [1/6] fmt
 go fmt ./...
 

cmd/cmds/actions-migration-help.go

@@ -14,8 +14,8 @@ var actionsMigrationHelp = &cobra.Command{
 	Use:   "actions-migration-help",
 	Short: "Prints out the steps needed to migrate from Concourse to Actions",
 	Run: func(cmd *cobra.Command, args []string) {
-		man, controller := getManifestAndController(formatInput(Input))
-		response := controller.Process(man)
+		man, controller := getManifestAndController(formatInput(Input), nil)
+		response, _ := controller.Process(man)
 
 		tpl, _ := template.New("").Parse(`
 To migrate a Concourse pipeline to Actions you must do the following steps

cmd/cmds/exec.go

@@ -0,0 +1,32 @@
+package cmds
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/springernature/halfpipe/renderers/shell"
+	"os"
+)
+
+func init() {
+	rootCmd.AddCommand(execCmd)
+}
+
+var execCmd = &cobra.Command{
+	Use:   "exec <task name>",
+	Short: "Execute a task locally",
+	Args:  cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		taskName := args[0]
+
+		shellRenderer := shell.New(taskName)
+		man, controller := getManifestAndController(formatInput(Input), shellRenderer)
+
+		response, err := controller.Process(man)
+		if err != nil {
+			printErr(err)
+			os.Exit(1)
+		}
+		outputLintResults(response.LintResults)
+		fmt.Println(response)
+	},
+}

cmd/cmds/helpers.go

@@ -132,7 +132,7 @@ func createController(projectData project.Data, fs afero.Afero, currentDir strin
 
 }
 
-func getManifestAndController(halfpipeFilenameOptions []string) (manifest.Manifest, halfpipe.Controller) {
+func getManifestAndController(halfpipeFilenameOptions []string, renderer halfpipe.Renderer) (manifest.Manifest, halfpipe.Controller) {
 	if err := checkVersion(); err != nil {
 		printErr(err)
 		os.Exit(1)
@@ -157,11 +157,12 @@ func getManifestAndController(halfpipeFilenameOptions []string) (manifest.Manife
 		outputLintResults(linters.LintResults{linters.NewLintResult("Halfpipe Manifest", "https://ee.public.springernature.app/rel-eng/halfpipe/manifest/", manErrors)})
 	}
 
-	var renderer halfpipe.Renderer
-	if man.Platform.IsActions() {
-		renderer = actions.NewActions(projectData.GitURI, projectData.HalfpipeFilePath)
-	} else {
-		renderer = concourse.NewPipeline(projectData.HalfpipeFilePath)
+	if renderer == nil {
+		if man.Platform.IsActions() {
+			renderer = actions.NewActions(projectData.GitURI, projectData.HalfpipeFilePath)
+		} else {
+			renderer = concourse.NewPipeline(projectData.HalfpipeFilePath)
+		}
 	}
 	controller := createController(projectData, fs, currentDir, renderer)
 

cmd/cmds/internal-representation.go

@@ -16,7 +16,7 @@ var internalRepresentation = &cobra.Command{
 	Short: `Prints the internal representation of the manifest`,
 	Run: func(cmd *cobra.Command, args []string) {
 
-		man, controller := getManifestAndController(formatInput(Input))
+		man, controller := getManifestAndController(formatInput(Input), nil)
 
 		defaultedAndMappedManifest, _ := controller.DefaultAndMap(man)
 

cmd/cmds/pipeline_name.go

@@ -14,7 +14,7 @@ var pipelineNameCmd = &cobra.Command{
 	Use:   "pipeline-name",
 	Short: "Prints the name of the pipeline",
 	Run: func(cmd *cobra.Command, args []string) {
-		man, _ := getManifestAndController(formatInput(Input))
+		man, _ := getManifestAndController(formatInput(Input), nil)
 		if man.PipelineName() == "" {
 			os.Exit(1)
 		}

cmd/cmds/root.go

@@ -11,8 +11,12 @@ var rootCmd = &cobra.Command{
 	Short: `halfpipe is a tool to lint and render pipelines
 Invoke without any arguments to lint your .halfpipe.io file and render a pipeline`,
 	Run: func(cmd *cobra.Command, args []string) {
-		man, controller := getManifestAndController(formatInput(Input))
-		response := controller.Process(man)
+		man, controller := getManifestAndController(formatInput(Input), nil)
+		response, err := controller.Process(man)
+		if err != nil {
+			printErr(err)
+			os.Exit(1)
+		}
 
 		if man.Platform.IsActions() && output == "" {
 			output = path.Join(response.Project.GitRootPath, ".github/workflows/", man.PipelineName()+".yml")

config/config.go

@@ -25,6 +25,8 @@ var (
 
 	ConcourseURL = "https://concourse." + Domain
 
+	ActionsRunnerName = getEnv("HALFPIPE_ACTIONS_RUNNER", "ee-runner")
+
 	CacheDirs = []string{
 		"../../../var/halfpipe/cache",
 		"../../../halfpipe-cache", // deprecated and should be removed after a while

controller.go

@@ -9,7 +9,7 @@ import (
 )
 
 type Controller interface {
-	Process(man manifest.Manifest) Response
+	Process(man manifest.Manifest) (Response, error)
 	DefaultAndMap(man manifest.Manifest) (updated manifest.Manifest, err error)
 }
 
@@ -20,6 +20,10 @@ type Response struct {
 	Platform    manifest.Platform
 }
 
+func (r Response) String() string {
+	return r.ConfigYaml
+}
+
 type Renderer interface {
 	Render(manifest manifest.Manifest) (string, error)
 }
@@ -40,7 +44,7 @@ func NewController(defaulter defaults.Defaults, mapper mapper.Mapper, linters []
 	}
 }
 
-func (c controller) Process(man manifest.Manifest) (response Response) {
+func (c controller) Process(man manifest.Manifest) (response Response, err error) {
 	defaultedManifest := c.defaulter.Apply(man)
 
 	for _, linter := range c.linters {
@@ -57,11 +61,11 @@ func (c controller) Process(man manifest.Manifest) (response Response) {
 		return
 	}
 
-	config, _ := c.renderer.Render(mappedManifest)
+	config, err := c.renderer.Render(mappedManifest)
 	response.ConfigYaml = config
 	response.Project = c.defaulter.Project
 	response.Platform = man.Platform
-	return response
+	return
 }
 
 func (c controller) DefaultAndMap(man manifest.Manifest) (updated manifest.Manifest, err error) {

controller_test.go

@@ -41,14 +41,14 @@ func testController() controller {
 func TestWorksForHalfpipeFileWithYMLExtension(t *testing.T) {
 	c := testController()
 
-	response := c.Process(validHalfpipeManifest)
+	response, _ := c.Process(validHalfpipeManifest)
 
 	assert.Len(t, response.LintResults.Error(), 0)
 }
 
 func TestWorksForHalfpipeFile(t *testing.T) {
 	c := testController()
-	response := c.Process(validHalfpipeManifest)
+	response, _ := c.Process(validHalfpipeManifest)
 
 	assert.Len(t, response.LintResults.Error(), 0)
 }
@@ -68,7 +68,7 @@ func TestAppliesAllLinters(t *testing.T) {
 	linter2 := fakeLinter{errors.New("error from linter2")}
 	c.linters = []linters.Linter{linter1, linter2}
 
-	response := c.Process(validHalfpipeManifest)
+	response, _ := c.Process(validHalfpipeManifest)
 
 	assert.Empty(t, response.ConfigYaml)
 	assert.Len(t, response.LintResults, 2)
@@ -79,7 +79,7 @@ func TestAppliesAllLinters(t *testing.T) {
 func TestGivesBackConfigWhenLinterPasses(t *testing.T) {
 	c := testController()
 
-	response := c.Process(validHalfpipeManifest)
+	response, _ := c.Process(validHalfpipeManifest)
 	assert.Len(t, response.LintResults, 0)
 	assert.Equal(t, "fake output", response.ConfigYaml)
 }
@@ -95,7 +95,7 @@ func (f FakeMapper) Apply(original manifest.Manifest) (updated manifest.Manifest
 func TestGivesBackABadTestResultWhenAMapperFails(t *testing.T) {
 	c := testController()
 	c.mapper = FakeMapper{err: errors.New("blurgh")}
-	response := c.Process(validHalfpipeManifest)
+	response, _ := c.Process(validHalfpipeManifest)
 
 	assert.Len(t, response.LintResults, 1)
 	assert.True(t, response.LintResults.HasErrors())

defaults/task_docker_push.go

@@ -27,5 +27,20 @@ func dockerPushDefaulter(original manifest.DockerPush, man manifest.Manifest, de
 		updated.Platforms = []string{"linux/amd64"}
 	}
 
+	if updated.Secrets == nil {
+		updated.Secrets = make(manifest.Vars)
+	}
+
+	if man.Platform.IsConcourse() {
+		updated.Secrets["ARTIFACTORY_URL"] = defaults.Artifactory.URL
+		updated.Secrets["ARTIFACTORY_USERNAME"] = defaults.Artifactory.Username
+		updated.Secrets["ARTIFACTORY_PASSWORD"] = defaults.Artifactory.Password
+	}
+	if man.Platform.IsActions() {
+		updated.Secrets["ARTIFACTORY_URL"] = "${{ secrets.EE_ARTIFACTORY_URL }}"
+		updated.Secrets["ARTIFACTORY_USERNAME"] = "${{ secrets.EE_ARTIFACTORY_USERNAME }}"
+		updated.Secrets["ARTIFACTORY_PASSWORD"] = "${{ secrets.EE_ARTIFACTORY_PASSWORD }}"
+	}
+
 	return updated
 }

defaults/task_docker_push_test.go

@@ -8,27 +8,16 @@ import (
 	"testing"
 )
 
-func TestWhenPublicImage(t *testing.T) {
+func TestWhenPublicImageDontSetUsernameAndPassword(t *testing.T) {
 	task := manifest.DockerPush{Image: "asdf", DockerfilePath: "something", ScanTimeout: 15}
-
-	defaultedTask := manifest.DockerPush{Image: "asdf", DockerfilePath: "something", ScanTimeout: 15, Platforms: []string{"linux/amd64"}}
-
-	assert.Equal(t, defaultedTask, dockerPushDefaulter(task, manifest.Manifest{}, Concourse))
+	assert.Empty(t, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Username)
+	assert.Empty(t, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Password)
 }
 
-func TestPrivateImage(t *testing.T) {
+func TestPrivateImageSetsUsernameAndPassword(t *testing.T) {
 	task := manifest.DockerPush{Image: path.Join(config.DockerRegistry, "push-me"), DockerfilePath: "something"}
-
-	expected := manifest.DockerPush{
-		Image:          path.Join(config.DockerRegistry, "push-me"),
-		DockerfilePath: "something",
-		Username:       Concourse.Docker.Username,
-		Password:       Concourse.Docker.Password,
-		ScanTimeout:    15,
-		Platforms:      []string{"linux/amd64"},
-	}
-
-	assert.Equal(t, expected, dockerPushDefaulter(task, manifest.Manifest{}, Concourse))
+	assert.Equal(t, Concourse.Docker.Username, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Username)
+	assert.Equal(t, Concourse.Docker.Password, dockerPushDefaulter(task, manifest.Manifest{}, Concourse).Password)
 }
 
 func TestSetsTheDockerFilePath(t *testing.T) {

e2e/actions/consumer-integration-test/workflowExpected.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
     - name: Vault secrets
       id: secrets
-      uses: hashicorp/[email protected].3
+      uses: hashicorp/[email protected].4
       with:
         exportEnv: false
         method: approle
@@ -103,7 +103,7 @@ jobs:
     steps:
     - name: Vault secrets
       id: secrets
-      uses: hashicorp/[email protected].3
+      uses: hashicorp/[email protected].4
       with:
         exportEnv: false
         method: approle