Merge pull request #49 from semaphoreci/db/params-in-promotion-ppl-file

Evaluate template expressions everywhere except in job commands

go.mod

@@ -3,6 +3,7 @@ module github.com/semaphoreci/spc
 go 1.21
 
 require (
+	github.com/42atomys/sprout v0.2.0
 	github.com/Jeffail/gabs/v2 v2.7.0
 	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/ghodss/yaml v1.0.0
@@ -12,7 +13,6 @@ require (
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
-	github.com/42atomys/sprout v0.2.0 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/bitfield/gotestdox v0.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

pkg/pipelines/template_evaluator.go

@@ -42,184 +42,34 @@ func (e *templateEvaluator) Run() error {
 }
 
 func (e *templateEvaluator) ExtractAll() {
-	e.ExtractPipelineName()
-	e.ExtractFromQueue()
-	e.ExtractFromSecrets()
-	e.ExtractFromBlockNames()
-	e.ExtractFromJobNames()
-	e.ExtractFromAgents()
-	e.ExtractFromJobMatrices()
-	e.ExtractFromParallelisms()
+	e.ExtractTemplateExpression(e.pipeline.raw, []string{})
 }
 
-func (e *templateEvaluator) ExtractPipelineName() {
-	e.tryExtractingFromPath([]string{"name"})
-}
-
-func (e *templateEvaluator) ExtractFromQueue() {
-	e.tryExtractingFromPath([]string{"queue", "name"})
-
-	for index := range e.pipeline.QueueRules() {
-		e.tryExtractingFromPath([]string{"queue", strconv.Itoa(index), "name"})
-	}
-}
-
-func (e *templateEvaluator) ExtractFromSecrets() {
-	e.ExtractFromGlobalSecrets()
-	e.ExtractFromBlockSecrets()
-	e.ExtractFromAfterPipelineSecrets()
-}
-
-func (e *templateEvaluator) ExtractFromGlobalSecrets() {
-	e.extractFromSecretsAt(e.pipeline.GlobalJobConfig(), []string{"global_job_config"})
-}
-
-func (e *templateEvaluator) ExtractFromBlockSecrets() {
-	for blockIndex, block := range e.pipeline.Blocks() {
-		e.extractFromSecretsAt(block.Search("task"), []string{"blocks", strconv.Itoa(blockIndex), "task"})
-	}
-}
-
-func (e *templateEvaluator) ExtractFromAfterPipelineSecrets() {
-	e.extractFromSecretsAt(e.pipeline.AfterPipelineTask(), []string{"after_pipeline", "task"})
-}
-
-func (e *templateEvaluator) ExtractFromJobMatrices() {
-	e.ExtractFromBlockJobMatrices()
-	e.ExtractFromAfterPipelineJobMatrices()
-}
-
-func (e *templateEvaluator) ExtractFromBlockJobMatrices() {
-	for blockIndex, block := range e.pipeline.Blocks() {
-		blockTaskPath := []string{"blocks", strconv.Itoa(blockIndex), "task"}
+func (e *templateEvaluator) ExtractTemplateExpression(parent *gabs.Container, parentPath []string){
+	path := []string{}
 
-		for jobIndex, job := range block.Search("task", "jobs").Children() {
-			jobPath := concatPaths(blockTaskPath, []string{"jobs", strconv.Itoa(jobIndex)})
-			e.extractFromJobMatricesAt(job, jobPath)
-		}
-	}
-}
-
-func (e *templateEvaluator) ExtractFromAfterPipelineJobMatrices() {
-	afterPipelineTask := e.pipeline.AfterPipelineTask()
-
-	for jobIndex, job := range afterPipelineTask.Search("jobs").Children() {
-		jobPath := []string{"after_pipeline", "task", "jobs", strconv.Itoa(jobIndex)}
-		e.extractFromJobMatricesAt(job, jobPath)
-	}
-}
-
-func (e *templateEvaluator) ExtractFromParallelisms() {
-	e.ExtractFromBlockJobParallelisms()
-	e.ExtractFromAfterPipelineParallelisms()
-}
+	switch parent.Data().(type) {
 
-func (e *templateEvaluator) ExtractFromBlockJobParallelisms() {
-	for blockIndex, block := range e.pipeline.Blocks() {
-		blockTaskPath := []string{"blocks", strconv.Itoa(blockIndex), "task"}
-
-		for jobIndex := range block.Search("task", "jobs").Children() {
-			jobPath := concatPaths(blockTaskPath, []string{"jobs", strconv.Itoa(jobIndex)})
-			e.tryExtractingFromPath(jobPath, []string{"parallelism"})
-		}
-	}
-}
-
-func (e *templateEvaluator) ExtractFromAfterPipelineParallelisms() {
-	afterPipelineTask := e.pipeline.AfterPipelineTask()
-
-	for jobIndex := range afterPipelineTask.Search("jobs").Children() {
-		jobPath := []string{"after_pipeline", "task", "jobs", strconv.Itoa(jobIndex)}
-		e.tryExtractingFromPath(jobPath, []string{"parallelism"})
-	}
-}
+		case []interface{}:
+			for childIndex, child := range parent.Children() {
+				path = concatPaths(parentPath, []string{strconv.Itoa(childIndex)})
+				e.ExtractTemplateExpression(child, path)
+			}
 
-func (e *templateEvaluator) ExtractFromAgents() {
-	e.ExtractFromTopLevelAgent()
-	e.ExtractFromBlockAgents()
-	e.ExtractFromAfterPipelineAgents()
-}
-
-func (e *templateEvaluator) ExtractFromTopLevelAgent() {
-	e.extractFromAgentAt(e.pipeline.Agent(), []string{"agent"})
-}
-
-func (e *templateEvaluator) ExtractFromBlockAgents() {
-	for blockIndex, block := range e.pipeline.Blocks() {
-		agent := block.Search("task", "agent")
-		agentPath := []string{"blocks", strconv.Itoa(blockIndex), "task", "agent"}
-		e.extractFromAgentAt(agent, agentPath)
-	}
-}
-
-func (e *templateEvaluator) ExtractFromAfterPipelineAgents() {
-	afterPipelineTask := e.pipeline.AfterPipelineTask()
-	if afterPipelineTask == nil {
-		return
-	}
-
-	e.extractFromAgentAt(afterPipelineTask, []string{"after_pipeline", "task"})
-}
-
-func (e *templateEvaluator) ExtractFromBlockNames() {
-	for blockIndex := range e.pipeline.Blocks() {
-		e.tryExtractingFromPath([]string{"blocks", strconv.Itoa(blockIndex), "name"})
-	}
-}
-
-func (e *templateEvaluator) ExtractFromJobNames() {
-	e.ExtractFromBlockJobNames()
-	e.ExtractFromAfterPipelineJobNames()
-}
-
-func (e *templateEvaluator) ExtractFromBlockJobNames() {
-	for blockIndex, block := range e.pipeline.Blocks() {
-		blockTaskPath := []string{"blocks", strconv.Itoa(blockIndex), "task"}
-		for jobIndex := range block.Search("task", "jobs").Children() {
-			e.tryExtractingFromPath(blockTaskPath, []string{"jobs", strconv.Itoa(jobIndex), "name"})
-		}
-	}
-}
-
-func (e *templateEvaluator) ExtractFromAfterPipelineJobNames() {
-	afterPipelineTask := e.pipeline.AfterPipelineTask()
-
-	for jobIndex := range afterPipelineTask.Search("jobs").Children() {
-		e.tryExtractingFromPath([]string{"after_pipeline", "task", "jobs", strconv.Itoa(jobIndex), "name"})
-	}
-}
-
-func (e *templateEvaluator) extractFromAgentAt(agent *gabs.Container, agentPath []string) {
-	e.tryExtractingFromPath(agentPath, []string{"machine", "type"})
-	e.tryExtractingFromPath(agentPath, []string{"machine", "os_image"})
-
-	for containerIndex, container := range agent.Search("containers").Children() {
-		containerPath := concatPaths(agentPath, []string{"containers", strconv.Itoa(containerIndex)})
-		e.extractFromContainerAt(container, containerPath)
-	}
-}
-
-func (e *templateEvaluator) extractFromContainerAt(container *gabs.Container, containerPath []string) {
-	e.tryExtractingFromPath(containerPath, []string{"name"})
-	e.tryExtractingFromPath(containerPath, []string{"image"})
-	e.extractFromSecretsAt(container, containerPath)
-}
-
-func (e *templateEvaluator) extractFromSecretsAt(parent *gabs.Container, parentPath []string) {
-	for secretIndex := range parent.Search("secrets").Children() {
-		e.tryExtractingFromPath(parentPath, []string{"secrets", strconv.Itoa(secretIndex), "name"})
-	}
-}
+		case map[string]interface{}:
+			for key, child := range parent.ChildrenMap() {
+				if key != "commands" {
+					path = concatPaths(parentPath, []string{key})
+					e.ExtractTemplateExpression(child, path)
+				}
+			}
 
-func (e *templateEvaluator) extractFromJobMatricesAt(parent *gabs.Container, parentPath []string) {
-	for matrixIndex := range parent.Search("matrix").Children() {
-		e.tryExtractingFromPath(parentPath, []string{"matrix", strconv.Itoa(matrixIndex), "env_var"})
-		e.tryExtractingFromPath(parentPath, []string{"matrix", strconv.Itoa(matrixIndex), "values"})
+		default:
+			e.tryExtractingFromPath(parentPath)
 	}
 }
 
-func (e *templateEvaluator) tryExtractingFromPath(paths ...[]string) {
-	path := concatPaths(paths...)
+func (e *templateEvaluator) tryExtractingFromPath(path []string) {
 	if !e.pipeline.PathExists(path) {
 		return
 	}

pkg/pipelines/template_evaluator_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"os"
 	"testing"
+	"reflect"
 
 	templates "github.com/semaphoreci/spc/pkg/templates"
 	assert "github.com/stretchr/testify/assert"
@@ -164,14 +165,55 @@ func Test__TemplateEvaluatorExtractAll(t *testing.T) {
 			YamlPath:   yamlPath,
 			Value:      nil,
 		},
+		{
+			Expression: "Promotion to ${{parameters.DEPLOY_ENV}}",
+			Path:       []string{"promotions", "0", "name"},
+			YamlPath:   yamlPath,
+			Value:      nil,
+		},
+		{
+			Expression: "${{parameters.DEPLOY_ENV}}_deployment.yml",
+			Path:       []string{"promotions", "0", "pipeline_file"},
+			YamlPath:   yamlPath,
+			Value:      nil,
+		},
+		{
+			Expression: "${{parameters.DEPLOY_ENV}}_deployment_target",
+			Path:       []string{"promotions", "0", "deployment_target"},
+			YamlPath:   yamlPath,
+			Value:      nil,
+		},
+		{
+			Expression: "${{parameters.DEPLOY_ENV | upper}}_SERVER_ID",
+			Path:       []string{"promotions", "0", "parameters", "env_vars", "0", "name"},
+			YamlPath:   yamlPath,
+			Value:      nil,
+		},
+		{
+			Expression: "${{parameters.SERVER}}",
+			Path:       []string{"promotions", "0", "parameters", "env_vars", "0", "default_value"},
+			YamlPath:   yamlPath,
+			Value:      nil,
+		},
 	}
 
 	assert.Equal(t, len(expectedExpressions), len(e.list))
-	for i, e1 := range e.list {
-		assert.Equal(t, expectedExpressions[i].Expression, e1.Expression)
-		assert.Equal(t, expectedExpressions[i].Path, e1.Path)
-		assert.Equal(t, expectedExpressions[i].YamlPath, e1.YamlPath)
+	for _, e1 := range e.list {
+		expectedExpr := findExpression(e1, expectedExpressions)
+
+		assert.Equal(t, expectedExpr.Expression, e1.Expression)
+		assert.Equal(t, expectedExpr.Path, e1.Path)
+		assert.Equal(t, expectedExpr.YamlPath, e1.YamlPath)
+	}
+}
+
+func findExpression(expr templates.Expression, expectedList []templates.Expression) templates.Expression {
+	for _, e := range expectedList {
+		if reflect.DeepEqual(e.Path, expr.Path) {
+			return e
+		}
 	}
+	return templates.Expression{}
 }
 
 func Test__Run(t *testing.T) {
@@ -217,6 +259,16 @@ func Test__Run(t *testing.T) {
 	assertValueOnPath(t, e, []string{"after_pipeline", "task", "jobs", "0", "matrix", "0", "values"}, []interface{}{"#engineering", "#general"})
 	assertValueOnPath(t, e, []string{"after_pipeline", "task", "jobs", "1", "name"}, "Ping prod from 2 jobs")
 	assertValueOnPath(t, e, []string{"after_pipeline", "task", "jobs", "1", "parallelism"}, json.Number("2"))
+	assertValueOnPath(t, e, []string{"promotions", "0", "name"}, "Promotion to prod")
+	assertValueOnPath(t, e, []string{"promotions", "0", "pipeline_file"}, "prod_deployment.yml")
+	assertValueOnPath(t, e, []string{"promotions", "0", "deployment_target"}, "prod_deployment_target")
+	assertValueOnPath(t, e, []string{"promotions", "0", "parameters", "env_vars", "0", "name"}, "PROD_SERVER_ID")
+	assertValueOnPath(t, e, []string{"promotions", "0", "parameters", "env_vars", "0", "default_value"}, "server_1")
+
+	// template expressions are not evaluated in block and after_pipeline job's commands
+	expectedString := "echo \"Template expressions are not evaluated here ${{parameters.SERVER}}\""
+	assertValueOnPath(t, e, []string{"blocks", "0", "task", "jobs", "0", "commands", "1"}, expectedString)
+	assertValueOnPath(t, e, []string{"after_pipeline", "task", "jobs", "1", "commands", "1"}, expectedString)
 
 }
 

test/e2e/parameters_and_change_in.rb

@@ -1,7 +1,7 @@
 # rubocop:disable all
 
 #
-# This test verifies that copiler can process all possible locations for both the
+# This test verifies that compiler can process all possible locations for both the
 # parameters and changs in the same yaml file.
 #
 
@@ -53,6 +53,7 @@
         - name: Test
           commands:
             - make test
+            - echo "Template evaluation should not work here ${{parameters.DEPLOY_ENV}}"
           matrix:
             - env_var: "INTEGRATION_TEST"
               values: "%{{ \\"true,false\\" | splitList \\",\\" }}"
@@ -87,8 +88,15 @@
 
 promotions:
   - name: Performance tests
+    pipeline_file: perf_test.yml
     auto_promote:
       when: "branch = 'master' and change_in('/lib')"
+  - name: Smoke tests on ${{parameters.DEPLOY_ENV}} env
+    pipeline_file: ${{parameters.DEPLOY_ENV}}_smoke_test.yml
+    parameters:
+      env_vars:
+        - name: ${{parameters.DEPLOY_ENV | upper}}_SERVICE_ID
+          default_value: ${{parameters.DEPLOY_ENV}}_${{parameters.SERVICE}}
 }
 
 origin = TestRepoForChangeIn.setup()
@@ -163,6 +171,7 @@
         - name: Test
           commands:
             - make test
+            - echo "Template evaluation should not work here ${{parameters.DEPLOY_ENV}}"
           matrix: 
             - env_var: INTEGRATION_TEST
               values: ["true", "false"]
@@ -198,6 +207,13 @@
 
 promotions:
   - name: Performance tests
+    pipeline_file: perf_test.yml
     auto_promote:
       when: "(branch = 'master') and true"
+  - name: Smoke tests on prod env
+    pipeline_file: prod_smoke_test.yml
+    parameters:
+      env_vars:
+        - name: PROD_SERVICE_ID
+          default_value: prod_web_server
 }))

test/fixtures/all_template_locations.yml

@@ -32,6 +32,7 @@ blocks:
         - name: Run tests
           commands:
             - echo "Running tests"
+            - echo "Template expressions are not evaluated here ${{parameters.SERVER}}"
           parallelism: "%{{parameters.PARALLELISM | mul 2}}"
 
   - name: Build and push image
@@ -66,4 +67,14 @@ after_pipeline:
       - name: Ping ${{parameters.DEPLOY_ENV}} from %{{parameters.PARALLELISM}} jobs
         commands:
           - echo "Pinging environment"
+          - echo "Template expressions are not evaluated here ${{parameters.SERVER}}"
         parallelism: "%{{parameters.PARALLELISM | int64 }}"
+
+promotions:
+  - name: Promotion to ${{parameters.DEPLOY_ENV}}
+    pipeline_file: ${{parameters.DEPLOY_ENV}}_deployment.yml
+    deployment_target: ${{parameters.DEPLOY_ENV}}_deployment_target
+    parameters:
+      env_vars:
+        - name: "${{parameters.DEPLOY_ENV | upper}}_SERVER_ID"
+          default_value: ${{parameters.SERVER}}