From 5fdea37998aafd04eb5b85d595b7c2260821a190 Mon Sep 17 00:00:00 2001 From: lpusok <7979773+lpusok@users.noreply.github.com> Date: Tue, 12 May 2020 11:00:41 +0200 Subject: [PATCH] Use exported envman function for step input expansion. (#721) * Use exported envman function for step input expansion. * Adding comment for exported function * dep update * dep update * Dep update * dep update vendors * dep update * Fill missing defaults for expanded vars * Moving step input template expansion to a seperate function, updating error messages. * Adding explanation and test case for not checking if expanded input is in the result environment map. * Check if input key is present in evironment map * normalize error messages * Use common env var expansion during step input template parsing. * removing unused field * Normalize inputs: --- Gopkg.lock | 6 +- Gopkg.toml | 2 +- cli/analytics.go | 45 ++++ cli/analytics_test.go | 55 ++++ cli/run_util.go | 184 +++++--------- cli/run_util_test.go | 239 ------------------ cli/step_environment.go | 80 ++++++ cli/step_environment_test.go | 88 +++++++ tools/tools.go | 5 +- .../github.com/bitrise-io/envman/bitrise.yml | 4 + .../github.com/bitrise-io/envman/cli/run.go | 42 +-- .../bitrise-io/envman/env/expand.go | 195 ++++++++++++++ .../bitrise-io/envman/env/sharedtestcases.go | 200 +++++++++++++++ 13 files changed, 744 insertions(+), 401 deletions(-) create mode 100644 cli/analytics.go create mode 100644 cli/analytics_test.go create mode 100644 cli/step_environment.go create mode 100644 cli/step_environment_test.go create mode 100644 vendor/github.com/bitrise-io/envman/env/expand.go create mode 100644 vendor/github.com/bitrise-io/envman/env/sharedtestcases.go diff --git a/Gopkg.lock b/Gopkg.lock index 4c559a961..af6e4f067 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -19,18 +19,19 @@ [[projects]] branch = "master" - digest = "1:45f93a7af77540da03239e89a5159b435efa4fcc1fbd49b90c30c88bf95e9513" + digest = "1:bef3c009b291d0bb9addd9de6a6df3c93bbdf8abc2b970354d0ce9c747c8c5d0" name = "github.com/bitrise-io/envman" packages = [ ".", "cli", + "env", "envman", "models", "output", "version", ] pruneopts = "UT" - revision = "24a8f72875981710a8b21e60d81c4126a3af3c67" + revision = "335e05a8a8e3c3cdd22ab857f83a2bdad25d1e00" [[projects]] branch = "master" @@ -178,6 +179,7 @@ input-imports = [ "github.com/Sirupsen/logrus", "github.com/bitrise-io/envman", + "github.com/bitrise-io/envman/env", "github.com/bitrise-io/envman/models", "github.com/bitrise-io/go-utils/colorstring", "github.com/bitrise-io/go-utils/command", diff --git a/Gopkg.toml b/Gopkg.toml index 158103bee..87774c033 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -4,8 +4,8 @@ required = ["github.com/bitrise-io/go-utils/envutil", "github.com/bitrise-io/stepman"] [[constraint]] - name = "github.com/sirupsen/logrus" version = "1.4.2" + name = "github.com/Sirupsen/logrus" [[constraint]] branch = "master" diff --git a/cli/analytics.go b/cli/analytics.go new file mode 100644 index 000000000..09c14d232 --- /dev/null +++ b/cli/analytics.go @@ -0,0 +1,45 @@ +package cli + +import ( + "bytes" + "fmt" + "io" + + "github.com/bitrise-io/bitrise/tools/filterwriter" + "github.com/bitrise-io/envman/models" +) + +func redactStepInputs(environment map[string]string, inputs []models.EnvironmentItemModel, secrets []string) (map[string]string, error) { + redactedStepInputs := make(map[string]string) + + // Filter inputs from enviroments + for _, input := range inputs { + inputKey, _, err := input.GetKeyValuePair() + if err != nil { + return map[string]string{}, fmt.Errorf("failed to get input key: %s", err) + } + + // If input key may not be present in the result environment. + // This can happen if the input has the "skip_if_empty" property set to true, and it is empty. + inputValue, ok := environment[inputKey] + if !ok { + redactedStepInputs[inputKey] = "" + continue + } + + src := bytes.NewReader([]byte(inputValue)) + dstBuf := new(bytes.Buffer) + secretFilterDst := filterwriter.New(secrets, dstBuf) + + if _, err := io.Copy(secretFilterDst, src); err != nil { + return map[string]string{}, fmt.Errorf("failed to redact secrets, stream copy failed: %s", err) + } + if _, err := secretFilterDst.Flush(); err != nil { + return map[string]string{}, fmt.Errorf("failed to redact secrets, stream flush failed: %s", err) + } + + redactedStepInputs[inputKey] = dstBuf.String() + } + + return redactedStepInputs, nil +} diff --git a/cli/analytics_test.go b/cli/analytics_test.go new file mode 100644 index 000000000..33bf68247 --- /dev/null +++ b/cli/analytics_test.go @@ -0,0 +1,55 @@ +package cli + +import ( + "testing" + + "github.com/bitrise-io/envman/models" + envmanModels "github.com/bitrise-io/envman/models" + "github.com/stretchr/testify/require" +) + +func Test_expandStepInputsForAnalytics(t *testing.T) { + type args struct { + environments map[string]string + inputs []envmanModels.EnvironmentItemModel + secretValues []string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "Secret filtering", + args: args{ + environments: map[string]string{"secret_simulator_device": "secret_a_secret_b_secret_c"}, + inputs: []models.EnvironmentItemModel{ + {"secret_simulator_device": "secret_a_secret_b_secret_c"}, + }, + secretValues: []string{"secret_a_secret_b_secret_c"}, + }, + want: map[string]string{ + "secret_simulator_device": "[REDACTED]", + }, + }, + { + name: "Input is empty, and skip_if_empty is true", + args: args{ + environments: map[string]string{}, + inputs: []models.EnvironmentItemModel{ + {"myinput": ""}, + }, + }, + want: map[string]string{ + "myinput": "", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := redactStepInputs(tt.args.environments, tt.args.inputs, tt.args.secretValues) + require.NoError(t, err, "expandStepInputsForAnalytics") + require.Equal(t, tt.want, got) + }) + } +} diff --git a/cli/run_util.go b/cli/run_util.go index abb8e2228..e0c7630e4 100644 --- a/cli/run_util.go +++ b/cli/run_util.go @@ -20,6 +20,7 @@ import ( "github.com/bitrise-io/bitrise/plugins" "github.com/bitrise-io/bitrise/toolkits" "github.com/bitrise-io/bitrise/tools" + "github.com/bitrise-io/envman/env" envmanModels "github.com/bitrise-io/envman/models" "github.com/bitrise-io/go-utils/colorstring" "github.com/bitrise-io/go-utils/command" @@ -424,53 +425,10 @@ func runStep( return checkAndInstallStepDependencies(step) }); err != nil { - return 1, []envmanModels.EnvironmentItemModel{}, fmt.Errorf("Failed to install Step dependency, error: %s", err) + return 1, []envmanModels.EnvironmentItemModel{}, + fmt.Errorf("Failed to install Step dependency, error: %s", err) } - // Collect step inputs - if err := tools.EnvmanInitAtPath(configs.InputEnvstorePath); err != nil { - return 1, []envmanModels.EnvironmentItemModel{}, fmt.Errorf("Failed to init envman for the Step, error: %s", err) - } - - if err := tools.ExportEnvironmentsList(configs.InputEnvstorePath, environments); err != nil { - return 1, []envmanModels.EnvironmentItemModel{}, fmt.Errorf("Failed to export environment list for the Step, error: %s", err) - } - - evaluatedInputs := []envmanModels.EnvironmentItemModel{} - for _, input := range step.Inputs { - key, value, err := input.GetKeyValuePair() - if err != nil { - return 1, []envmanModels.EnvironmentItemModel{}, err - } - - options, err := input.GetOptions() - if err != nil { - return 1, []envmanModels.EnvironmentItemModel{}, err - } - - if options.IsTemplate != nil && *options.IsTemplate { - outStr, err := tools.EnvmanJSONPrint(configs.InputEnvstorePath) - if err != nil { - return 1, []envmanModels.EnvironmentItemModel{}, fmt.Errorf("EnvmanJSONPrint failed, err: %s", err) - } - - envList, err := envmanModels.NewEnvJSONList(outStr) - if err != nil { - return 1, []envmanModels.EnvironmentItemModel{}, fmt.Errorf("CreateFromJSON failed, err: %s", err) - } - - evaluatedValue, err := bitrise.EvaluateTemplateToString(value, configs.IsCIMode, configs.IsPullRequestMode, buildRunResults, envList) - if err != nil { - return 1, []envmanModels.EnvironmentItemModel{}, err - } - - input[key] = evaluatedValue - } - - evaluatedInputs = append(evaluatedInputs, input) - } - environments = append(environments, evaluatedInputs...) - if err := tools.EnvmanInitAtPath(configs.InputEnvstorePath); err != nil { return 1, []envmanModels.EnvironmentItemModel{}, err } @@ -580,53 +538,6 @@ func activateStepLibStep(stepIDData models.StepIDData, destination, stepYMLCopyP return info, didStepLibUpdate, nil } -func expandStepInputs( - inputs []envmanModels.EnvironmentItemModel, - environments []envmanModels.EnvironmentItemModel, -) map[string]string { - stepInputs := make(map[string]string) - - var mappingFuncFactory func([]envmanModels.EnvironmentItemModel) func(string) string - mappingFuncFactory = func(environments []envmanModels.EnvironmentItemModel) func(key string) string { - return func(key string) string { - for inputName, inputValue := range stepInputs { - if inputName == key { - return os.Expand(inputValue, mappingFuncFactory(environments)) - } - } - - for index, environmentItem := range environments { - if envKey, envValue, err := environmentItem.GetKeyValuePair(); err == nil && envKey == key { - return os.Expand(envValue, mappingFuncFactory(environments[:index])) - } - } - - return os.Getenv(key) - } - } - - // Retrieve all non-sensitive input values - for _, input := range inputs { - if err := input.FillMissingDefaults(); err != nil { - log.Warnf("Failed to fill missing defaults, skipping input: %s", err) - continue - } - - options, err := input.GetOptions() - if err == nil && *options.IsSensitive == false { - if inputName, inputValue, err := input.GetKeyValuePair(); err == nil { - stepInputs[inputName] = os.Expand(inputValue, mappingFuncFactory(environments)) - } else { - log.Warnf("Failed to get input value for '%s', skipping input: %s", inputName, err) - } - } else if err != nil { - log.Warnf("Failed to get input options, skipping input: %s", err) - } - } - - return stepInputs -} - func activateAndRunSteps( workflow models.WorkflowModel, defaultStepLibSource string, @@ -642,7 +553,8 @@ func activateAndRunSteps( // ------------------------------------------ // In function method - Registration methods, for register step run results. registerStepRunResults := func(step stepmanModels.StepModel, stepInfoPtr stepmanModels.StepInfoModel, - stepIdxPtr int, runIf string, resultCode, exitCode int, err error, isLastStep, printStepHeader bool) { + stepIdxPtr int, runIf string, resultCode, exitCode int, err error, isLastStep, printStepHeader bool, + redactedStepInputs map[string]string) { if printStepHeader { bitrise.PrintRunningStepHeader(stepInfoPtr, step, stepIdxPtr) @@ -666,7 +578,7 @@ func activateAndRunSteps( stepResults := models.StepRunResultsModel{ StepInfo: stepInfoCopy, - StepInputs: expandStepInputs(step.Inputs, *environments), + StepInputs: redactedStepInputs, Status: resultCode, Idx: buildRunResults.ResultsCount(), RunTime: time.Now().Sub(stepStartTime), @@ -737,7 +649,7 @@ func activateAndRunSteps( if err := bitrise.CleanupStepWorkDir(); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } @@ -745,13 +657,13 @@ func activateAndRunSteps( // Preparing the step if err := tools.EnvmanInitAtPath(configs.InputEnvstorePath); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } if err := tools.ExportEnvironmentsList(configs.InputEnvstorePath, *environments); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } @@ -759,7 +671,7 @@ func activateAndRunSteps( compositeStepIDStr, workflowStep, err := models.GetStepIDStepDataPair(stepListItm) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } stepInfoPtr.ID = compositeStepIDStr @@ -772,7 +684,7 @@ func activateAndRunSteps( stepIDData, err := models.CreateStepIDDataFromString(compositeStepIDStr, defaultStepLibSource) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } stepInfoPtr.ID = stepIDData.IDorURI @@ -793,7 +705,7 @@ func activateAndRunSteps( stepAbsLocalPth, err := pathutil.AbsPath(stepIDData.IDorURI) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } @@ -802,13 +714,13 @@ func activateAndRunSteps( origStepYMLPth = filepath.Join(stepAbsLocalPth, "step.yml") if err := command.CopyFile(origStepYMLPth, stepYMLPth); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } if err := command.CopyDir(stepAbsLocalPth, stepDir, true); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } } else if stepIDData.SteplibSource == "git" { @@ -816,7 +728,7 @@ func activateAndRunSteps( repo, err := git.New(stepDir) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) } if err := repo.CloneTagOrBranch(stepIDData.IDorURI, stepIDData.Version).Run(); err != nil { if strings.HasPrefix(stepIDData.IDorURI, "git@") { @@ -826,13 +738,13 @@ func activateAndRunSteps( fmt.Println(colorstring.Yellow(`even if the repository is open source!`)) } registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } if err := command.CopyFile(filepath.Join(stepDir, "step.yml"), stepYMLPth); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } } else if stepIDData.SteplibSource == "_" { @@ -842,18 +754,18 @@ func activateAndRunSteps( stepYMLPth = "" if err := workflowStep.FillMissingDefaults(); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } repo, err := git.New(stepDir) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) } if err := repo.CloneTagOrBranch(stepIDData.IDorURI, stepIDData.Version).Run(); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } } else if stepIDData.SteplibSource != "" { @@ -874,12 +786,13 @@ func activateAndRunSteps( if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } } else { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, fmt.Errorf("Invalid stepIDData: No SteplibSource or LocalPath defined (%v)", stepIDData), isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, fmt.Errorf("Invalid stepIDData: No SteplibSource or LocalPath defined (%v)", stepIDData), + isLastStep, true, map[string]string{}) continue } @@ -896,14 +809,15 @@ func activateAndRunSteps( ymlPth = origStepYMLPth } registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, fmt.Errorf("failed to parse step definition (%s): %s", ymlPth, err), isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, fmt.Errorf("failed to parse step definition (%s): %s", ymlPth, err), + isLastStep, true, map[string]string{}) continue } mergedStep, err = models.MergeStepWith(specStep, workflowStep) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) + "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true, map[string]string{}) continue } } @@ -922,26 +836,28 @@ func activateAndRunSteps( outStr, err := tools.EnvmanJSONPrint(configs.InputEnvstorePath) if err != nil { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, - *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, fmt.Errorf("EnvmanJSONPrint failed, err: %s", err), isLastStep, false) + *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, fmt.Errorf("EnvmanJSONPrint failed, err: %s", err), + isLastStep, false, map[string]string{}) continue } envList, err := envmanModels.NewEnvJSONList(outStr) if err != nil { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, - *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, fmt.Errorf("CreateFromJSON failed, err: %s", err), isLastStep, false) + *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, fmt.Errorf("CreateFromJSON failed, err: %s", err), + isLastStep, false, map[string]string{}) continue } isRun, err := bitrise.EvaluateTemplateToBool(*mergedStep.RunIf, configs.IsCIMode, configs.IsPullRequestMode, buildRunResults, envList) if err != nil { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, - *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, err, isLastStep, false) + *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, err, isLastStep, false, map[string]string{}) continue } if !isRun { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, - *mergedStep.RunIf, models.StepRunStatusCodeSkippedWithRunIf, 0, err, isLastStep, false) + *mergedStep.RunIf, models.StepRunStatusCodeSkippedWithRunIf, 0, err, isLastStep, false, map[string]string{}) continue } } @@ -955,7 +871,7 @@ func activateAndRunSteps( if buildRunResults.IsBuildFailed() && !isAlwaysRun { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, - *mergedStep.RunIf, models.StepRunStatusCodeSkipped, 0, err, isLastStep, false) + *mergedStep.RunIf, models.StepRunStatusCodeSkipped, 0, err, isLastStep, false, map[string]string{}) } else { // beside of the envs coming from the current parent process these will be added as an extra var additionalEnvironments []envmanModels.EnvironmentItemModel @@ -978,11 +894,29 @@ func activateAndRunSteps( }) } - exit, outEnvironments, err := runStep( - mergedStep, stepIDData, stepDir, - append(*environments, additionalEnvironments...), secrets, - buildRunResults, - ) + stepDeclaredEnvironments, expandedStepEnvironment, err := prepareStepEnvironment(prepareStepInputParams{ + environment: append(*environments, additionalEnvironments...), + inputs: mergedStep.Inputs, + buildRunResults: buildRunResults, + isCIMode: configs.IsCIMode, + isPullRequestMode: configs.IsPullRequestMode, + }, &env.DefaultEnvironmentSource{}) + if err != nil { + registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, + *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, + fmt.Errorf("failed to prepare step environment variables: %s", err), + isLastStep, false, map[string]string{}) + } + + redactedStepInputs, err := redactStepInputs(expandedStepEnvironment, mergedStep.Inputs, tools.GetSecretValues(secrets)) + if err != nil { + registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, + *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, + fmt.Errorf("failed to redact step inputs: %s", err), + isLastStep, false, map[string]string{}) + } + + exit, outEnvironments, err := runStep(mergedStep, stepIDData, stepDir, stepDeclaredEnvironments, secrets, buildRunResults) if testDirPath != "" { if err := addTestMetadata(testDirPath, models.TestResultStepInfo{Number: idx, Title: *mergedStep.Title, ID: stepIDData.IDorURI, Version: stepIDData.Version}); err != nil { @@ -998,14 +932,14 @@ func activateAndRunSteps( if err != nil { if *mergedStep.IsSkippable { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, - *mergedStep.RunIf, models.StepRunStatusCodeFailedSkippable, exit, err, isLastStep, false) + *mergedStep.RunIf, models.StepRunStatusCodeFailedSkippable, exit, err, isLastStep, false, redactedStepInputs) } else { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, - *mergedStep.RunIf, models.StepRunStatusCodeFailed, exit, err, isLastStep, false) + *mergedStep.RunIf, models.StepRunStatusCodeFailed, exit, err, isLastStep, false, redactedStepInputs) } } else { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, - *mergedStep.RunIf, models.StepRunStatusCodeSuccess, 0, nil, isLastStep, false) + *mergedStep.RunIf, models.StepRunStatusCodeSuccess, 0, nil, isLastStep, false, redactedStepInputs) } } } diff --git a/cli/run_util_test.go b/cli/run_util_test.go index eda761e11..3801dc682 100644 --- a/cli/run_util_test.go +++ b/cli/run_util_test.go @@ -722,242 +722,3 @@ func Test_activateStepLibStep(t *testing.T) { }) } } - -func TestExpandStepInputsMissingOptionsDoNotCauseCrash(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"simulator_os_version": "13.3", "opts": map[string]interface{}{}}, - envmanModels.EnvironmentItemModel{"simulator_device": "iPhone 8 Plus", "opts": map[string]interface{}{}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{} - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Equal(t, "13.3", expandedInputs["simulator_os_version"]) - require.Equal(t, "iPhone 8 Plus", expandedInputs["simulator_device"]) -} - -func TestExpandStepInputsDoNotNeedExpansion(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"simulator_os_version": "13.3", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"simulator_device": "iPhone 8 Plus", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{} - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Equal(t, "13.3", expandedInputs["simulator_os_version"]) - require.Equal(t, "iPhone 8 Plus", expandedInputs["simulator_device"]) -} - -func TestExpandStepInputsSecretRemoved(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"simulator_os_version": "13.3", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"simulator_device": "iPhone 8 Plus", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"secret_input": "top secret", "opts": map[string]interface{}{"is_sensitive": true}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{} - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Empty(t, expandedInputs["secret_input"]) - require.Equal(t, "13.3", expandedInputs["simulator_os_version"]) -} - -func TestExpandStepInputsNeedExpansion(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"simulator_os_version": "$SIMULATOR_OS_VERSION", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"simulator_device": "iPhone 8 Plus", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"SIMULATOR_OS_VERSION": "13.3", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Equal(t, "13.3", expandedInputs["simulator_os_version"]) -} - -func TestExpandStepInputsNeedsFloatExpansion(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"simulator_os_version": "$SIMULATOR_OS_VERSION", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"simulator_device": "iPhone 8 Plus", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"SIMULATOR_OS_VERSION": 13.3, "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"SIMULATOR_DEVICE_BOOL": true, "opts": map[string]interface{}{"is_sensitive": false}}, - } - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Equal(t, "iPhone 8 Plus", expandedInputs["simulator_device"]) - require.Equal(t, "13.3", expandedInputs["simulator_os_version"]) -} - -func TestExpandStepInputsNeedsBoolExpansion(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"random_bool": "BOOL: $RANDOM_BOOL", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"simulator_device": "iPhone 8 $SIMULATOR_DEVICE_BOOL", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"RANDOM_BOOL": true, "opts": map[string]interface{}{"is_sensitive": false}}, - } - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Equal(t, "BOOL: true", expandedInputs["random_bool"]) -} - -func TestExpandStepInputsNeedExpansionWithinExpansion(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"simulator_os_version": "$SIMULATOR_OS_VERSION", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"simulator_device": "iPhone 8 Plus", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"SIMULATOR_OS_MAJOR_VERSION": "13", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"SIMULATOR_OS_MINOR_VERSION": "3", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"SIMULATOR_OS_VERSION": "$SIMULATOR_OS_MAJOR_VERSION.$SIMULATOR_OS_MINOR_VERSION", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Empty(t, expandedInputs["secret_input"]) - require.Equal(t, "13.3", expandedInputs["simulator_os_version"]) -} - -func TestExpandStepInputsNeedCrossInputExpansion(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"simulator_os_version": "13.3", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"simulator_device": "iPhone 8 ($simulator_os_version)", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"simulator_os_version": "12.1", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Empty(t, expandedInputs["secret_input"]) - require.Equal(t, "iPhone 8 (13.3)", expandedInputs["simulator_device"]) -} - -func TestExpandSimpleLoopSkipped(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"loop": "$loop", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"env_loop": "$ENV_LOOP", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"ENV_LOOP": "$ENV_LOOP", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Equal(t, "", expandedInputs["loop"]) - require.Equal(t, "", expandedInputs["env_loop"]) -} - -func TestExpandLoopWithLengthOneSkipped(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"loop": "Something: $loop", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"env_loop": "$ENV_LOOP", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"ENV_LOOP": "Env Something: $ENV_LOOP", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Equal(t, "Something: ", expandedInputs["loop"]) - require.Equal(t, "Env Something: ", expandedInputs["env_loop"]) -} - -func TestExpandPrefixNotSkipped(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"similar2": "anything", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"similar": "$similar2", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"env": "Something: $similar", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{} - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Equal(t, "Something: anything", expandedInputs["env"]) -} - -func TestExpandMultiLengthLoopsSkipped(t *testing.T) { - // Arrange - testInputs := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"a": "$b", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"b": "$c", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"c": "$a", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"env": "$A", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - testEnvironment := []envmanModels.EnvironmentItemModel{ - envmanModels.EnvironmentItemModel{"B": "$A", "opts": map[string]interface{}{"is_sensitive": false}}, - envmanModels.EnvironmentItemModel{"A": "$B", "opts": map[string]interface{}{"is_sensitive": false}}, - } - - // Act - expandedInputs := expandStepInputs(testInputs, testEnvironment) - - // Assert - require.NotNil(t, expandedInputs) - require.Equal(t, "", expandedInputs["a"]) - require.Equal(t, "", expandedInputs["b"]) - require.Equal(t, "", expandedInputs["c"]) - require.Equal(t, "", expandedInputs["env"]) -} diff --git a/cli/step_environment.go b/cli/step_environment.go new file mode 100644 index 000000000..f483c7bbe --- /dev/null +++ b/cli/step_environment.go @@ -0,0 +1,80 @@ +package cli + +import ( + "fmt" + + "github.com/bitrise-io/bitrise/bitrise" + "github.com/bitrise-io/bitrise/models" + "github.com/bitrise-io/envman/env" + envmanModels "github.com/bitrise-io/envman/models" +) + +type prepareStepInputParams struct { + environment []envmanModels.EnvironmentItemModel + inputs []envmanModels.EnvironmentItemModel + buildRunResults models.BuildRunResultsModel + isCIMode, isPullRequestMode bool +} + +func prepareStepEnvironment(params prepareStepInputParams, envSource env.EnvironmentSource) ([]envmanModels.EnvironmentItemModel, map[string]string, error) { + for _, envVar := range params.environment { + if err := envVar.Normalize(); err != nil { + return []envmanModels.EnvironmentItemModel{}, map[string]string{}, + fmt.Errorf("failed to normalize declared environment variable: %s", err) + } + + if err := envVar.FillMissingDefaults(); err != nil { + return []envmanModels.EnvironmentItemModel{}, map[string]string{}, + fmt.Errorf("failed to fill missing declared environment variable defaults: %s", err) + } + } + + // Expand templates + evaluatedInputs := []envmanModels.EnvironmentItemModel{} + for _, input := range params.inputs { + if err := input.FillMissingDefaults(); err != nil { + return []envmanModels.EnvironmentItemModel{}, map[string]string{}, + fmt.Errorf("failed to fill input missing default properties: %s", err) + } + + key, value, err := input.GetKeyValuePair() + if err != nil { + return []envmanModels.EnvironmentItemModel{}, map[string]string{}, + fmt.Errorf("failed to get input key: %s", err) + } + + options, err := input.GetOptions() + if err != nil { + return []envmanModels.EnvironmentItemModel{}, map[string]string{}, + fmt.Errorf("failed to get options: %s", err) + } + + if options.IsTemplate != nil && *options.IsTemplate { + envs, err := env.GetDeclarationsSideEffects(params.environment, &env.DefaultEnvironmentSource{}) + if err != nil { + return []envmanModels.EnvironmentItemModel{}, map[string]string{}, + fmt.Errorf("GetDeclarationsSideEffects() failed, %s", err) + } + + evaluatedValue, err := bitrise.EvaluateTemplateToString(value, params.isCIMode, params.isPullRequestMode, params.buildRunResults, envs.ResultEnvironment) + if err != nil { + return []envmanModels.EnvironmentItemModel{}, map[string]string{}, + fmt.Errorf("failed to evaluate template: %s", err) + } + + input[key] = evaluatedValue + } + + evaluatedInputs = append(evaluatedInputs, input) + } + + stepEnvironment := append(params.environment, evaluatedInputs...) + + declarationSideEffects, err := env.GetDeclarationsSideEffects(stepEnvironment, envSource) + if err != nil { + return []envmanModels.EnvironmentItemModel{}, map[string]string{}, + fmt.Errorf("failed to get environment variable declaration results: %s", err) + } + + return stepEnvironment, declarationSideEffects.ResultEnvironment, nil +} diff --git a/cli/step_environment_test.go b/cli/step_environment_test.go new file mode 100644 index 000000000..965245b98 --- /dev/null +++ b/cli/step_environment_test.go @@ -0,0 +1,88 @@ +package cli + +import ( + "testing" + + "github.com/bitrise-io/envman/models" + envmanModels "github.com/bitrise-io/envman/models" + "github.com/stretchr/testify/require" +) + +type EmptyEnvironment struct{} + +func (*EmptyEnvironment) GetEnvironment() map[string]string { + return map[string]string{} +} + +func newBool(v bool) *bool { + b := v + return &b +} + +func Test_prepareStepEnvironment(t *testing.T) { + tests := []struct { + name string + params prepareStepInputParams + want1 []envmanModels.EnvironmentItemModel + want2 map[string]string + wantErr bool + }{ + { + name: "Template expansion works", + params: prepareStepInputParams{ + environment: []envmanModels.EnvironmentItemModel{}, + inputs: []envmanModels.EnvironmentItemModel{ + {"D": "{{.IsCI}}", "opts": models.EnvironmentItemOptionsModel{IsTemplate: newBool(true)}}, + }, + isCIMode: true, + }, + want1: []envmanModels.EnvironmentItemModel{ + {"D": "true", "opts": models.EnvironmentItemOptionsModel{IsTemplate: newBool(true)}}, + }, + want2: map[string]string{ + "D": "true", + }, + }, + { + name: "Default expansion flag is applied", + params: prepareStepInputParams{ + environment: []envmanModels.EnvironmentItemModel{ + {"A": "B", "opts": map[string]interface{}{}}, + }, + inputs: []envmanModels.EnvironmentItemModel{ + {"myinput": "$A", "opts": models.EnvironmentItemOptionsModel{IsExpand: nil}}, + }, + }, + want1: []envmanModels.EnvironmentItemModel{ + {"A": "B", "opts": models.EnvironmentItemOptionsModel{IsExpand: newBool(true)}}, + {"myinput": "$A", "opts": models.EnvironmentItemOptionsModel{IsExpand: newBool(true)}}, + }, + want2: map[string]string{ + "A": "B", + "myinput": "B", + }, + }, + } + + for _, tt := range tests { + for _, envVar := range tt.want1 { + if err := envVar.FillMissingDefaults(); err != nil { + t.Fatalf("prepare: failed to set missing defaults: %s", err) + } + } + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got1, got2, err := prepareStepEnvironment(tt.params, &EmptyEnvironment{}) + if tt.wantErr { + require.Error(t, err, "prepareStepEnvironment() expected to return error") + } else { + require.NoError(t, err, "prepareStepEnvironment()") + } + + require.Equal(t, tt.want1, got1, "prepareStepEnvironment() first return value") + require.Equal(t, tt.want2, got2, "prepareStepEnvironment() second return value") + }) + } +} diff --git a/tools/tools.go b/tools/tools.go index b8a702940..d7b4769be 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -338,7 +338,8 @@ func EnvmanClear(envstorePth string) error { return nil } -func getSecretValues(secrets []envmanModels.EnvironmentItemModel) []string { +// GetSecretValues filters out built in configuration parameters from the secret envs +func GetSecretValues(secrets []envmanModels.EnvironmentItemModel) []string { var secretValues []string for _, secret := range secrets { key, value, err := secret.GetKeyValuePair() @@ -375,7 +376,7 @@ func EnvmanRun(envstorePth, errWriter = os.Stderr } else { - outWriter = filterwriter.New(getSecretValues(secrets), os.Stdout) + outWriter = filterwriter.New(GetSecretValues(secrets), os.Stdout) errWriter = outWriter } diff --git a/vendor/github.com/bitrise-io/envman/bitrise.yml b/vendor/github.com/bitrise-io/envman/bitrise.yml index e8ac35bba..7c38df981 100644 --- a/vendor/github.com/bitrise-io/envman/bitrise.yml +++ b/vendor/github.com/bitrise-io/envman/bitrise.yml @@ -31,6 +31,10 @@ workflows: export PR="" PULL_REQUEST_ID="" export INTEGRATION_TEST_BINARY_PATH="$current_envman" + + # prevent the env var content (with the content of this script) + # being added to the test process environment + unset content go test -v ./_tests/integration/... create-binaries: diff --git a/vendor/github.com/bitrise-io/envman/cli/run.go b/vendor/github.com/bitrise-io/envman/cli/run.go index 6e47936bc..2ed5049d9 100644 --- a/vendor/github.com/bitrise-io/envman/cli/run.go +++ b/vendor/github.com/bitrise-io/envman/cli/run.go @@ -1,10 +1,10 @@ package cli import ( - "fmt" "os" log "github.com/Sirupsen/logrus" + "github.com/bitrise-io/envman/env" "github.com/bitrise-io/envman/envman" "github.com/bitrise-io/envman/models" "github.com/bitrise-io/go-utils/command" @@ -22,40 +22,18 @@ func expandEnvsInString(inp string) string { return os.ExpandEnv(inp) } -func commandEnvs(envs []models.EnvironmentItemModel) ([]string, error) { - for _, env := range envs { - key, value, err := env.GetKeyValuePair() - if err != nil { - return []string{}, err - } - - opts, err := env.GetOptions() - if err != nil { - return []string{}, err - } - - if opts.Unset != nil && *opts.Unset { - if err := os.Unsetenv(key); err != nil { - return []string{}, fmt.Errorf("unset env (%s): %s", key, err) - } - continue - } - - if *opts.SkipIfEmpty && value == "" { - continue - } - - var valueStr string - if *opts.IsExpand { - valueStr = expandEnvsInString(value) - } else { - valueStr = value - } +func commandEnvs(newEnvs []models.EnvironmentItemModel) ([]string, error) { + result, err := env.GetDeclarationsSideEffects(newEnvs, &env.DefaultEnvironmentSource{}) + if err != nil { + return nil, err + } - if err := os.Setenv(key, valueStr); err != nil { - return []string{}, err + for _, command := range result.CommandHistory { + if err := env.ExecuteCommand(command); err != nil { + return nil, err } } + return os.Environ(), nil } diff --git a/vendor/github.com/bitrise-io/envman/env/expand.go b/vendor/github.com/bitrise-io/envman/env/expand.go new file mode 100644 index 000000000..26894199c --- /dev/null +++ b/vendor/github.com/bitrise-io/envman/env/expand.go @@ -0,0 +1,195 @@ +package env + +import ( + "fmt" + "os" + "strings" + + "github.com/bitrise-io/envman/models" +) + +// Action is a possible action changing an environment variable +type Action int + +const ( + // InvalidAction represents an unexpected state + InvalidAction Action = iota + 1 + // SetAction is an environment variable assignement, like os.Setenv + SetAction + // UnsetAction is an action to clear (if existing) an environment variable, like os.Unsetenv + UnsetAction + // SkipAction means that no action is performed (usually for an env with an empty value) + SkipAction +) + +// Command describes an action performed on an envrionment variable +type Command struct { + Action Action + Variable Variable +} + +// Variable is an environment variable +type Variable struct { + Key string + Value string +} + +// DeclarationSideEffects is returned by GetDeclarationsSideEffects() +type DeclarationSideEffects struct { + // CommandHistory is an ordered list of commands: when performed in sequence, + // will result in a environment that contains the declared env vars + CommandHistory []Command + // ResultEnvironment is returned for reference, + // it will equal the environment after performing the commands + ResultEnvironment map[string]string +} + +// EnvironmentSource implementations can return an initial environment +type EnvironmentSource interface { + GetEnvironment() map[string]string +} + +// DefaultEnvironmentSource is a default implementation of EnvironmentSource, returns the current environment +type DefaultEnvironmentSource struct{} + +// GetEnvironment returns the current process' environment +func (*DefaultEnvironmentSource) GetEnvironment() map[string]string { + processEnvs := os.Environ() + envs := make(map[string]string) + + // String names can be duplicated (on Unix), and the Go libraries return the first instance of them: + // https://github.com/golang/go/blob/98d20fb23551a7ab900fcfe9d25fd9cb6a98a07f/src/syscall/env_unix.go#L45 + // From https://pubs.opengroup.org/onlinepubs/9699919799/: + // > "There is no meaning associated with the order of strings in the environment. + // > If more than one string in an environment of a process has the same name, the consequences are undefined." + for _, env := range processEnvs { + key, value := SplitEnv(env) + if key == "" { + continue + } + + envs[key] = value + } + + return envs +} + +// SplitEnv splits an env returned by os.Environ +func SplitEnv(env string) (key string, value string) { + const sep = "=" + split := strings.SplitAfterN(env, sep, 2) + if split == nil { + return "", "" + } + key = strings.TrimSuffix(split[0], sep) + if len(split) > 1 { + value = split[1] + } + return +} + +// GetDeclarationsSideEffects iterates over the list of ordered new declared variables sequentally and returns the needed +// commands (like os.Setenv) to add the variables to the current environment. +// The current process environment is not changed. +// Variable expansion is done also, every new variable can reference the previous and initial environments (via EnvironmentSource) +// The new variables (models.EnvironmentItemModel) can be defined in the envman definition file, or filled in directly. +// If the source of the variables (models.EnvironmentItemModel) is the bitrise.yml workflow, +// they will be in this order: +// - Bitrise CLI configuration paramters (IS_CI, IS_DEBUG) +// - App secrets +// - App level envs +// - Workflow level envs +// - Additional Step inputs envs (BITRISE_STEP_SOURCE_DIR; BitriseTestDeployDirEnvKey ("BITRISE_TEST_DEPLOY_DIR"), PWD) +// - Input envs +func GetDeclarationsSideEffects(newEnvs []models.EnvironmentItemModel, envSource EnvironmentSource) (DeclarationSideEffects, error) { + envs := envSource.GetEnvironment() + commandHistory := make([]Command, len(newEnvs)) + + for i, env := range newEnvs { + command, err := getDeclarationCommand(env, envs) + if err != nil { + return DeclarationSideEffects{}, fmt.Errorf("failed to parse new environment variable (%s): %s", env, err) + } + + commandHistory[i] = command + + switch command.Action { + case SetAction: + envs[command.Variable.Key] = command.Variable.Value + case UnsetAction: + delete(envs, command.Variable.Key) + case SkipAction: + default: + return DeclarationSideEffects{}, fmt.Errorf("invalid case for environement declaration action: %#v", command) + } + } + + return DeclarationSideEffects{ + CommandHistory: commandHistory, + ResultEnvironment: envs, + }, nil +} + +// getDeclarationCommand maps a variable to be daclered (env) to an expanded env key and value. +// The current process environment is not changed. +func getDeclarationCommand(env models.EnvironmentItemModel, envs map[string]string) (Command, error) { + envKey, envValue, err := env.GetKeyValuePair() + if err != nil { + return Command{}, fmt.Errorf("failed to get new environment variable name and value: %s", err) + } + + options, err := env.GetOptions() + if err != nil { + return Command{}, fmt.Errorf("failed to get new environment options: %s", err) + } + + if options.Unset != nil && *options.Unset { + return Command{ + Action: UnsetAction, + Variable: Variable{Key: envKey}, + }, nil + } + + if options.SkipIfEmpty != nil && *options.SkipIfEmpty && envValue == "" { + return Command{ + Action: SkipAction, + Variable: Variable{Key: envKey}, + }, nil + } + + mappingFuncFactory := func(envs map[string]string) func(string) string { + return func(key string) string { + if _, ok := envs[key]; !ok { + return "" + } + + return envs[key] + } + } + + if options.IsExpand != nil && *options.IsExpand { + envValue = os.Expand(envValue, mappingFuncFactory(envs)) + } + + return Command{ + Action: SetAction, + Variable: Variable{ + Key: envKey, + Value: envValue, + }, + }, nil +} + +// ExecuteCommand sets the current process's envrionment +func ExecuteCommand(command Command) error { + switch command.Action { + case SetAction: + return os.Setenv(command.Variable.Key, command.Variable.Value) + case UnsetAction: + return os.Unsetenv(command.Variable.Key) + case SkipAction: + return nil + default: + return fmt.Errorf("invalid case for environement declaration action: %#v", command) + } +} diff --git a/vendor/github.com/bitrise-io/envman/env/sharedtestcases.go b/vendor/github.com/bitrise-io/envman/env/sharedtestcases.go new file mode 100644 index 000000000..280dd6a8b --- /dev/null +++ b/vendor/github.com/bitrise-io/envman/env/sharedtestcases.go @@ -0,0 +1,200 @@ +package env + +import ( + "github.com/bitrise-io/envman/models" +) + +// EnvmanSharedTestCases are test cases used as unit and integration tests. +var EnvmanSharedTestCases = []struct { + Name string + Envs []models.EnvironmentItemModel + Want []Command +}{ + { + Name: "empty env list", + Envs: []models.EnvironmentItemModel{}, + Want: []Command{}, + }, + { + Name: "unset env", + Envs: []models.EnvironmentItemModel{ + {"A": "B", "opts": map[string]interface{}{"unset": true}}, + }, + Want: []Command{ + {Action: UnsetAction, Variable: Variable{Key: "A"}}, + }, + }, + { + Name: "set env", + Envs: []models.EnvironmentItemModel{ + {"A": "B", "opts": map[string]interface{}{}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "A", Value: "B"}}, + }, + }, + { + Name: "set multiple envs", + Envs: []models.EnvironmentItemModel{ + {"A": "B", "opts": map[string]interface{}{}}, + {"B": "C", "opts": map[string]interface{}{}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "A", Value: "B"}}, + {Action: SetAction, Variable: Variable{Key: "B", Value: "C"}}, + }, + }, + { + Name: "set int env", + Envs: []models.EnvironmentItemModel{ + {"A": 12, "opts": map[string]interface{}{}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "A", Value: "12"}}, + }, + }, + { + Name: "skip env", + Envs: []models.EnvironmentItemModel{ + {"A": "B", "opts": map[string]interface{}{}}, + {"S": "", "opts": map[string]interface{}{"skip_if_empty": true}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "A", Value: "B"}}, + {Action: SkipAction, Variable: Variable{Key: "S"}}, + }, + }, + { + Name: "skip env, do not skip if not empty", + Envs: []models.EnvironmentItemModel{ + {"A": "B", "opts": map[string]interface{}{}}, + {"S": "T", "opts": map[string]interface{}{"skip_if_empty": true}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "A", Value: "B"}}, + {Action: SetAction, Variable: Variable{Key: "S", Value: "T"}}, + }, + }, + { + Name: "Env does only depend on envs declared before them", + Envs: []models.EnvironmentItemModel{ + {"simulator_device": "$simulator_major", "opts": map[string]interface{}{"is_expand": true}}, + {"simulator_major": "12", "opts": map[string]interface{}{"is_expand": false}}, + {"simulator_os_version": "$simulator_device", "opts": map[string]interface{}{"is_expand": true}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "simulator_device", Value: ""}}, + {Action: SetAction, Variable: Variable{Key: "simulator_major", Value: "12"}}, + {Action: SetAction, Variable: Variable{Key: "simulator_os_version", Value: ""}}, + }, + }, + { + Name: "Env does only depend on envs declared before them (input order switched)", + Envs: []models.EnvironmentItemModel{ + {"simulator_device": "$simulator_major", "opts": map[string]interface{}{"is_expand": true}}, + {"simulator_os_version": "$simulator_device", "opts": map[string]interface{}{"is_sensitive": false}}, + {"simulator_major": "12", "opts": map[string]interface{}{"is_expand": true}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "simulator_device", Value: ""}}, + {Action: SetAction, Variable: Variable{Key: "simulator_os_version", Value: ""}}, + {Action: SetAction, Variable: Variable{Key: "simulator_major", Value: "12"}}, + }, + }, + { + Name: "Env does only depend on envs declared before them, envs in a loop", + Envs: []models.EnvironmentItemModel{ + {"A": "$C", "opts": map[string]interface{}{"is_expand": true}}, + {"B": "$A", "opts": map[string]interface{}{"is_expand": true}}, + {"C": "$B", "opts": map[string]interface{}{"is_expand": true}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "A", Value: ""}}, + {Action: SetAction, Variable: Variable{Key: "B", Value: ""}}, + {Action: SetAction, Variable: Variable{Key: "C", Value: ""}}, + }, + }, + { + Name: "Do not expand env if is_expand is false", + Envs: []models.EnvironmentItemModel{ + {"SIMULATOR_OS_VERSION": "13.3", "opts": map[string]interface{}{"is_expand": true}}, + {"simulator_os_version": "$SIMULATOR_OS_VERSION", "opts": map[string]interface{}{"is_expand": false}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "SIMULATOR_OS_VERSION", Value: "13.3"}}, + {Action: SetAction, Variable: Variable{Key: "simulator_os_version", Value: "$SIMULATOR_OS_VERSION"}}, + }, + }, + { + Name: "Expand env, self reference", + Envs: []models.EnvironmentItemModel{ + {"SIMULATOR_OS_VERSION": "$SIMULATOR_OS_VERSION", "opts": map[string]interface{}{"is_expand": true}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "SIMULATOR_OS_VERSION", Value: ""}}, + }, + }, + { + Name: "Expand env, input contains env var", + Envs: []models.EnvironmentItemModel{ + {"SIMULATOR_OS_VERSION": "13.3", "opts": map[string]interface{}{"is_expand": false}}, + {"simulator_os_version": "$SIMULATOR_OS_VERSION", "opts": map[string]interface{}{"is_expand": true}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "SIMULATOR_OS_VERSION", Value: "13.3"}}, + {Action: SetAction, Variable: Variable{Key: "simulator_os_version", Value: "13.3"}}, + }, + }, + { + Name: "Multi level env var expansion", + Envs: []models.EnvironmentItemModel{ + {"A": "1", "opts": map[string]interface{}{"is_expand": true}}, + {"B": "$A", "opts": map[string]interface{}{"is_expand": true}}, + {"C": "prefix $B", "opts": map[string]interface{}{"is_expand": true}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "A", Value: "1"}}, + {Action: SetAction, Variable: Variable{Key: "B", Value: "1"}}, + {Action: SetAction, Variable: Variable{Key: "C", Value: "prefix 1"}}, + }, + }, + { + Name: "Multi level env var expansion 2", + Envs: []models.EnvironmentItemModel{ + {"SIMULATOR_OS_MAJOR_VERSION": "13", "opts": map[string]interface{}{"is_expand": true}}, + {"SIMULATOR_OS_MINOR_VERSION": "3", "opts": map[string]interface{}{"is_expand": true}}, + {"SIMULATOR_OS_VERSION": "$SIMULATOR_OS_MAJOR_VERSION.$SIMULATOR_OS_MINOR_VERSION", "opts": map[string]interface{}{"is_expand": true}}, + {"simulator_os_version": "$SIMULATOR_OS_VERSION", "opts": map[string]interface{}{"is_expand": true}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "SIMULATOR_OS_MAJOR_VERSION", Value: "13"}}, + {Action: SetAction, Variable: Variable{Key: "SIMULATOR_OS_MINOR_VERSION", Value: "3"}}, + {Action: SetAction, Variable: Variable{Key: "SIMULATOR_OS_VERSION", Value: "13.3"}}, + {Action: SetAction, Variable: Variable{Key: "simulator_os_version", Value: "13.3"}}, + }, + }, + { + Name: "Env expand, duplicate env declarations", + Envs: []models.EnvironmentItemModel{ + {"simulator_os_version": "12.1", "opts": map[string]interface{}{}}, + {"simulator_device": "iPhone 8 ($simulator_os_version)", "opts": map[string]interface{}{"is_expand": "true"}}, + {"simulator_os_version": "13.3", "opts": map[string]interface{}{}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "simulator_os_version", Value: "12.1"}}, + {Action: SetAction, Variable: Variable{Key: "simulator_device", Value: "iPhone 8 (12.1)"}}, + {Action: SetAction, Variable: Variable{Key: "simulator_os_version", Value: "13.3"}}, + }, + }, + { + Name: "is_sensitive property is not affecting input expansion", + Envs: []models.EnvironmentItemModel{ + {"SECRET_ENV": "top secret", "opts": map[string]interface{}{"is_sensitive": true}}, + {"simulator_device": "iPhone $SECRET_ENV", "opts": map[string]interface{}{"is_expand": true, "is_sensitive": false}}, + }, + Want: []Command{ + {Action: SetAction, Variable: Variable{Key: "SECRET_ENV", Value: "top secret"}}, + {Action: SetAction, Variable: Variable{Key: "simulator_device", Value: "iPhone top secret"}}, + }, + }, +}