From 4c3f655d25d216363c386323b45abaf992bcba68 Mon Sep 17 00:00:00 2001 From: Mario Manno Date: Tue, 7 Aug 2018 13:42:55 +0200 Subject: [PATCH] Switch to BOSH manifest style variables --- app/fissile.go | 2 +- kube/pod.go | 16 +- kube/pod_test.go | 117 ++++++------ kube/secret.go | 16 +- kube/secret_test.go | 92 ++++----- kube/values.go | 20 +- model/configuration.go | 26 +++ model/mustache.go | 58 +++--- model/roles.go | 178 +++--------------- model/roles_test.go | 3 +- model/variables.go | 117 ++++++++++++ model/variables_test.go | 29 +++ .../deployment-manifests/bosh-deployment.yml | 21 +++ test-assets/role-manifests/app/hashmat.yml | 4 +- .../app/tor-validation-issues.yml | 10 +- .../role-manifests/app/tor-validation-ok.yml | 15 +- .../role-manifests/builder/tor-good.yml | 10 +- ...ainers-with-stateful-set-and-empty-dir.yml | 7 +- test-assets/role-manifests/kube/volumes.yml | 7 +- .../model/bad-cv-type-internal.yml | 9 +- .../role-manifests/model/bad-cv-type.yml | 13 +- .../role-manifests/model/bosh-run-missing.yml | 10 +- .../role-manifests/model/templates-non.yml | 8 +- .../model/variable-expansion.yml | 10 +- .../model/variables-badly-sorted.yml | 12 +- .../model/variables-with-dup-prev-names.yml | 14 +- .../model/variables-without-decl.yml | 8 +- .../model/variables-without-usage.yml | 3 +- 28 files changed, 466 insertions(+), 369 deletions(-) create mode 100644 model/configuration.go create mode 100644 model/variables.go create mode 100644 model/variables_test.go create mode 100644 test-assets/deployment-manifests/bosh-deployment.yml diff --git a/app/fissile.go b/app/fissile.go index d8990f04..a366cd83 100644 --- a/app/fissile.go +++ b/app/fissile.go @@ -912,7 +912,7 @@ func (f *Fissile) GenerateKube(roleManifestPath string, defaultFiles []string, s cvs := model.MakeMapOfVariables(settings.RoleManifest) for key, value := range cvs { - if !value.Secret { + if !value.CVOptions.Secret { delete(cvs, key) } } diff --git a/kube/pod.go b/kube/pod.go index 71160b68..36e8f1ab 100644 --- a/kube/pod.go +++ b/kube/pod.go @@ -330,7 +330,7 @@ func getEnvVars(role *model.InstanceGroup, settings ExportSettings) (helm.Node, return getEnvVarsFromConfigs(configs, settings) } -func getEnvVarsFromConfigs(configs model.ConfigurationVariableSlice, settings ExportSettings) (helm.Node, error) { +func getEnvVarsFromConfigs(configs model.Variables, settings ExportSettings) (helm.Node, error) { sizingCountRegexp := regexp.MustCompile("^KUBE_SIZING_([A-Z][A-Z_]*)_COUNT$") sizingPortsRegexp := regexp.MustCompile("^KUBE_SIZING_([A-Z][A-Z_]*)_PORTS_([A-Z][A-Z_]*)_(MIN|MAX)$") @@ -344,7 +344,7 @@ func getEnvVarsFromConfigs(configs model.ConfigurationVariableSlice, settings Ex if role == nil { return nil, fmt.Errorf("Role %s for %s not found", roleName, config.Name) } - if config.Secret { + if config.CVOptions.Secret { return nil, fmt.Errorf("%s must not be a secret variable", config.Name) } if settings.CreateHelmChart { @@ -366,7 +366,7 @@ func getEnvVarsFromConfigs(configs model.ConfigurationVariableSlice, settings Ex if role == nil { return nil, fmt.Errorf("Role %s for %s not found", roleName, config.Name) } - if config.Secret { + if config.CVOptions.Secret { return nil, fmt.Errorf("%s must not be a secret variable", config.Name) } @@ -434,14 +434,14 @@ func getEnvVarsFromConfigs(configs model.ConfigurationVariableSlice, settings Ex continue } - if config.Secret { + if config.CVOptions.Secret { if !settings.CreateHelmChart { env = append(env, makeSecretVar(config.Name, false)) } else { - if config.Immutable && config.Generator != nil { + if config.CVOptions.Immutable && config.Type != "" { // Users cannot override immutable secrets that are generated env = append(env, makeSecretVar(config.Name, true)) - } else if config.Generator == nil { + } else if config.Type == "" { env = append(env, makeSecretVar(config.Name, false)) } else { // Generated secrets can be overridden by the user (unless immutable) @@ -456,9 +456,9 @@ func getEnvVarsFromConfigs(configs model.ConfigurationVariableSlice, settings Ex } var stringifiedValue string - if settings.CreateHelmChart && config.Type == model.CVTypeUser { + if settings.CreateHelmChart && config.CVOptions.Type == model.CVTypeUser { required := `""` - if config.Required { + if config.CVOptions.Required { required = fmt.Sprintf(`{{fail "env.%s has not been set"}}`, config.Name) } name := ".Values.env." + config.Name diff --git a/kube/pod_test.go b/kube/pod_test.go index a93a9088..b87f19c7 100644 --- a/kube/pod_test.go +++ b/kube/pod_test.go @@ -40,14 +40,15 @@ func podTemplateTestLoadRole(assert *assert.Assertions) *model.InstanceGroup { } // Force a broadcast SECRET_VAR into the manifest, to see in all roles - manifest.Configuration.Variables = - append(manifest.Configuration.Variables, - &model.ConfigurationVariable{ - Name: "SECRET_VAR", + manifest.Variables = append(manifest.Variables, + &model.VariableDefinition{ + Name: "SECRET_VAR", + CVOptions: model.CVOptions{ Type: model.CVTypeUser, Secret: true, Internal: true, - }) + }, + }) return instanceGroup } @@ -390,8 +391,8 @@ func TestPodGetEnvVarsFromConfigSizingCountKube(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "KUBE_SIZING_FOO_COUNT", }, }, ExportSettings{ @@ -427,8 +428,8 @@ func TestPodGetEnvVarsFromConfigSizingCountHelm(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "KUBE_SIZING_FOO_COUNT", }, }, ExportSettings{ @@ -464,11 +465,11 @@ func TestPodGetEnvVarsFromConfigSizingPortsKube(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "KUBE_SIZING_FOO_PORTS_STORE_MIN", }, - &model.ConfigurationVariable{ + &model.VariableDefinition{ Name: "KUBE_SIZING_FOO_PORTS_STORE_MAX", }, }, ExportSettings{ @@ -511,11 +512,11 @@ func TestPodGetEnvVarsFromConfigSizingPortsHelm(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "KUBE_SIZING_FOO_PORTS_STORE_MIN", }, - &model.ConfigurationVariable{ + &model.VariableDefinition{ Name: "KUBE_SIZING_FOO_PORTS_STORE_MAX", }, }, ExportSettings{ @@ -562,8 +563,8 @@ func TestPodGetEnvVarsFromConfigGenerationCounterKube(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "KUBE_SECRETS_GENERATION_COUNTER", }, }, ExportSettings{ @@ -594,8 +595,8 @@ func TestPodGetEnvVarsFromConfigGenerationCounterHelm(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "KUBE_SECRETS_GENERATION_COUNTER", }, }, ExportSettings{ @@ -631,8 +632,8 @@ func TestPodGetEnvVarsFromConfigGenerationNameKube(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "KUBE_SECRETS_GENERATION_NAME", }, }, ExportSettings{ @@ -663,8 +664,8 @@ func TestPodGetEnvVarsFromConfigGenerationNameHelm(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "KUBE_SECRETS_GENERATION_NAME", }, }, ExportSettings{ @@ -700,10 +701,12 @@ func TestPodGetEnvVarsFromConfigGenerationNameHelm(t *testing.T) { func TestPodGetEnvVarsFromConfigSecretsKube(t *testing.T) { assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ - Name: "A_SECRET", - Secret: true, + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ + Name: "A_SECRET", + CVOptions: model.CVOptions{ + Secret: true, + }, }, }, ExportSettings{ RoleManifest: &model.RoleManifest{ @@ -749,10 +752,12 @@ func TestPodGetEnvVarsFromConfigSecretsHelm(t *testing.T) { t.Run("Plain", func(t *testing.T) { t.Parallel() - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ - Name: "A_SECRET", - Secret: true, + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ + Name: "A_SECRET", + CVOptions: model.CVOptions{ + Secret: true, + }, }, }, settings) if !assert.NoError(err) { @@ -779,14 +784,12 @@ func TestPodGetEnvVarsFromConfigSecretsHelm(t *testing.T) { t.Run("Generated", func(t *testing.T) { t.Parallel() - cv := []*model.ConfigurationVariable{ - &model.ConfigurationVariable{ - Name: "A_SECRET", - Secret: true, - Generator: &model.ConfigurationVariableGenerator{ - ID: "no", - Type: model.GeneratorTypePassword, - ValueType: "foo-login", + cv := model.Variables{ + &model.VariableDefinition{ + Name: "A_SECRET", + Type: "password", + CVOptions: model.CVOptions{ + Secret: true, }, }, } @@ -846,7 +849,7 @@ func TestPodGetEnvVarsFromConfigSecretsHelm(t *testing.T) { `, actual) }) - cv[0].Immutable = true + cv[0].CVOptions.Immutable = true ev, err = getEnvVarsFromConfigs(cv, settings) if !assert.NoError(err) { return @@ -888,10 +891,12 @@ func TestPodGetEnvVarsFromConfigNonSecretKube(t *testing.T) { t.Run("Present", func(t *testing.T) { t.Parallel() - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ - Name: "SOMETHING", - Default: []string{"or", "other"}, + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ + Name: "SOMETHING", + CVOptions: model.CVOptions{ + Default: []string{"or", "other"}, + }, }, }, settings) @@ -912,8 +917,8 @@ func TestPodGetEnvVarsFromConfigNonSecretKube(t *testing.T) { t.Run("Missing", func(t *testing.T) { t.Parallel() - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "SOMETHING", }, }, settings) @@ -935,10 +940,12 @@ func TestPodGetEnvVarsFromConfigNonSecretHelmUserOptional(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ Name: "SOMETHING", - Type: model.CVTypeUser, + CVOptions: model.CVOptions{ + Type: model.CVTypeUser, + }, }, }, ExportSettings{ CreateHelmChart: true, @@ -1000,11 +1007,13 @@ func TestPodGetEnvVarsFromConfigNonSecretHelmUserRequired(t *testing.T) { t.Parallel() assert := assert.New(t) - ev, err := getEnvVarsFromConfigs([]*model.ConfigurationVariable{ - &model.ConfigurationVariable{ - Name: "SOMETHING", - Type: model.CVTypeUser, - Required: true, + ev, err := getEnvVarsFromConfigs(model.Variables{ + &model.VariableDefinition{ + Name: "SOMETHING", + CVOptions: model.CVOptions{ + Type: model.CVTypeUser, + Required: true, + }, }, }, ExportSettings{ CreateHelmChart: true, diff --git a/kube/secret.go b/kube/secret.go index ac38de6c..66d5785f 100644 --- a/kube/secret.go +++ b/kube/secret.go @@ -18,16 +18,16 @@ func MakeSecrets(secrets model.CVMap, settings ExportSettings) (helm.Node, error for name, cv := range secrets { key := util.ConvertNameToKey(name) var value interface{} - comment := cv.Description + comment := cv.CVOptions.Description if settings.CreateHelmChart { - if cv.Generator == nil { - if cv.Immutable { + if cv.Type == "" { + if cv.CVOptions.Immutable { comment += "\nThis value is immutable and must not be changed once set." } - comment += formattedExample(cv.Example, value) + comment += formattedExample(cv.CVOptions.Example, value) required := `{{"" | b64enc | quote}}` - if cv.Required { + if cv.CVOptions.Required { required = fmt.Sprintf(`{{fail "secrets.%s has not been set"}}`, cv.Name) } name := ".Values.secrets." + cv.Name @@ -35,8 +35,8 @@ func MakeSecrets(secrets model.CVMap, settings ExportSettings) (helm.Node, error `{{%s | toJson | b64enc | quote}}{{else}}{{%s | b64enc | quote}}{{end}}{{else}}%s{{end}}` value = fmt.Sprintf(tmpl, name, name, name, name, required) data.Add(key, helm.NewNode(value, helm.Comment(comment))) - } else if !cv.Immutable { - comment += formattedExample(cv.Example, value) + } else if !cv.CVOptions.Immutable { + comment += formattedExample(cv.CVOptions.Example, value) comment += "\nThis value uses a generated default." value = fmt.Sprintf(`{{ default "" .Values.secrets.%s | b64enc | quote }}`, cv.Name) generated.Add(key, helm.NewNode(value, helm.Comment(comment))) @@ -48,7 +48,7 @@ func MakeSecrets(secrets model.CVMap, settings ExportSettings) (helm.Node, error value = "" } value = base64.StdEncoding.EncodeToString([]byte(value)) - comment += formattedExample(cv.Example, value) + comment += formattedExample(cv.CVOptions.Example, value) data.Add(key, helm.NewNode(value, helm.Comment(comment))) } } diff --git a/kube/secret_test.go b/kube/secret_test.go index 8ae42bff..de97f394 100644 --- a/kube/secret_test.go +++ b/kube/secret_test.go @@ -63,56 +63,62 @@ func TestMakeSecretsEmpty(t *testing.T) { func testCVMap() model.CVMap { return model.CVMap{ - "optional": &model.ConfigurationVariable{ - // This variable is only defined to verify that missing non-required secrets won't throw any errors - Name: "optional", - Required: false, - Default: nil, + "optional": &model.VariableDefinition{ + Name: "optional", + CVOptions: model.CVOptions{ + // This variable is only defined to verify that missing non-required secrets won't throw any errors + Required: false, + Default: nil, + }, }, - "min": &model.ConfigurationVariable{ + "min": &model.VariableDefinition{ Name: "min", }, - "desc": &model.ConfigurationVariable{ - Name: "desc", - Description: "<<>>", - Example: "Use this", + "desc": &model.VariableDefinition{ + Name: "desc", + CVOptions: model.CVOptions{ + Description: "<<>>", + Example: "Use this", + }, }, - "valued": &model.ConfigurationVariable{ - Name: "valued", - Description: "<<>>", - Default: "you are very valued indeed", + "valued": &model.VariableDefinition{ + Name: "valued", + CVOptions: model.CVOptions{ + Description: "<<>>", + Default: "you are very valued indeed", + }, }, - "structured": &model.ConfigurationVariable{ - Name: "structured", - Description: "<<>>", - Example: "use: \"this\"\n", - Default: map[string]string{"non": "scalar"}, + "structured": &model.VariableDefinition{ + Name: "structured", + CVOptions: model.CVOptions{ + Description: "<<>>", + Example: "use: \"this\"\n", + Default: map[string]string{"non": "scalar"}, + }, }, - "const": &model.ConfigurationVariable{ - Name: "const", - Description: "<<>>", - Default: "rock solid", - Required: true, - Immutable: true, + "const": &model.VariableDefinition{ + Name: "const", + CVOptions: model.CVOptions{ + Description: "<<>>", + Default: "rock solid", + Required: true, + Immutable: true, + }, }, - "genie": &model.ConfigurationVariable{ - Name: "genie", - Description: "<<>>", - Generator: &model.ConfigurationVariableGenerator{ - ID: "xxx", - Type: model.GeneratorTypePassword, - ValueType: "snafu", + "genie": &model.VariableDefinition{ + Name: "genie", + Type: "password", + CVOptions: model.CVOptions{ + Description: "<<>>", }, }, - "guinevere": &model.ConfigurationVariable{ + "guinevere": &model.VariableDefinition{ // excluded from __helm__ output - Name: "guinevere", - Description: "<<