From e6a5d004dee13daaefc88abae1257836f731f1b6 Mon Sep 17 00:00:00 2001 From: John Ryan Date: Thu, 19 May 2022 12:01:25 -0700 Subject: [PATCH 1/3] Accept directory paths in ReadFilesFunc ... so that data values can be loaded via all the strategies that files are loaded from `--file` Also: - update the starting example to illustrate that all values are plain merged. - Share symlink options between files and DV options structs so that when the configuration is set, it's available to both RegularilesSourceOpts code and DataValuesFlag code. - Update help text with more broadly used file input formats. - Adjust some other help text to be more helpful. --- examples/data-values-directory/expected.txt | 12 +++ pkg/cmd/template/cmd.go | 9 +- pkg/cmd/template/cmd_data_values_file_test.go | 99 +++++++++++++++++-- pkg/cmd/template/cmd_data_values_test.go | 48 ++++----- pkg/cmd/template/data_values_flags.go | 92 ++++++++++------- pkg/cmd/template/regular_input.go | 6 +- pkg/cmd/template/schema_consumer_test.go | 22 ++--- pkg/files/file.go | 6 ++ test/e2e/e2e_test.go | 2 +- 9 files changed, 210 insertions(+), 86 deletions(-) create mode 100644 examples/data-values-directory/expected.txt diff --git a/examples/data-values-directory/expected.txt b/examples/data-values-directory/expected.txt new file mode 100644 index 00000000..c3abcac0 --- /dev/null +++ b/examples/data-values-directory/expected.txt @@ -0,0 +1,12 @@ +values: + name: suggestion-service + instances: 4 + accept_insecure_conns: false + cache: + driver: redis + config: + maxEntries: 1024 + strategy: MRU + host: localhost:6379 + tls-client-cert-file: client.crt + tls-client-key-file: client.key diff --git a/pkg/cmd/template/cmd.go b/pkg/cmd/template/cmd.go index 7b000b56..8f33a321 100644 --- a/pkg/cmd/template/cmd.go +++ b/pkg/cmd/template/cmd.go @@ -54,8 +54,13 @@ type FileSource interface { var _ []FileSource = []FileSource{&BulkFilesSource{}, &RegularFilesSource{}} +// NewOptions initializes a new instance of template.Options. func NewOptions() *Options { - return &Options{} + var opts files.SymlinkAllowOpts + return &Options{ + RegularFilesSourceOpts: RegularFilesSourceOpts{SymlinkAllowOpts: &opts}, + DataValuesFlags: DataValuesFlags{SymlinkAllowOpts: &opts}, + } } // BindFlags registers template flags for template command. @@ -66,7 +71,7 @@ func (o *Options) BindFlags(cmdFlags CmdFlags) { "Configure whether implicit map keys overrides are allowed") cmdFlags.BoolVarP(&o.StrictYAML, "strict", "s", false, "Configure to use _strict_ YAML subset") cmdFlags.BoolVar(&o.Debug, "debug", false, "Enable debug output") - cmdFlags.BoolVar(&o.InspectFiles, "files-inspect", false, "Inspect files") + cmdFlags.BoolVar(&o.InspectFiles, "files-inspect", false, "Determine the set of files that would be processed and display that result") o.BulkFilesSourceOpts.Set(cmdFlags) o.RegularFilesSourceOpts.Set(cmdFlags) diff --git a/pkg/cmd/template/cmd_data_values_file_test.go b/pkg/cmd/template/cmd_data_values_file_test.go index 9ab260b9..c1172c72 100644 --- a/pkg/cmd/template/cmd_data_values_file_test.go +++ b/pkg/cmd/template/cmd_data_values_file_test.go @@ -14,7 +14,7 @@ import ( "github.com/vmware-tanzu/carvel-ytt/pkg/files" ) -func TestDataValuesWithDataValuesFileFlags(t *testing.T) { +func TestDataValuesFilesFlag_acceptsPlainYAML(t *testing.T) { yamlTplData := []byte(` #@ load("@ytt:data", "data") values: #@ data.values`) @@ -94,14 +94,14 @@ array: opts.DataValuesFlags = cmdtpl.DataValuesFlags{ FromFiles: []string{"dvs1.yml", "dvs2.yml", "dvs3.yml"}, - ReadFileFunc: func(path string) ([]byte, error) { + ReadFilesFunc: func(path string) ([]*files.File, error) { switch path { case "dvs1.yml": - return dvs1, nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs1.yml", dvs1))}, nil case "dvs2.yml": - return dvs2, nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs2.yml", dvs2))}, nil case "dvs3.yml": - return dvs3, nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs3.yml", dvs3))}, nil default: return nil, fmt.Errorf("Unknown file '%s'", path) } @@ -118,7 +118,7 @@ array: assert.Equal(t, expectedYAMLTplData, string(file.Bytes())) } -func TestDataValuesWithDataValuesFileFlagsForbiddenComment(t *testing.T) { +func TestDataValuesFilesFlag_rejectsTemplatedYAML(t *testing.T) { yamlTplData := []byte(` #@ load("@ytt:data", "data") values: #@ data.values`) @@ -136,10 +136,10 @@ int: 123`) opts.DataValuesFlags = cmdtpl.DataValuesFlags{ FromFiles: []string{"dvs1.yml"}, - ReadFileFunc: func(path string) ([]byte, error) { + ReadFilesFunc: func(path string) ([]*files.File, error) { switch path { case "dvs1.yml": - return dvs1, nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs1.yml", dvs1))}, nil default: return nil, fmt.Errorf("Unknown file '%s'", path) } @@ -149,3 +149,86 @@ int: 123`) out := opts.RunWithFiles(cmdtpl.Input{Files: filesToProcess}, ui) require.EqualError(t, out.Err, "Extracting data value from file: Checking data values file 'dvs1.yml': Expected to be plain YAML, having no annotations (hint: remove comments starting with `#@`)") } + +func TestDataValuesFilesFlag_WithNonYAMLFiles(t *testing.T) { + t.Run("errors when file is explicitly specified", func(t *testing.T) { + yamlTplData := []byte(` +#@ load("@ytt:data", "data") +values: #@ data.values`) + + toml1 := ` +[foo] + bar = 123 +` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("tpl.yml", yamlTplData)), + }) + + ui := ui.NewTTY(false) + opts := cmdtpl.NewOptions() + + opts.DataValuesFlags = cmdtpl.DataValuesFlags{ + FromFiles: []string{"toml1.toml"}, + ReadFilesFunc: func(path string) ([]*files.File, error) { + switch path { + case "toml1.toml": + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("toml1.toml", []byte(toml1)))}, nil + default: + return nil, fmt.Errorf("Unknown file '%s'", path) + } + }, + } + + out := opts.RunWithFiles(cmdtpl.Input{Files: filesToProcess}, ui) + require.EqualError(t, out.Err, "Extracting data value from file: Unmarshaling YAML data values file 'toml1.toml': yaml: line 2: did not find expected ") + }) + t.Run("skips when path given is a directory", func(t *testing.T) { + yamlTplData := []byte(` +#@ load("@ytt:data", "data") +values: #@ data.values`) + + dvs1 := `--- +int: 123 +` + toml1 := ` +[foo] + bar = 456 +` + + expectedYAMLTplData := `values: + int: 123 +` + + filesToProcess := files.NewSortedFiles([]*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("tpl.yml", yamlTplData)), + }) + + ui := ui.NewTTY(false) + opts := cmdtpl.NewOptions() + + opts.DataValuesFlags = cmdtpl.DataValuesFlags{ + FromFiles: []string{"values"}, + ReadFilesFunc: func(path string) ([]*files.File, error) { + switch path { + case "values": + return []*files.File{ + files.MustNewFileFromSource(files.NewBytesSource("values/dvs1.yml", []byte(dvs1))), + files.MustNewFileFromSource(files.NewBytesSource("values/toml1.toml", []byte(toml1))), + }, nil + default: + return nil, fmt.Errorf("Unknown file '%s'", path) + } + }, + } + + out := opts.RunWithFiles(cmdtpl.Input{Files: filesToProcess}, ui) + require.NoError(t, out.Err) + require.Len(t, out.Files, 1, "unexpected number of output files") + + file := out.Files[0] + + assert.Equal(t, "tpl.yml", file.RelativePath()) + assert.Equal(t, expectedYAMLTplData, string(file.Bytes())) + }) +} diff --git a/pkg/cmd/template/cmd_data_values_test.go b/pkg/cmd/template/cmd_data_values_test.go index 7301a2ed..314cdecf 100644 --- a/pkg/cmd/template/cmd_data_values_test.go +++ b/pkg/cmd/template/cmd_data_values_test.go @@ -181,48 +181,48 @@ another: } func TestDataValuesWithLibraryAttachedFlags(t *testing.T) { - tplBytes := []byte(` + tplBytes := ` #@ load("@ytt:library", "library") #@ load("@ytt:template", "template") #@ load("@ytt:data", "data") root: #@ data.values ---- #@ template.replace(library.get("lib", alias="inst1").eval())`) +--- #@ template.replace(library.get("lib", alias="inst1").eval())` - libTplBytes := []byte(` + libTplBytes := ` #@ load("@ytt:library", "library") #@ load("@ytt:template", "template") #@ load("@ytt:data", "data") from_library: #@ data.values --- #@ template.replace(library.get("nested-lib").eval()) -`) +` - libValuesBytes := []byte(` + libValuesBytes := ` #@data/values --- val0: override-me -`) +` - nestedLibTplBytes := []byte(` + nestedLibTplBytes := ` #@ load("@ytt:data", "data") from_nested_lib: #@ data.values -`) +` - nestedLibValuesBytes := []byte(` + nestedLibValuesBytes := ` #@data/values --- val1: override-me -`) +` - dvs2 := []byte(`val2: 2`) + dvs2 := `val2: 2` - dvs3 := []byte(`val3: 3`) + dvs3 := `val3: 3` - dvs4 := []byte(`val4: 4`) + dvs4 := `val4: 4` - dvs6 := []byte(`6`) + dvs6 := `6` expectedYAMLTplData := `root: val2: 2 @@ -239,11 +239,11 @@ from_nested_lib: ` filesToProcess := files.NewSortedFiles([]*files.File{ - files.MustNewFileFromSource(files.NewBytesSource("config.yml", tplBytes)), - files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/values.yml", libValuesBytes)), - files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/config.yml", libTplBytes)), - files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/_ytt_lib/nested-lib/values.yml", nestedLibValuesBytes)), - files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/_ytt_lib/nested-lib/config.yml", nestedLibTplBytes)), + files.MustNewFileFromSource(files.NewBytesSource("config.yml", []byte(tplBytes))), + files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/values.yml", []byte(libValuesBytes))), + files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/config.yml", []byte(libTplBytes))), + files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/_ytt_lib/nested-lib/values.yml", []byte(nestedLibValuesBytes))), + files.MustNewFileFromSource(files.NewBytesSource("_ytt_lib/lib/_ytt_lib/nested-lib/config.yml", []byte(nestedLibTplBytes))), }) ui := ui.NewTTY(false) @@ -255,16 +255,16 @@ from_nested_lib: EnvFromStrings: []string{"@lib:DVS"}, EnvironFunc: func() []string { return []string{"DVS_val5=5"} }, KVsFromFiles: []string{"@lib:val6=c:\\User\\user\\dvs6.yml"}, - ReadFileFunc: func(path string) ([]byte, error) { + ReadFilesFunc: func(path string) ([]*files.File, error) { switch path { case "c:\\User\\user\\dvs2.yml": - return dvs2, nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs2.yml", []byte(dvs2)))}, nil case "dvs3.yml": - return dvs3, nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs3.yml", []byte(dvs3)))}, nil case "D:\\User\\user\\dvs4.yml": - return dvs4, nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs4.yml", []byte(dvs4)))}, nil case "c:\\User\\user\\dvs6.yml": - return dvs6, nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs6.yml", []byte(dvs6)))}, nil default: return nil, fmt.Errorf("Unknown file '%s'", path) } diff --git a/pkg/cmd/template/data_values_flags.go b/pkg/cmd/template/data_values_flags.go index c6e31eb8..68074945 100644 --- a/pkg/cmd/template/data_values_flags.go +++ b/pkg/cmd/template/data_values_flags.go @@ -5,7 +5,6 @@ package template import ( "fmt" - "io/ioutil" "os" "strings" @@ -40,8 +39,10 @@ type DataValuesFlags struct { InspectSchema bool SkipValidation bool - EnvironFunc func() []string - ReadFileFunc func(string) ([]byte, error) + EnvironFunc func() []string + ReadFilesFunc func(paths string) ([]*files.File, error) + + *files.SymlinkAllowOpts } // Set registers data values ingestion flags and wires-up those flags up to this @@ -52,11 +53,10 @@ func (s *DataValuesFlags) Set(cmdFlags CmdFlags) { cmdFlags.StringArrayVarP(&s.KVsFromStrings, "data-value", "v", nil, "Set specific data value to given value, as string (format: all.key1.subkey=123) (can be specified multiple times)") cmdFlags.StringArrayVar(&s.KVsFromYAML, "data-value-yaml", nil, "Set specific data value to given value, parsed as YAML (format: all.key1.subkey=true) (can be specified multiple times)") - cmdFlags.StringArrayVar(&s.KVsFromFiles, "data-value-file", nil, "Set specific data value to given file contents, as string (format: all.key1.subkey=/file/path) (can be specified multiple times)") - - cmdFlags.StringArrayVar(&s.FromFiles, "data-values-file", nil, "Set multiple data values via a YAML file (format: /file/path.yml) (can be specified multiple times)") + cmdFlags.StringArrayVar(&s.FromFiles, "data-value-file", nil, "Set specific data value to contents of a file (format: [@lib1:]all.key1.subkey={file path, HTTP URL, or '-' (i.e. stdin)}) (can be specified multiple times)") + cmdFlags.StringArrayVar(&s.FromFiles, "data-values-file", nil, "Set multiple data values via plain YAML files (format: [@lib1:]{file path, HTTP URL, or '-' (i.e. stdin)}) (can be specified multiple times)") - cmdFlags.BoolVar(&s.Inspect, "data-values-inspect", false, "Calculate the final data values (applying any overlays) and display that result") + cmdFlags.BoolVar(&s.Inspect, "data-values-inspect", false, "Determine the final data values (applying any overlays) and display that result") if experiments.IsValidationsEnabled() { cmdFlags.BoolVar(&s.SkipValidation, "dangerous-data-values-disable-validation", false, "Skip validating data values (not recommended: may result in templates failing or invalid output)") } @@ -149,34 +149,43 @@ func (s *DataValuesFlags) file(fullPath string, strict bool) ([]*datavalues.Enve return nil, err } - contents, err := s.readFile(path) - if err != nil { - return nil, fmt.Errorf("Reading file '%s': %s", path, err) - } - - docSetOpts := yamlmeta.DocSetOpts{ - AssociatedName: path, - Strict: strict, - } - - docSet, err := yamlmeta.NewDocumentSetFromBytes(contents, docSetOpts) + dvFiles, err := s.asFiles(path) if err != nil { - return nil, fmt.Errorf("Unmarshaling YAML data values file '%s': %s", path, err) + return nil, fmt.Errorf("Find files '%s': %s", path, err) } var result []*datavalues.Envelope + for _, dvFile := range dvFiles { + // Users may want to store other files (docs, etc.) within this directory; ignore those. + if dvFile.IsImplied() && !(dvFile.Type() == files.TypeYAML) { + continue + } + contents, err := dvFile.Bytes() + if err != nil { + return nil, fmt.Errorf("Reading file '%s': %s", dvFile.RelativePath(), err) + } - for _, doc := range docSet.Items { - if doc.Value != nil { - dvsOverlay, err := NewDataValuesFile(doc).AsOverlay() - if err != nil { - return nil, fmt.Errorf("Checking data values file '%s': %s", path, err) - } - dvs, err := datavalues.NewEnvelopeWithLibRef(dvsOverlay, libRef) - if err != nil { - return nil, err + docSetOpts := yamlmeta.DocSetOpts{ + AssociatedName: dvFile.RelativePath(), + Strict: strict, + } + docSet, err := yamlmeta.NewDocumentSetFromBytes(contents, docSetOpts) + if err != nil { + return nil, fmt.Errorf("Unmarshaling YAML data values file '%s': %s", dvFile.RelativePath(), err) + } + + for _, doc := range docSet.Items { + if doc.Value != nil { + dvsOverlay, err := NewDataValuesFile(doc).AsOverlay() + if err != nil { + return nil, fmt.Errorf("Checking data values file '%s': %s", path, err) + } + dvs, err := datavalues.NewEnvelopeWithLibRef(dvsOverlay, libRef) + if err != nil { + return nil, err + } + result = append(result, dvs) } - result = append(result, dvs) } } @@ -267,7 +276,15 @@ func (s *DataValuesFlags) kvFile(kv string) (*datavalues.Envelope, error) { return nil, fmt.Errorf("Expected format key=/file/path") } - contents, err := s.readFile(pieces[1]) + dvFile, err := s.asFiles(pieces[1]) + if err != nil { + return nil, fmt.Errorf("Finding file '%s': %s", pieces[1], err) + } + if len(dvFile) > 1 { + return nil, fmt.Errorf("Expected '%s' to be a file, but is a directory", pieces[1]) + } + + contents, err := dvFile[0].Bytes() if err != nil { return nil, fmt.Errorf("Reading file '%s'", pieces[1]) } @@ -357,12 +374,13 @@ func (s *DataValuesFlags) buildOverlay(keyPieces []string, value interface{}, de return &yamlmeta.Document{Value: resultMap, Position: pos} } -func (s *DataValuesFlags) readFile(path string) ([]byte, error) { - if s.ReadFileFunc != nil { - return s.ReadFileFunc(path) - } - if path == "-" { - return files.ReadStdin() +// asFiles enumerates the files that are found at "path" +// +// If a DataValuesFlags.ReadFilesFunc has been injected, that service is used. +// Otherwise, uses files.NewSortedFilesFromPaths() is used. +func (s *DataValuesFlags) asFiles(path string) ([]*files.File, error) { + if s.ReadFilesFunc != nil { + return s.ReadFilesFunc(path) } - return ioutil.ReadFile(path) + return files.NewSortedFilesFromPaths([]string{path}, *s.SymlinkAllowOpts) } diff --git a/pkg/cmd/template/regular_input.go b/pkg/cmd/template/regular_input.go index 37df2257..69df63b4 100644 --- a/pkg/cmd/template/regular_input.go +++ b/pkg/cmd/template/regular_input.go @@ -22,7 +22,7 @@ type RegularFilesSourceOpts struct { OutputFiles string OutputType OutputType - files.SymlinkAllowOpts + *files.SymlinkAllowOpts } // OutputType holds the user's desire for two (2) categories of output: @@ -35,7 +35,7 @@ type OutputType struct { // Set registers flags related to sourcing ordinary files/directories and wires-up those flags up to this // RegularFilesSourceOpts to be set when the corresponding cobra.Command is executed. func (s *RegularFilesSourceOpts) Set(cmdFlags CmdFlags) { - cmdFlags.StringArrayVarP(&s.files, "file", "f", nil, "File (ie local path, HTTP URL, -) (can be specified multiple times)") + cmdFlags.StringArrayVarP(&s.files, "file", "f", nil, "File(s) to process {filepath, HTTP URL, or '-' (i.e. stdin)} (can be specified multiple times)") cmdFlags.StringVar(&s.outputDir, "dangerous-emptied-output-directory", "", "Delete given directory, and then create it with output files") @@ -65,7 +65,7 @@ func (s *RegularFilesSource) HasInput() bool { return len(s.opts.files) > 0 } func (s *RegularFilesSource) HasOutput() bool { return true } func (s *RegularFilesSource) Input() (Input, error) { - filesToProcess, err := files.NewSortedFilesFromPaths(s.opts.files, s.opts.SymlinkAllowOpts) + filesToProcess, err := files.NewSortedFilesFromPaths(s.opts.files, *s.opts.SymlinkAllowOpts) if err != nil { return Input{}, err } diff --git a/pkg/cmd/template/schema_consumer_test.go b/pkg/cmd/template/schema_consumer_test.go index 66eab997..ab29f391 100644 --- a/pkg/cmd/template/schema_consumer_test.go +++ b/pkg/cmd/template/schema_consumer_test.go @@ -255,10 +255,10 @@ foo: opts.DataValuesFlags = cmdtpl.DataValuesFlags{ FromFiles: []string{"dvs1.yml"}, - ReadFileFunc: func(path string) ([]byte, error) { + ReadFilesFunc: func(path string) ([]*files.File, error) { switch path { case "dvs1.yml": - return []byte(dvs1), nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs1.yml", []byte(dvs1)))}, nil default: return nil, fmt.Errorf("Unknown file '%s'", path) } @@ -707,10 +707,10 @@ rendered: #@ data.values.foo ` cmdOpts.DataValuesFlags = cmdtpl.DataValuesFlags{ KVsFromFiles: []string{"foo=dvs1.yml"}, - ReadFileFunc: func(path string) ([]byte, error) { + ReadFilesFunc: func(path string) ([]*files.File, error) { switch path { case "dvs1.yml": - return []byte(dvs1), nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs1.yml", []byte(dvs1)))}, nil default: return nil, fmt.Errorf("Unknown file '%s'", path) } @@ -752,10 +752,10 @@ rendered: #@ data.values.foo ` cmdOpts.DataValuesFlags = cmdtpl.DataValuesFlags{ FromFiles: []string{"dvs1.yml"}, - ReadFileFunc: func(path string) ([]byte, error) { + ReadFilesFunc: func(path string) ([]*files.File, error) { switch path { case "dvs1.yml": - return []byte(dvs1), nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs1.yml", []byte(dvs1)))}, nil default: return nil, fmt.Errorf("Unknown file '%s'", path) } @@ -787,11 +787,11 @@ dvs1.yml: --- hostname: "" ` - dvs1Data := `--- + dvs1 := `--- not_in_schema: this should be the only violation reported ` - dvs2Data := `--- + dvs2 := `--- hostname: 14 # wrong type; but will never be caught ` templateYAML := `--- @@ -803,12 +803,12 @@ rendered: true` }) opts.DataValuesFlags = cmdtpl.DataValuesFlags{ FromFiles: []string{"dvs1.yml", "dvs2.yml"}, - ReadFileFunc: func(path string) ([]byte, error) { + ReadFilesFunc: func(path string) ([]*files.File, error) { switch path { case "dvs1.yml": - return []byte(dvs1Data), nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs1.yml", []byte(dvs1)))}, nil case "dvs2.yml": - return []byte(dvs2Data), nil + return []*files.File{files.MustNewFileFromSource(files.NewBytesSource("dvs2.yml", []byte(dvs2)))}, nil default: return nil, fmt.Errorf("Unknown file '%s'", path) } diff --git a/pkg/files/file.go b/pkg/files/file.go index 9be81954..26a6a990 100644 --- a/pkg/files/file.go +++ b/pkg/files/file.go @@ -233,6 +233,12 @@ func (r *File) isTemplate() bool { return !r.IsLibrary() && (t == TypeYAML || t == TypeText) } +// IsImplied reports whether this file was implicitly included (found within an explicitly named directory) or not +// (named explicitly as input). +func (r *File) IsImplied() bool { + return strings.ContainsRune(r.OriginalRelativePath(), os.PathSeparator) +} + func (r *File) IsLibrary() bool { exts := strings.Split(filepath.Base(r.RelativePath()), ".") diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 9938622e..0f93abee 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -49,7 +49,7 @@ func TestCheckStdInReadingOnlyOnce(t *testing.T) { {"-f": "-"}, } actualOutput := runYttExpectingError(t, nil, "../../examples/data-values/values-file.yml", flags, nil) - expectedOutput := "ytt: Error: Extracting data value from file:\n Reading file '-':\n Standard input has already been read, has the '-' argument been used in more than one flag?\n" + expectedOutput := "ytt: Error: Extracting data value from file:\n Reading file 'stdin.yml':\n Standard input has already been read, has the '-' argument been used in more than one flag?\n" require.Equal(t, expectedOutput, actualOutput) } From c87dc85475df5e2e48284a146094df4b1ab9efcf Mon Sep 17 00:00:00 2001 From: John Ryan Date: Sat, 21 May 2022 07:11:29 -0700 Subject: [PATCH 2/3] Make end-to-end tests more navigable ... so that as a contributor, when I read this list of tests, I can focus my attention to a section of the test file, rather than having to comprehend the whole thing at once. - group and sort by purpose - feature-specific tests, first - input/output mechanism verification, next - groups of examples-based tests for verifying the examples, last - add the "Overlay" Playground example group (currently untested) - add missing expectations for Overlay examples --- examples/overlay-files/expected.txt | 251 ++++++++ examples/overlay-regular-files/expected.txt | 251 ++++++++ examples/overlay/expected.txt | 251 ++++++++ test/e2e/e2e_test.go | 639 +++++++++----------- 4 files changed, 1048 insertions(+), 344 deletions(-) create mode 100644 examples/overlay-files/expected.txt create mode 100644 examples/overlay-regular-files/expected.txt create mode 100644 examples/overlay/expected.txt diff --git a/examples/overlay-files/expected.txt b/examples/overlay-files/expected.txt new file mode 100644 index 00000000..ac0977e8 --- /dev/null +++ b/examples/overlay-files/expected.txt @@ -0,0 +1,251 @@ +name: bosh +releases: +- name: bosh + version: 268.6.0 + url: https://s3.amazonaws.com/... + sha1: 480b15380f446bcd6fb86511e1ad39b4f1019e37 +- name: bpm + version: 0.12.3 + url: https://s3.amazonaws.com/... + sha1: 54fbf8e2ecf14c69ee761ddde0624edd228ac478 +- name: os-conf + version: 18 + url: https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=18 + sha1: 78d79f08ff5001cc2a24f572837c7a9c59a0e796 +resource_pools: +- name: vms + network: default + env: + bosh: + password: '*' + mbus: + cert: ((mbus_bootstrap_ssl)) +disk_pools: +- name: disks + disk_size: 65536 +networks: +- name: default + type: manual + subnets: + - range: ((internal_cidr)) + gateway: ((internal_gw)) + static: + - ((internal_ip)) + dns: + - 8.8.8.8 +instance_groups: +- name: bosh + instances: 1 + jobs: + - name: bpm + release: bpm + - name: nats + release: bosh + - name: postgres-10 + release: bosh + - name: blobstore + release: bosh + - name: director + release: bosh + - name: health_monitor + release: bosh + - name: user_add + release: os-conf + properties: + users: + - name: jumpbox + public_key: ((jumpbox_ssh.public_key)) + resource_pool: vms + persistent_disk_pool: disks + networks: + - name: default + static_ips: + - ((internal_ip)) + properties: + agent: + mbus: nats://nats:((nats_password))@((internal_ip)):4222 + env: + bosh: + blobstores: + - provider: dav + options: + endpoint: https://((internal_ip)):25250 + user: agent + password: ((blobstore_agent_password)) + tls: + cert: + ca: ((blobstore_ca.certificate)) + nats: + address: ((internal_ip)) + user: nats + password: ((nats_password)) + tls: + ca: ((nats_server_tls.ca)) + client_ca: + certificate: ((nats_ca.certificate)) + private_key: ((nats_ca.private_key)) + server: + certificate: ((nats_server_tls.certificate)) + private_key: ((nats_server_tls.private_key)) + director: + certificate: ((nats_clients_director_tls.certificate)) + private_key: ((nats_clients_director_tls.private_key)) + health_monitor: + certificate: ((nats_clients_health_monitor_tls.certificate)) + private_key: ((nats_clients_health_monitor_tls.private_key)) + postgres: + listen_address: 127.0.0.1 + host: 127.0.0.1 + user: postgres + password: ((postgres_password)) + database: bosh + adapter: postgres + blobstore: + address: ((internal_ip)) + port: 25250 + provider: dav + director: + user: director + password: ((blobstore_director_password)) + agent: + user: agent + password: ((blobstore_agent_password)) + tls: + cert: + ca: ((blobstore_ca.certificate)) + certificate: ((blobstore_server_tls.certificate)) + private_key: ((blobstore_server_tls.private_key)) + director: + address: 127.0.0.1 + name: ((director_name)) + db: + listen_address: 127.0.0.1 + host: 127.0.0.1 + user: postgres + password: ((postgres_password)) + database: bosh + adapter: postgres + flush_arp: true + enable_post_deploy: true + generate_vm_passwords: true + enable_dedicated_status_worker: true + enable_nats_delivered_templates: true + workers: 4 + local_dns: + enabled: true + events: + record_events: true + ssl: + key: ((director_ssl.private_key)) + cert: ((director_ssl.certificate)) + user_management: + provider: local + local: + users: + - name: admin + password: ((admin_password)) + - name: hm + password: ((hm_password)) + default_ssh_options: + gateway_user: jumpbox + hm: + director_account: + user: hm + password: ((hm_password)) + ca_cert: ((director_ssl.ca)) + resurrector_enabled: true + ntp: + - time1.google.com + - time2.google.com + - time3.google.com + - time4.google.com +cloud_provider: + mbus: https://mbus:((mbus_bootstrap_password))@((internal_ip)):6868 + cert: ((mbus_bootstrap_ssl)) + properties: + agent: + mbus: https://mbus:((mbus_bootstrap_password))@0.0.0.0:6868 + blobstore: + provider: local + path: /var/vcap/micro_bosh/data/cache + ntp: + - time1.google.com + - time2.google.com + - time3.google.com + - time4.google.com +variables: +- name: admin_password + type: password +- name: blobstore_director_password + type: password +- name: blobstore_agent_password + type: password +- name: hm_password + type: password +- name: mbus_bootstrap_password + type: password +- name: nats_password + type: password +- name: postgres_password + type: password +- name: default_ca + type: certificate + options: + is_ca: true + common_name: ca +- name: mbus_bootstrap_ssl + type: certificate + options: + ca: default_ca + common_name: ((internal_ip)) + alternative_names: + - ((internal_ip)) +- name: director_ssl + type: certificate + options: + ca: default_ca + common_name: ((internal_ip)) + alternative_names: + - ((internal_ip)) +- name: nats_ca + type: certificate + options: + is_ca: true + common_name: default.nats-ca.bosh-internal +- name: nats_server_tls + type: certificate + options: + ca: nats_ca + common_name: default.nats.bosh-internal + alternative_names: + - ((internal_ip)) + extended_key_usage: + - server_auth +- name: nats_clients_director_tls + type: certificate + options: + ca: nats_ca + common_name: default.director.bosh-internal + extended_key_usage: + - client_auth +- name: nats_clients_health_monitor_tls + type: certificate + options: + ca: nats_ca + common_name: default.hm.bosh-internal + extended_key_usage: + - client_auth +- name: blobstore_ca + type: certificate + options: + is_ca: true + common_name: default.blobstore-ca.bosh-internal +- name: blobstore_server_tls + type: certificate + options: + ca: blobstore_ca + common_name: ((internal_ip)) + alternative_names: + - ((internal_ip)) +- name: jumpbox_ssh + type: ssh diff --git a/examples/overlay-regular-files/expected.txt b/examples/overlay-regular-files/expected.txt new file mode 100644 index 00000000..ac0977e8 --- /dev/null +++ b/examples/overlay-regular-files/expected.txt @@ -0,0 +1,251 @@ +name: bosh +releases: +- name: bosh + version: 268.6.0 + url: https://s3.amazonaws.com/... + sha1: 480b15380f446bcd6fb86511e1ad39b4f1019e37 +- name: bpm + version: 0.12.3 + url: https://s3.amazonaws.com/... + sha1: 54fbf8e2ecf14c69ee761ddde0624edd228ac478 +- name: os-conf + version: 18 + url: https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=18 + sha1: 78d79f08ff5001cc2a24f572837c7a9c59a0e796 +resource_pools: +- name: vms + network: default + env: + bosh: + password: '*' + mbus: + cert: ((mbus_bootstrap_ssl)) +disk_pools: +- name: disks + disk_size: 65536 +networks: +- name: default + type: manual + subnets: + - range: ((internal_cidr)) + gateway: ((internal_gw)) + static: + - ((internal_ip)) + dns: + - 8.8.8.8 +instance_groups: +- name: bosh + instances: 1 + jobs: + - name: bpm + release: bpm + - name: nats + release: bosh + - name: postgres-10 + release: bosh + - name: blobstore + release: bosh + - name: director + release: bosh + - name: health_monitor + release: bosh + - name: user_add + release: os-conf + properties: + users: + - name: jumpbox + public_key: ((jumpbox_ssh.public_key)) + resource_pool: vms + persistent_disk_pool: disks + networks: + - name: default + static_ips: + - ((internal_ip)) + properties: + agent: + mbus: nats://nats:((nats_password))@((internal_ip)):4222 + env: + bosh: + blobstores: + - provider: dav + options: + endpoint: https://((internal_ip)):25250 + user: agent + password: ((blobstore_agent_password)) + tls: + cert: + ca: ((blobstore_ca.certificate)) + nats: + address: ((internal_ip)) + user: nats + password: ((nats_password)) + tls: + ca: ((nats_server_tls.ca)) + client_ca: + certificate: ((nats_ca.certificate)) + private_key: ((nats_ca.private_key)) + server: + certificate: ((nats_server_tls.certificate)) + private_key: ((nats_server_tls.private_key)) + director: + certificate: ((nats_clients_director_tls.certificate)) + private_key: ((nats_clients_director_tls.private_key)) + health_monitor: + certificate: ((nats_clients_health_monitor_tls.certificate)) + private_key: ((nats_clients_health_monitor_tls.private_key)) + postgres: + listen_address: 127.0.0.1 + host: 127.0.0.1 + user: postgres + password: ((postgres_password)) + database: bosh + adapter: postgres + blobstore: + address: ((internal_ip)) + port: 25250 + provider: dav + director: + user: director + password: ((blobstore_director_password)) + agent: + user: agent + password: ((blobstore_agent_password)) + tls: + cert: + ca: ((blobstore_ca.certificate)) + certificate: ((blobstore_server_tls.certificate)) + private_key: ((blobstore_server_tls.private_key)) + director: + address: 127.0.0.1 + name: ((director_name)) + db: + listen_address: 127.0.0.1 + host: 127.0.0.1 + user: postgres + password: ((postgres_password)) + database: bosh + adapter: postgres + flush_arp: true + enable_post_deploy: true + generate_vm_passwords: true + enable_dedicated_status_worker: true + enable_nats_delivered_templates: true + workers: 4 + local_dns: + enabled: true + events: + record_events: true + ssl: + key: ((director_ssl.private_key)) + cert: ((director_ssl.certificate)) + user_management: + provider: local + local: + users: + - name: admin + password: ((admin_password)) + - name: hm + password: ((hm_password)) + default_ssh_options: + gateway_user: jumpbox + hm: + director_account: + user: hm + password: ((hm_password)) + ca_cert: ((director_ssl.ca)) + resurrector_enabled: true + ntp: + - time1.google.com + - time2.google.com + - time3.google.com + - time4.google.com +cloud_provider: + mbus: https://mbus:((mbus_bootstrap_password))@((internal_ip)):6868 + cert: ((mbus_bootstrap_ssl)) + properties: + agent: + mbus: https://mbus:((mbus_bootstrap_password))@0.0.0.0:6868 + blobstore: + provider: local + path: /var/vcap/micro_bosh/data/cache + ntp: + - time1.google.com + - time2.google.com + - time3.google.com + - time4.google.com +variables: +- name: admin_password + type: password +- name: blobstore_director_password + type: password +- name: blobstore_agent_password + type: password +- name: hm_password + type: password +- name: mbus_bootstrap_password + type: password +- name: nats_password + type: password +- name: postgres_password + type: password +- name: default_ca + type: certificate + options: + is_ca: true + common_name: ca +- name: mbus_bootstrap_ssl + type: certificate + options: + ca: default_ca + common_name: ((internal_ip)) + alternative_names: + - ((internal_ip)) +- name: director_ssl + type: certificate + options: + ca: default_ca + common_name: ((internal_ip)) + alternative_names: + - ((internal_ip)) +- name: nats_ca + type: certificate + options: + is_ca: true + common_name: default.nats-ca.bosh-internal +- name: nats_server_tls + type: certificate + options: + ca: nats_ca + common_name: default.nats.bosh-internal + alternative_names: + - ((internal_ip)) + extended_key_usage: + - server_auth +- name: nats_clients_director_tls + type: certificate + options: + ca: nats_ca + common_name: default.director.bosh-internal + extended_key_usage: + - client_auth +- name: nats_clients_health_monitor_tls + type: certificate + options: + ca: nats_ca + common_name: default.hm.bosh-internal + extended_key_usage: + - client_auth +- name: blobstore_ca + type: certificate + options: + is_ca: true + common_name: default.blobstore-ca.bosh-internal +- name: blobstore_server_tls + type: certificate + options: + ca: blobstore_ca + common_name: ((internal_ip)) + alternative_names: + - ((internal_ip)) +- name: jumpbox_ssh + type: ssh diff --git a/examples/overlay/expected.txt b/examples/overlay/expected.txt new file mode 100644 index 00000000..ac0977e8 --- /dev/null +++ b/examples/overlay/expected.txt @@ -0,0 +1,251 @@ +name: bosh +releases: +- name: bosh + version: 268.6.0 + url: https://s3.amazonaws.com/... + sha1: 480b15380f446bcd6fb86511e1ad39b4f1019e37 +- name: bpm + version: 0.12.3 + url: https://s3.amazonaws.com/... + sha1: 54fbf8e2ecf14c69ee761ddde0624edd228ac478 +- name: os-conf + version: 18 + url: https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=18 + sha1: 78d79f08ff5001cc2a24f572837c7a9c59a0e796 +resource_pools: +- name: vms + network: default + env: + bosh: + password: '*' + mbus: + cert: ((mbus_bootstrap_ssl)) +disk_pools: +- name: disks + disk_size: 65536 +networks: +- name: default + type: manual + subnets: + - range: ((internal_cidr)) + gateway: ((internal_gw)) + static: + - ((internal_ip)) + dns: + - 8.8.8.8 +instance_groups: +- name: bosh + instances: 1 + jobs: + - name: bpm + release: bpm + - name: nats + release: bosh + - name: postgres-10 + release: bosh + - name: blobstore + release: bosh + - name: director + release: bosh + - name: health_monitor + release: bosh + - name: user_add + release: os-conf + properties: + users: + - name: jumpbox + public_key: ((jumpbox_ssh.public_key)) + resource_pool: vms + persistent_disk_pool: disks + networks: + - name: default + static_ips: + - ((internal_ip)) + properties: + agent: + mbus: nats://nats:((nats_password))@((internal_ip)):4222 + env: + bosh: + blobstores: + - provider: dav + options: + endpoint: https://((internal_ip)):25250 + user: agent + password: ((blobstore_agent_password)) + tls: + cert: + ca: ((blobstore_ca.certificate)) + nats: + address: ((internal_ip)) + user: nats + password: ((nats_password)) + tls: + ca: ((nats_server_tls.ca)) + client_ca: + certificate: ((nats_ca.certificate)) + private_key: ((nats_ca.private_key)) + server: + certificate: ((nats_server_tls.certificate)) + private_key: ((nats_server_tls.private_key)) + director: + certificate: ((nats_clients_director_tls.certificate)) + private_key: ((nats_clients_director_tls.private_key)) + health_monitor: + certificate: ((nats_clients_health_monitor_tls.certificate)) + private_key: ((nats_clients_health_monitor_tls.private_key)) + postgres: + listen_address: 127.0.0.1 + host: 127.0.0.1 + user: postgres + password: ((postgres_password)) + database: bosh + adapter: postgres + blobstore: + address: ((internal_ip)) + port: 25250 + provider: dav + director: + user: director + password: ((blobstore_director_password)) + agent: + user: agent + password: ((blobstore_agent_password)) + tls: + cert: + ca: ((blobstore_ca.certificate)) + certificate: ((blobstore_server_tls.certificate)) + private_key: ((blobstore_server_tls.private_key)) + director: + address: 127.0.0.1 + name: ((director_name)) + db: + listen_address: 127.0.0.1 + host: 127.0.0.1 + user: postgres + password: ((postgres_password)) + database: bosh + adapter: postgres + flush_arp: true + enable_post_deploy: true + generate_vm_passwords: true + enable_dedicated_status_worker: true + enable_nats_delivered_templates: true + workers: 4 + local_dns: + enabled: true + events: + record_events: true + ssl: + key: ((director_ssl.private_key)) + cert: ((director_ssl.certificate)) + user_management: + provider: local + local: + users: + - name: admin + password: ((admin_password)) + - name: hm + password: ((hm_password)) + default_ssh_options: + gateway_user: jumpbox + hm: + director_account: + user: hm + password: ((hm_password)) + ca_cert: ((director_ssl.ca)) + resurrector_enabled: true + ntp: + - time1.google.com + - time2.google.com + - time3.google.com + - time4.google.com +cloud_provider: + mbus: https://mbus:((mbus_bootstrap_password))@((internal_ip)):6868 + cert: ((mbus_bootstrap_ssl)) + properties: + agent: + mbus: https://mbus:((mbus_bootstrap_password))@0.0.0.0:6868 + blobstore: + provider: local + path: /var/vcap/micro_bosh/data/cache + ntp: + - time1.google.com + - time2.google.com + - time3.google.com + - time4.google.com +variables: +- name: admin_password + type: password +- name: blobstore_director_password + type: password +- name: blobstore_agent_password + type: password +- name: hm_password + type: password +- name: mbus_bootstrap_password + type: password +- name: nats_password + type: password +- name: postgres_password + type: password +- name: default_ca + type: certificate + options: + is_ca: true + common_name: ca +- name: mbus_bootstrap_ssl + type: certificate + options: + ca: default_ca + common_name: ((internal_ip)) + alternative_names: + - ((internal_ip)) +- name: director_ssl + type: certificate + options: + ca: default_ca + common_name: ((internal_ip)) + alternative_names: + - ((internal_ip)) +- name: nats_ca + type: certificate + options: + is_ca: true + common_name: default.nats-ca.bosh-internal +- name: nats_server_tls + type: certificate + options: + ca: nats_ca + common_name: default.nats.bosh-internal + alternative_names: + - ((internal_ip)) + extended_key_usage: + - server_auth +- name: nats_clients_director_tls + type: certificate + options: + ca: nats_ca + common_name: default.director.bosh-internal + extended_key_usage: + - client_auth +- name: nats_clients_health_monitor_tls + type: certificate + options: + ca: nats_ca + common_name: default.hm.bosh-internal + extended_key_usage: + - client_auth +- name: blobstore_ca + type: certificate + options: + is_ca: true + common_name: default.blobstore-ca.bosh-internal +- name: blobstore_server_tls + type: certificate + options: + ca: blobstore_ca + common_name: ((internal_ip)) + alternative_names: + - ((internal_ip)) +- name: jumpbox_ssh + type: ssh diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 0f93abee..977449ad 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,155 +17,167 @@ import ( "github.com/stretchr/testify/require" ) -func TestCheckStdInReading(t *testing.T) { - actualOutput := runYtt(t, []string{"-", "../../examples/eirini/input.yml"}, "../../examples/eirini/config.yml", nil, nil) - - expectedFileOutput, err := ioutil.ReadFile("../../examples/eirini/config-result.yml") - require.NoError(t, err) - require.Equal(t, string(expectedFileOutput), actualOutput) -} - -func TestDataValuesFileWithStdin(t *testing.T) { +func TestDataValues(t *testing.T) { + t.Run("can be set through command-line flags", func(t *testing.T) { + t.Run("--data-value flag", func(t *testing.T) { + flags := yttFlags{ + {"--data-value": "nothing=null"}, + {"--data-value": "string=str"}, + {"--data-value": "bool=true"}, + {"--data-value": "int=123"}, + {"--data-value": "float=123.123"}, + } + actualOutput := runYtt(t, testInputFiles{"../../examples/data-values/config.yml", "../../examples/data-values/values.yml"}, "", flags, nil) + expectedOutput := `nothing: "null" +string: str +bool: "true" +int: "123" +float: "123.123" +` - t.Run("--data-values-file with stdin", func(t *testing.T) { - flags := yttFlags{ - {"--data-values-file": "-"}, - {"--data-values-inspect": ""}, - } - actualOutput := runYtt(t, []string{}, "../../examples/data-values/values-file.yml", flags, nil) - expectedOutput := `nothing: something + require.Equal(t, expectedOutput, actualOutput) + }) + t.Run("--data-value-yaml flag", func(t *testing.T) { + flags := yttFlags{ + {"--data-value-yaml": "nothing=null"}, + {"--data-value-yaml": "string=str"}, + {"--data-value-yaml": "bool=true"}, + {"--data-value-yaml": "int=123"}, + {"--data-value-yaml": "float=123.123"}, + } + actualOutput := runYtt(t, testInputFiles{"../../examples/data-values/config.yml", "../../examples/data-values/values.yml"}, "", flags, nil) + expectedOutput := `nothing: null string: str bool: true -int: 124 -new_thing: new +int: 123 +float: 123.123 ` - require.Equal(t, expectedOutput, actualOutput) - }) -} -func TestCheckStdInReadingOnlyOnce(t *testing.T) { - flags := yttFlags{ - {"--data-values-file": "-"}, - {"-f": "-"}, - } - actualOutput := runYttExpectingError(t, nil, "../../examples/data-values/values-file.yml", flags, nil) - expectedOutput := "ytt: Error: Extracting data value from file:\n Reading file 'stdin.yml':\n Standard input has already been read, has the '-' argument been used in more than one flag?\n" - require.Equal(t, expectedOutput, actualOutput) -} + require.Equal(t, expectedOutput, actualOutput) + }) + t.Run("--data-values-env flag", func(t *testing.T) { + flags := yttFlags{ + {"--data-values-env": "STR_VAL"}, + {"--data-values-env-yaml": "YAML_VAL"}, + } + envs := []string{ + "STR_VAL_nothing=null", + "YAML_VAL_string=str", + "YAML_VAL_bool=true", + "YAML_VAL_int=123", + "YAML_VAL_float=123.123", + } + actualOutput := runYtt(t, testInputFiles{"../../examples/data-values/config.yml", "../../examples/data-values/values.yml"}, "", flags, envs) + expectedOutput := `nothing: "null" +string: str +bool: true +int: 123 +float: 123.123 +` -func TestSanityCheckTemplateWithDataValues(t *testing.T) { - t.Run("template file with data value", func(t *testing.T) { - actualOutput := runYtt(t, []string{"../../examples/eirini/config.yml", "../../examples/eirini/input.yml"}, "", nil, nil) + require.Equal(t, expectedOutput, actualOutput) + }) + t.Run("--data-value-yaml && --data-values-env-yaml flag", func(t *testing.T) { + flags := yttFlags{ + {"--data-value-yaml": "nothing=[1,2,3]"}, + {"--data-values-env-yaml": "YAML_VAL"}, + } + envs := []string{ + "YAML_VAL_string=[1,2,4]", + "YAML_VAL_bool=true", + "YAML_VAL_int=123", + "YAML_VAL_float=123.123", + } + actualOutput := runYtt(t, testInputFiles{"../../examples/data-values/config.yml", "../../examples/data-values/values.yml"}, "", flags, envs) + expectedOutput := `nothing: +- 1 +- 2 +- 3 +string: +- 1 +- 2 +- 4 +bool: true +int: 123 +float: 123.123 +` - expectedFileOutput, err := ioutil.ReadFile("../../examples/eirini/config-result.yml") - require.NoError(t, err) - require.Equal(t, string(expectedFileOutput), actualOutput) + require.Equal(t, expectedOutput, actualOutput) + }) }) - - t.Run("another template file with data value", func(t *testing.T) { - actualOutput := runYtt(t, []string{"../../examples/eirini/config-alt1.yml", "../../examples/eirini/input.yml"}, "", nil, nil) - - expectedFileOutput, err := ioutil.ReadFile("../../examples/eirini/config-result.yml") + t.Run("can be 'required'", func(t *testing.T) { + expectedFileOutput, err := ioutil.ReadFile("../../examples/data-values-required/expected.txt") require.NoError(t, err) - require.Equal(t, string(expectedFileOutput), actualOutput) - }) -} - -func TestCheckDirectoryReading(t *testing.T) { - tempOutputDir, err := ioutil.TempDir(os.TempDir(), "ytt-check-dir") - require.NoError(t, err) - defer os.Remove(tempOutputDir) - flags := yttFlags{{fmt.Sprintf("--dangerous-emptied-output-directory=%s", tempOutputDir): ""}} - runYtt(t, []string{"../../examples/eirini/"}, "", flags, nil) - expectedFileOutput, err := ioutil.ReadFile("../../examples/eirini/config-result.yml") - require.NoError(t, err) - - actualOutput, err := ioutil.ReadFile(filepath.Join(tempOutputDir, "config-result.yml")) - require.NoError(t, err) - - require.Equal(t, string(expectedFileOutput), string(actualOutput)) -} - -func TestPlaygroundExamples(t *testing.T) { - - filepath.WalkDir("../../examples/playground/basics", func(path string, d fs.DirEntry, err error) error { - if !d.IsDir() { - return filepath.SkipDir + dirs := []string{ + "data-values-required/inline", + "data-values-required/function", + "data-values-required/bulk", } - - switch d.Name() { - case "example-assert", "example-load-custom-library-module", "example-ytt-library-module": - return filepath.SkipDir - case "basics": - return nil - default: - t.Run(fmt.Sprintf("playground %s", d.Name()), func(t *testing.T) { - runYtt(t, testInputFiles{path}, "", nil, nil) + for _, dir := range dirs { + t.Run(dir, func(t *testing.T) { + dirPath := fmt.Sprintf("../../examples/%s", dir) + flags := yttFlags{ + {"-v": "version=123"}, + } + actualOutput := runYtt(t, testInputFiles{dirPath}, "", flags, nil) + + require.Equal(t, string(expectedFileOutput), actualOutput) }) - return nil } }) -} - -func TestOverlays(t *testing.T) { - t.Run(fmt.Sprintf("overlay ../../examples/overlay"), func(t *testing.T) { - runYtt(t, testInputFiles{"../../examples/overlay"}, "", nil, nil) - }) - - t.Run(fmt.Sprintf("overlay ../../examples/overlay-files"), func(t *testing.T) { - runYtt(t, testInputFiles{"../../examples/overlay-files"}, "", nil, nil) - }) + t.Run("example supporting multi-environment scenarios", func(t *testing.T) { + t.Run("with defaults", func(t *testing.T) { + flags := yttFlags{ + {"-v": "version=123"}, + } + actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-multiple-envs/config/"}, "", flags, nil) - t.Run(fmt.Sprintf("overlay ../../examples/overlay-regular-files"), func(t *testing.T) { - runYtt(t, testInputFiles{"../../examples/overlay"}, "", yttFlags{{"--file-mark": "file.yml:type=yaml-plain"}}, nil) + expectedOutput := `app_config: + version: "123" + ports: + - 8080 +` + require.Equal(t, expectedOutput, actualOutput) + }) + t.Run("with 'dev' overrides", func(t *testing.T) { + actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-multiple-envs/config/", "../../examples/data-values-multiple-envs/envs/dev.yml"}, "", nil, nil) + expectedOutput := `app_config: + version: v1alpha1 + ports: + - 8080 +` + require.Equal(t, expectedOutput, actualOutput) + }) + t.Run("with 'staging' overrides", func(t *testing.T) { + actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-multiple-envs/config/", "../../examples/data-values-multiple-envs/envs/staging.yml"}, "", nil, nil) + expectedOutput := `app_config: + version: v1beta1 + ports: + - 8081 +` + require.Equal(t, expectedOutput, actualOutput) + }) + t.Run("with 'prod' overrides", func(t *testing.T) { + actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-multiple-envs/config/", "../../examples/data-values-multiple-envs/envs/prod.yml"}, "", nil, nil) + expectedOutput := `app_config: + version: v1 + ports: + - 80 +` + require.Equal(t, expectedOutput, actualOutput) + }) }) } -func TestDifferentUsages(t *testing.T) { +func TestSchema(t *testing.T) { dirs := []string{ - "k8s-add-global-label", - "k8s-adjust-rbac-version", - "k8s-docker-secret", - "k8s-relative-rolling-update", - "k8s-config-map-files", - "k8s-update-env-var", - "k8s-overlay-all-containers", - "k8s-overlay-remove-resources", - "k8s-overlay-in-config-map", - // test that @ytt:toml module works in default ytt binary - // as it is loaded in a different way than other @ytt:* modules. - "toml-serialize", - } - for _, k8sDirToTest := range dirs { - t.Run(fmt.Sprintf("k8s: %s", k8sDirToTest), func(t *testing.T) { - dirPath := fmt.Sprintf("../../examples/%s", k8sDirToTest) - actualOutput := runYtt(t, testInputFiles{dirPath}, "", nil, nil) - - expectedOutput, err := ioutil.ReadFile(filepath.Join(dirPath, "expected.txt")) - require.NoError(t, err) - - require.Equal(t, string(expectedOutput), actualOutput) - }) - } - - dirs = []string{"concourse-overlay"} - for _, concourseDirToTest := range dirs { - t.Run(fmt.Sprintf("concourse: %s", concourseDirToTest), func(t *testing.T) { - dirPath := fmt.Sprintf("../../examples/%s", concourseDirToTest) - actualOutput := runYtt(t, testInputFiles{dirPath}, "", nil, nil) - - expectedOutput, err := ioutil.ReadFile(filepath.Join(dirPath, "expected.txt")) - require.NoError(t, err) - - require.Equal(t, string(expectedOutput), actualOutput) - }) + "schema", + "schema-arrays", } - - dirs = []string{"overlay-not-matcher"} - for _, overlayDirToTest := range dirs { - t.Run(fmt.Sprintf("overlay-not-matcher: %s", overlayDirToTest), func(t *testing.T) { - dirPath := fmt.Sprintf("../../examples/%s", overlayDirToTest) + for _, dir := range dirs { + t.Run(dir, func(t *testing.T) { + dirPath := fmt.Sprintf("../../examples/%s", dir) actualOutput := runYtt(t, testInputFiles{dirPath}, "", nil, nil) expectedOutput, err := ioutil.ReadFile(filepath.Join(dirPath, "expected.txt")) @@ -174,14 +186,18 @@ func TestDifferentUsages(t *testing.T) { require.Equal(t, string(expectedOutput), actualOutput) }) } +} - dirs = []string{ - "schema", - "schema-arrays", +func TestOverlays(t *testing.T) { + dirs := []string{ + "overlay", + "overlay-files", + "overlay-not-matcher", + "overlay-regular-files", } - for _, schemaDirToTest := range dirs { - t.Run(fmt.Sprintf("schema: %s", schemaDirToTest), func(t *testing.T) { - dirPath := fmt.Sprintf("../../examples/%s", schemaDirToTest) + for _, dir := range dirs { + t.Run(dir, func(t *testing.T) { + dirPath := fmt.Sprintf("../../examples/%s", dir) actualOutput := runYtt(t, testInputFiles{dirPath}, "", nil, nil) expectedOutput, err := ioutil.ReadFile(filepath.Join(dirPath, "expected.txt")) @@ -192,11 +208,8 @@ func TestDifferentUsages(t *testing.T) { } } -func TestJsonOutput(t *testing.T) { - actualOutput := runYtt(t, testInputFiles{"../../examples/k8s-adjust-rbac-version"}, "", yttFlags{{"-o": "json"}}, nil) - expectedOutput := `{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole"}{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole"}` - - require.Equal(t, expectedOutput, actualOutput) +func TestVersionIsValid(t *testing.T) { + runYtt(t, testInputFiles{"../../examples/version-constraint"}, "", yttFlags{}, nil) } func TestPipes(t *testing.T) { @@ -215,249 +228,186 @@ func TestPipes(t *testing.T) { }) } -func TestDataValues(t *testing.T) { - t.Run("--data-value flag", func(t *testing.T) { - flags := yttFlags{ - {"--data-value": "nothing=null"}, - {"--data-value": "string=str"}, - {"--data-value": "bool=true"}, - {"--data-value": "int=123"}, - {"--data-value": "float=123.123"}, - } - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values/config.yml", "../../examples/data-values/values.yml"}, "", flags, nil) - expectedOutput := `nothing: "null" -string: str -bool: "true" -int: "123" -float: "123.123" -` - - require.Equal(t, expectedOutput, actualOutput) - }) - - t.Run("--data-value-yaml flag", func(t *testing.T) { - flags := yttFlags{ - {"--data-value-yaml": "nothing=null"}, - {"--data-value-yaml": "string=str"}, - {"--data-value-yaml": "bool=true"}, - {"--data-value-yaml": "int=123"}, - {"--data-value-yaml": "float=123.123"}, - } - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values/config.yml", "../../examples/data-values/values.yml"}, "", flags, nil) - expectedOutput := `nothing: null -string: str -bool: true -int: 123 -float: 123.123 -` +func TestReadingFromStandardIn(t *testing.T) { + t.Run("through --file", func(t *testing.T) { + actualOutput := runYtt(t, []string{"-", "../../examples/eirini/input.yml"}, "../../examples/eirini/config.yml", nil, nil) - require.Equal(t, expectedOutput, actualOutput) + expectedFileOutput, err := ioutil.ReadFile("../../examples/eirini/config-result.yml") + require.NoError(t, err) + require.Equal(t, string(expectedFileOutput), actualOutput) }) - - t.Run("--data-values-env flag", func(t *testing.T) { + t.Run("through --data-values-file", func(t *testing.T) { flags := yttFlags{ - {"--data-values-env": "STR_VAL"}, - {"--data-values-env-yaml": "YAML_VAL"}, - } - envs := []string{ - "STR_VAL_nothing=null", - "YAML_VAL_string=str", - "YAML_VAL_bool=true", - "YAML_VAL_int=123", - "YAML_VAL_float=123.123", + {"--data-values-file": "-"}, + {"--data-values-inspect": ""}, } - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values/config.yml", "../../examples/data-values/values.yml"}, "", flags, envs) - expectedOutput := `nothing: "null" + actualOutput := runYtt(t, []string{}, "../../examples/data-values/values-file.yml", flags, nil) + expectedOutput := `nothing: something string: str bool: true -int: 123 -float: 123.123 +int: 124 +new_thing: new ` - require.Equal(t, expectedOutput, actualOutput) }) - - t.Run("--data-value-yaml && --data-values-env-yaml flag", func(t *testing.T) { + t.Run("can only be read once", func(t *testing.T) { flags := yttFlags{ - {"--data-value-yaml": "nothing=[1,2,3]"}, - {"--data-values-env-yaml": "YAML_VAL"}, - } - envs := []string{ - "YAML_VAL_string=[1,2,4]", - "YAML_VAL_bool=true", - "YAML_VAL_int=123", - "YAML_VAL_float=123.123", + {"--data-values-file": "-"}, + {"-f": "-"}, } - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values/config.yml", "../../examples/data-values/values.yml"}, "", flags, envs) - expectedOutput := `nothing: -- 1 -- 2 -- 3 -string: -- 1 -- 2 -- 4 -bool: true -int: 123 -float: 123.123 -` - + actualOutput := runYttExpectingError(t, nil, "../../examples/data-values/values-file.yml", flags, nil) + expectedOutput := "ytt: Error: Extracting data value from file:\n Reading file 'stdin.yml':\n Standard input has already been read, has the '-' argument been used in more than one flag?\n" require.Equal(t, expectedOutput, actualOutput) }) } -func TestSchema(t *testing.T) { - t.Run("--data-value flag", func(t *testing.T) { - flags := yttFlags{ - {"--data-value": "nothing=a new string"}, - {"--data-value": "string=str"}, - } - - actualOutput := runYtt(t, testInputFiles{"../../examples/schema/config.yml", "../../examples/schema/schema.yml"}, "", flags, nil) - expectedOutput := `nothing: a new string -string: str -bool: false -int: 0 -float: 0.1 -any: anything -` +func TestCheckDirectoryOutput(t *testing.T) { + tempOutputDir, err := ioutil.TempDir(os.TempDir(), "ytt-check-dir") + require.NoError(t, err) + defer os.Remove(tempOutputDir) + flags := yttFlags{{fmt.Sprintf("--dangerous-emptied-output-directory=%s", tempOutputDir): ""}} + runYtt(t, []string{"../../examples/eirini/"}, "", flags, nil) - require.Equal(t, expectedOutput, actualOutput) - }) + expectedFileOutput, err := ioutil.ReadFile("../../examples/eirini/config-result.yml") + require.NoError(t, err) - t.Run("--data-value-yaml flag", func(t *testing.T) { - flags := yttFlags{ - {"--data-value-yaml": "nothing=a new string"}, - {"--data-value-yaml": "string=str"}, - {"--data-value-yaml": "bool=true"}, - {"--data-value-yaml": "int=123"}, - {"--data-value-yaml": "float=123.123"}, - {"--data-value-yaml": "any=[1,2,4]"}, - } - actualOutput := runYtt(t, testInputFiles{"../../examples/schema/config.yml", "../../examples/schema/schema.yml"}, "", flags, nil) - expectedOutput := `nothing: a new string -string: str -bool: true -int: 123 -float: 123.123 -any: -- 1 -- 2 -- 4 -` + actualOutput, err := ioutil.ReadFile(filepath.Join(tempOutputDir, "config-result.yml")) + require.NoError(t, err) - require.Equal(t, expectedOutput, actualOutput) - }) + require.Equal(t, string(expectedFileOutput), string(actualOutput)) +} - t.Run("--data-values-env && --data-values-env-yaml flag", func(t *testing.T) { - flags := yttFlags{ - {"--data-values-env": "STR_VAL"}, - {"--data-values-env-yaml": "YAML_VAL"}, - } - envs := []string{ - "STR_VAL_nothing=a new string", - "YAML_VAL_string=str", - "YAML_VAL_bool=true", - "YAML_VAL_int=123", - "YAML_VAL_float=123.123", - "YAML_VAL_any=[1,2,4]", - } - actualOutput := runYtt(t, testInputFiles{"../../examples/schema/config.yml", "../../examples/schema/schema.yml"}, "", flags, envs) - expectedOutput := `nothing: a new string -string: str -bool: true -int: 123 -float: 123.123 -any: -- 1 -- 2 -- 4 -` +func TestJsonOutput(t *testing.T) { + actualOutput := runYtt(t, testInputFiles{"../../examples/k8s-adjust-rbac-version"}, "", yttFlags{{"-o": "json"}}, nil) + expectedOutput := `{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole"}{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole"}` - require.Equal(t, expectedOutput, actualOutput) - }) + require.Equal(t, expectedOutput, actualOutput) } -func TestDataValuesRequired(t *testing.T) { - expectedFileOutput, err := ioutil.ReadFile("../../examples/data-values-required/expected.txt") - require.NoError(t, err) - - t.Run("inline", func(t *testing.T) { - flags := yttFlags{ - {"-v": "version=123"}, - } - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-required/inline"}, "", flags, nil) +func TestPlaygroundExamplesExecuteWithoutError(t *testing.T) { + t.Run("Basics", func(t *testing.T) { + filepath.WalkDir("../../examples/playground/basics", func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + return filepath.SkipDir + } - require.Equal(t, string(expectedFileOutput), actualOutput) + switch d.Name() { + case "example-assert", "example-load-custom-library-module", "example-ytt-library-module": + return filepath.SkipDir + case "basics": + return nil + default: + t.Run(fmt.Sprintf("playground %s", d.Name()), func(t *testing.T) { + runYtt(t, testInputFiles{path}, "", nil, nil) + }) + return nil + } + }) }) + t.Run("Overlays", func(t *testing.T) { + filepath.WalkDir("../../examples/playground/overlays", func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + return filepath.SkipDir + } - t.Run("function", func(t *testing.T) { - flags := yttFlags{ - {"-v": "version=123"}, - } - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-required/function"}, "", flags, nil) - - require.Equal(t, string(expectedFileOutput), actualOutput) + if d.Name() == "overlays" { + return nil + } + t.Run(fmt.Sprintf("playground %s", d.Name()), func(t *testing.T) { + runYtt(t, testInputFiles{path}, "", nil, nil) + }) + return nil + }) }) +} - t.Run("bulk", func(t *testing.T) { - flags := yttFlags{ - {"-v": "version=123"}, +func TestApplicationSpecificScenarios(t *testing.T) { + t.Run("Kubernetes", func(t *testing.T) { + dirs := []string{ + "k8s-add-global-label", + "k8s-adjust-rbac-version", + "k8s-docker-secret", + "k8s-relative-rolling-update", + "k8s-config-map-files", + "k8s-update-env-var", + "k8s-overlay-all-containers", + "k8s-overlay-remove-resources", + "k8s-overlay-in-config-map", } - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-required/bulk"}, "", flags, nil) + for _, dir := range dirs { + t.Run(dir, func(t *testing.T) { + dirPath := fmt.Sprintf("../../examples/%s", dir) + actualOutput := runYtt(t, testInputFiles{dirPath}, "", nil, nil) - require.Equal(t, string(expectedFileOutput), actualOutput) - }) -} + expectedOutput, err := ioutil.ReadFile(filepath.Join(dirPath, "expected.txt")) + require.NoError(t, err) -func TestDataValuesUsages(t *testing.T) { - t.Run("multiple envs", func(t *testing.T) { - flags := yttFlags{ - {"-v": "version=123"}, + require.Equal(t, string(expectedOutput), actualOutput) + }) } - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-multiple-envs/config/"}, "", flags, nil) - - expectedOutput := `app_config: - version: "123" - ports: - - 8080 -` - require.Equal(t, expectedOutput, actualOutput) }) + t.Run("Concourse", func(t *testing.T) { + dirs := []string{"concourse-overlay"} + for _, dir := range dirs { + t.Run(dir, func(t *testing.T) { + dirPath := fmt.Sprintf("../../examples/%s", dir) + actualOutput := runYtt(t, testInputFiles{dirPath}, "", nil, nil) - t.Run("when using data values in dev", func(t *testing.T) { - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-multiple-envs/config/", "../../examples/data-values-multiple-envs/envs/dev.yml"}, "", nil, nil) - expectedOutput := `app_config: - version: v1alpha1 - ports: - - 8080 -` - require.Equal(t, expectedOutput, actualOutput) - }) + expectedOutput, err := ioutil.ReadFile(filepath.Join(dirPath, "expected.txt")) + require.NoError(t, err) - t.Run("when using data values in staging", func(t *testing.T) { - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-multiple-envs/config/", "../../examples/data-values-multiple-envs/envs/staging.yml"}, "", nil, nil) - expectedOutput := `app_config: - version: v1beta1 - ports: - - 8081 -` - require.Equal(t, expectedOutput, actualOutput) + require.Equal(t, string(expectedOutput), actualOutput) + }) + } }) + t.Run("Eirini", func(t *testing.T) { + t.Run("config.yml", func(t *testing.T) { + actualOutput := runYtt(t, []string{"../../examples/eirini/config.yml", "../../examples/eirini/input.yml"}, "", nil, nil) - t.Run("when using data values in prod", func(t *testing.T) { - actualOutput := runYtt(t, testInputFiles{"../../examples/data-values-multiple-envs/config/", "../../examples/data-values-multiple-envs/envs/prod.yml"}, "", nil, nil) - expectedOutput := `app_config: - version: v1 - ports: - - 80 -` - require.Equal(t, expectedOutput, actualOutput) + expectedFileOutput, err := ioutil.ReadFile("../../examples/eirini/config-result.yml") + require.NoError(t, err) + require.Equal(t, string(expectedFileOutput), actualOutput) + }) + t.Run("config-alt1.yml", func(t *testing.T) { + actualOutput := runYtt(t, []string{"../../examples/eirini/config-alt1.yml", "../../examples/eirini/input.yml"}, "", nil, nil) + + expectedFileOutput, err := ioutil.ReadFile("../../examples/eirini/config-result.yml") + require.NoError(t, err) + require.Equal(t, string(expectedFileOutput), actualOutput) + }) + t.Run("config-alt2.yml and friends", func(t *testing.T) { + actualOutput := runYtt(t, []string{"../../examples/eirini/config-alt2.yml", + "../../examples/eirini/config.lib.yaml", + "../../examples/eirini/config.star", + "../../examples/eirini/data.txt", + "../../examples/eirini/data2.lib.txt", + "../../examples/eirini/input.yml"}, + "", nil, nil) + + expectedFileOutput, err := ioutil.ReadFile("../../examples/eirini/config-result.yml") + require.NoError(t, err) + require.Equal(t, string(expectedFileOutput), actualOutput) + }) }) } -func TestVersionIsValid(t *testing.T) { - runYtt(t, testInputFiles{"../../examples/version-constraint"}, "", yttFlags{}, nil) +func TestRemainingExamples(t *testing.T) { + dirs := []string{ + // test that @ytt:toml module works in default ytt binary + // as it is loaded in a different way than other @ytt:* modules. + "toml-serialize", + } + for _, dir := range dirs { + t.Run(dir, func(t *testing.T) { + dirPath := fmt.Sprintf("../../examples/%s", dir) + actualOutput := runYtt(t, testInputFiles{dirPath}, "", nil, nil) + + expectedOutput, err := ioutil.ReadFile(filepath.Join(dirPath, "expected.txt")) + require.NoError(t, err) + + require.Equal(t, string(expectedOutput), actualOutput) + }) + } } type testInputFiles []string @@ -478,6 +428,7 @@ func runYtt(t *testing.T, files testInputFiles, stdinFileName string, flags yttF return string(output) } + func runYttExpectingError(t *testing.T, files testInputFiles, stdinFileName string, flags yttFlags, envs []string) string { command, stdError := buildCommand(files, flags, envs) From 5d603373c2fa3160e81a676e451d230cae54def7 Mon Sep 17 00:00:00 2001 From: John Ryan Date: Wed, 18 May 2022 14:53:39 -0700 Subject: [PATCH 3/3] Add E2E for data-values-file reading a dir --- examples/data-values-directory/README.md | 23 +++++++++++++++++++ .../data-values-directory/config/config.yml | 8 +++++++ .../config/values-schema.yml | 10 ++++++++ examples/data-values-directory/run.sh | 6 +++++ .../values/appdev-overrides/values.yaml | 8 +++++++ .../50-operations-overrides.yml | 10 ++++++++ .../99-opssec-overrides.yml | 3 +++ .../values/operator-overrides/approvals.toml | 7 ++++++ test/e2e/e2e_test.go | 12 ++++++++++ 9 files changed, 87 insertions(+) create mode 100644 examples/data-values-directory/README.md create mode 100644 examples/data-values-directory/config/config.yml create mode 100644 examples/data-values-directory/config/values-schema.yml create mode 100755 examples/data-values-directory/run.sh create mode 100644 examples/data-values-directory/values/appdev-overrides/values.yaml create mode 100644 examples/data-values-directory/values/operator-overrides/50-operations-overrides.yml create mode 100644 examples/data-values-directory/values/operator-overrides/99-opssec-overrides.yml create mode 100644 examples/data-values-directory/values/operator-overrides/approvals.toml diff --git a/examples/data-values-directory/README.md b/examples/data-values-directory/README.md new file mode 100644 index 00000000..0167b7eb --- /dev/null +++ b/examples/data-values-directory/README.md @@ -0,0 +1,23 @@ +This example demonstrates how ytt handles specifying a directory for Data Values +inputs. (originally https://kubernetes.slack.com/archives/CH8KCCKA5/p1651167583289939) + +With this: +``` +├── config +│ ├── config.yml +│ └── values-schema.yml +└── values <-- any YAML under here is assumed to be plain Data Values + ├── appdev-overrides <-- sorted by full pathname, alphabetically + │ └── values.yaml + └── operator-overrides + ├── 50-operations-overrides.yml + ├── 99-opssec-overrides.yml + └── approvals.toml <-- non YAML files are ignored. +``` + +Executing this: + +```console +$ ytt -f examples/data-values-directory/config.yml \ + --data-values-file examples/data-values-directory/values/ +``` \ No newline at end of file diff --git a/examples/data-values-directory/config/config.yml b/examples/data-values-directory/config/config.yml new file mode 100644 index 00000000..8e881602 --- /dev/null +++ b/examples/data-values-directory/config/config.yml @@ -0,0 +1,8 @@ +#! In the real-world, this configuration would be the manifests that describe +#! this application's deployment. For this example, we simply render the +#! net values; this is the point of this example. + +#@ load("@ytt:data", "data") + +--- +values: #@ data.values \ No newline at end of file diff --git a/examples/data-values-directory/config/values-schema.yml b/examples/data-values-directory/config/values-schema.yml new file mode 100644 index 00000000..47054c7a --- /dev/null +++ b/examples/data-values-directory/config/values-schema.yml @@ -0,0 +1,10 @@ +#@data/values-schema +--- +name: suggestion-service +instances: 0 +accept_insecure_conns: true +#@schema/nullable +cache: + driver: "" + #@schema/type any=True + config: {} \ No newline at end of file diff --git a/examples/data-values-directory/run.sh b/examples/data-values-directory/run.sh new file mode 100755 index 00000000..c33acb13 --- /dev/null +++ b/examples/data-values-directory/run.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +./ytt -f examples/data-values-directory/config/ \ + --data-values-file examples/data-values-directory/values/ diff --git a/examples/data-values-directory/values/appdev-overrides/values.yaml b/examples/data-values-directory/values/appdev-overrides/values.yaml new file mode 100644 index 00000000..659806f3 --- /dev/null +++ b/examples/data-values-directory/values/appdev-overrides/values.yaml @@ -0,0 +1,8 @@ +--- +instances: 1 +accept_insecure_conns: true +cache: + driver: in-memory + config: + maxEntries: 1024 + strategy: MRU \ No newline at end of file diff --git a/examples/data-values-directory/values/operator-overrides/50-operations-overrides.yml b/examples/data-values-directory/values/operator-overrides/50-operations-overrides.yml new file mode 100644 index 00000000..f4b08191 --- /dev/null +++ b/examples/data-values-directory/values/operator-overrides/50-operations-overrides.yml @@ -0,0 +1,10 @@ +--- +# 2022-05-13: avg + rms over last 3 months +instances: 4 + +cache: + driver: redis + config: + host: localhost:6379 + tls-client-cert-file: client.crt + tls-client-key-file: client.key \ No newline at end of file diff --git a/examples/data-values-directory/values/operator-overrides/99-opssec-overrides.yml b/examples/data-values-directory/values/operator-overrides/99-opssec-overrides.yml new file mode 100644 index 00000000..68cdf259 --- /dev/null +++ b/examples/data-values-directory/values/operator-overrides/99-opssec-overrides.yml @@ -0,0 +1,3 @@ +--- +# Policy 1.31a: all north-south connections must be encrypted. +accept_insecure_conns: false diff --git a/examples/data-values-directory/values/operator-overrides/approvals.toml b/examples/data-values-directory/values/operator-overrides/approvals.toml new file mode 100644 index 00000000..b751a5c8 --- /dev/null +++ b/examples/data-values-directory/values/operator-overrides/approvals.toml @@ -0,0 +1,7 @@ +[[approvals]] + dept = "operations" + user = "grumpy-grey-beard@example.com" + +[[approvals]] + dept = "Operations Security" + user = "pedantic-protector@example.com" diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 977449ad..991f0064 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -104,6 +104,18 @@ float: 123.123 require.Equal(t, expectedOutput, actualOutput) }) + t.Run("--data-values-file enumerates YAML files from given directory", func(t *testing.T) { + testDir := "../../examples/data-values-directory" + flags := yttFlags{ + {"--data-values-file": testDir + "/values"}, + } + actualOutput := runYtt(t, testInputFiles{testDir + "/config"}, "", flags, nil) + + expectedOutput, err := ioutil.ReadFile(filepath.Join(testDir, "expected.txt")) + require.NoError(t, err) + + require.Equal(t, string(expectedOutput), actualOutput) + }) }) t.Run("can be 'required'", func(t *testing.T) { expectedFileOutput, err := ioutil.ReadFile("../../examples/data-values-required/expected.txt")