From 9e83f8f03a6833817dc77ea2686790a46220832a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 15:38:59 +0000 Subject: [PATCH 01/87] early WIP have an example of a yaml file being read and validated --- go.mod | 7 +- go.sum | 4 + src/cmd/prepare.go | 14 +++ src/pkg/utils/helpers/example-schema.json | 31 +++++++ src/pkg/utils/helpers/example.yaml | 6 ++ src/pkg/utils/helpers/validate_schema.go | 88 +++++++++++++++++++ src/pkg/utils/helpers/validate_schema_test.go | 28 ++++++ zarf.schema.json | 3 +- 8 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 src/pkg/utils/helpers/example-schema.json create mode 100644 src/pkg/utils/helpers/example.yaml create mode 100644 src/pkg/utils/helpers/validate_schema.go create mode 100644 src/pkg/utils/helpers/validate_schema_test.go diff --git a/go.mod b/go.mod index fb60c15555..a47cd7d54e 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 + github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.13.0 golang.org/x/sync v0.3.0 golang.org/x/term v0.12.0 @@ -58,6 +59,8 @@ require ( sigs.k8s.io/yaml v1.3.0 ) +require github.com/santhosh-tekuri/jsonschema v1.2.4 + require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect @@ -367,6 +370,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sassoftware/go-rpmutils v0.2.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect @@ -411,7 +415,6 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -452,7 +455,7 @@ require ( gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v3 v3.0.1 inet.af/netaddr v0.0.0-20220811202034-502d2d690317 // indirect k8s.io/apiextensions-apiserver v0.28.2 // indirect k8s.io/apiserver v0.28.2 // indirect diff --git a/go.sum b/go.sum index 7032a61d88..03cea811c7 100644 --- a/go.sum +++ b/go.sum @@ -1412,6 +1412,10 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index f53dcdcae6..a4613ba8e4 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -7,6 +7,7 @@ package cmd import ( "fmt" "io" + "log" "os" "strings" @@ -20,6 +21,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/spf13/cobra" + "github.com/xeipuuv/gojsonschema" + "sigs.k8s.io/yaml" ) var prepareCmd = &cobra.Command{ @@ -157,6 +160,16 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } +var prepareValidateSchema = &cobra.Command{ + Use: "lint [ FILENAME ] [ FILENAME ]", + Args: cobra.MaximumNArgs(2), + Short: lang.CmdPrepareGenerateConfigShort, + Long: lang.CmdPrepareGenerateConfigLong, + Run: func(cmd *cobra.Command, args []string) { + + }, +} + func init() { v := common.InitViper() @@ -165,6 +178,7 @@ func init() { prepareCmd.AddCommand(prepareComputeFileSha256sum) prepareCmd.AddCommand(prepareFindImages) prepareCmd.AddCommand(prepareGenerateConfigFile) + prepareCmd.AddCommand(prepareValidateSchema) prepareFindImages.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables diff --git a/src/pkg/utils/helpers/example-schema.json b/src/pkg/utils/helpers/example-schema.json new file mode 100644 index 0000000000..513869f972 --- /dev/null +++ b/src/pkg/utils/helpers/example-schema.json @@ -0,0 +1,31 @@ +{ + "$id": "https://example.com/product.schema.json", + "title": "Product", + "description": "A product from Acme's catalog", + "type": "object", + "properties": { + "productId": { + "description": "The unique identifier for a product", + "type": "integer" + }, + "productName": { + "description": "Name of the product", + "type": "string" + }, + "price": { + "description": "The price of the product", + "type": "number", + "exclusiveMinimum": 0 + }, + "tags": { + "description": "Tags for the product", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ "productId", "productName", "price" ] +} diff --git a/src/pkg/utils/helpers/example.yaml b/src/pkg/utils/helpers/example.yaml new file mode 100644 index 0000000000..5d19224c8c --- /dev/null +++ b/src/pkg/utils/helpers/example.yaml @@ -0,0 +1,6 @@ +productId: 1 +productName: A green door +price: 12.50 +tags: +- home +- green diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go new file mode 100644 index 0000000000..a59364df8f --- /dev/null +++ b/src/pkg/utils/helpers/validate_schema.go @@ -0,0 +1,88 @@ +package helpers + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "log" + "os" + + "github.com/santhosh-tekuri/jsonschema" + "gopkg.in/yaml.v3" +) + +func ReadSchema(yamlFile, jsonFile string) bool { + + yamlBytes, err := os.ReadFile(yamlFile) + if err != nil { + log.Fatalf("Error reading YAML file: %s", yamlFile) + } + + jsonBytes, err := os.ReadFile(jsonFile) + if err != nil { + log.Fatalf("Error reading YAML file: %s", jsonFile) + } + + var m interface{} + err = yaml.Unmarshal(yamlBytes, &m) + if err != nil { + panic(err) + } + m, err = toStringKeys(m) + if err != nil { + panic(err) + } + + mJSON, err := json.Marshal(m) + if err != nil { + panic(err) + } + yamlJsonReader := bytes.NewReader(mJSON) + schemaReader := bytes.NewReader(jsonBytes) + + compiler := jsonschema.NewCompiler() + + if err := compiler.AddResource("schema.json", schemaReader); err != nil { + panic(err) + } + schema, err := compiler.Compile("schema.json") + if err != nil { + panic(err) + } + if err := schema.Validate(yamlJsonReader); err != nil { + panic(err) + } + fmt.Println("validation successfull") + return true +} + +func toStringKeys(val interface{}) (interface{}, error) { + var err error + switch val := val.(type) { + case map[interface{}]interface{}: + m := make(map[string]interface{}) + for k, v := range val { + k, ok := k.(string) + if !ok { + return nil, errors.New("found non-string key") + } + m[k], err = toStringKeys(v) + if err != nil { + return nil, err + } + } + return m, nil + case []interface{}: + var l = make([]interface{}, len(val)) + for i, v := range val { + l[i], err = toStringKeys(v) + if err != nil { + return nil, err + } + } + return l, nil + default: + return val, nil + } +} diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go new file mode 100644 index 0000000000..cb243189aa --- /dev/null +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -0,0 +1,28 @@ +package helpers + +import "testing" + +func TestReadSchema(t *testing.T) { + t.Run("basic read schema", func(t *testing.T) { + want := true + + if got := ReadSchema("example.yaml","example-schema.json"); got != want { + t.Errorf("AppleSauce() = %v, want %v", got, want) + } + }) +} + + +// func TestIsZarfValid(t *testing.T) { +// t.Run("first test", func(t *testing.T) { +// want := false + +// if got := ZarfYamlIsValid("/home/austin/code/zarf/zarf1.yaml"); got != want { +// t.Errorf("ZarfYamlIsValid = %v, want %v", got, want) +// } +// // if got := ZarfYamlIsValid("/home/austin/code/zarf/zarf.yaml"); got != want { +// // t.Errorf("AppleSauce() = %v, want %v", got, want) +// // } +// }) +// } + diff --git a/zarf.schema.json b/zarf.schema.json index 483c37ea02..65592ec16f 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -658,12 +658,11 @@ "description": "The name of the component to import from the referenced zarf.yaml" }, "path": { - "pattern": "^(?!.*###ZARF_PKG_TMPL_).*$", "type": "string", "description": "The relative path to a directory containing a zarf.yaml to import from" }, "url": { - "pattern": "^oci://(?!.*###ZARF_PKG_TMPL_).*$", + "pattern": "^oci://.*$", "type": "string", "description": "[beta] The URL to a Zarf package to import via OCI" } From 1a77654d88aecd3c67ab90ad380cde22cc1c7d98 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 15:40:24 +0000 Subject: [PATCH 02/87] refactor --- src/pkg/utils/helpers/validate_schema.go | 35 ------------------------ 1 file changed, 35 deletions(-) diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index a59364df8f..18d41c5921 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,7 +3,6 @@ package helpers import ( "bytes" "encoding/json" - "errors" "fmt" "log" "os" @@ -29,10 +28,6 @@ func ReadSchema(yamlFile, jsonFile string) bool { if err != nil { panic(err) } - m, err = toStringKeys(m) - if err != nil { - panic(err) - } mJSON, err := json.Marshal(m) if err != nil { @@ -56,33 +51,3 @@ func ReadSchema(yamlFile, jsonFile string) bool { fmt.Println("validation successfull") return true } - -func toStringKeys(val interface{}) (interface{}, error) { - var err error - switch val := val.(type) { - case map[interface{}]interface{}: - m := make(map[string]interface{}) - for k, v := range val { - k, ok := k.(string) - if !ok { - return nil, errors.New("found non-string key") - } - m[k], err = toStringKeys(v) - if err != nil { - return nil, err - } - } - return m, nil - case []interface{}: - var l = make([]interface{}, len(val)) - for i, v := range val { - l[i], err = toStringKeys(v) - if err != nil { - return nil, err - } - } - return l, nil - default: - return val, nil - } -} From fb3a56e1d5f3a663d4877d56696048c2ef40db03 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 15:47:34 +0000 Subject: [PATCH 03/87] WIP refactor --- src/pkg/utils/helpers/validate_schema.go | 25 ++++++++----------- src/pkg/utils/helpers/validate_schema_test.go | 17 +------------ 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 18d41c5921..c98b3706dc 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,7 +3,6 @@ package helpers import ( "bytes" "encoding/json" - "fmt" "log" "os" @@ -11,43 +10,41 @@ import ( "gopkg.in/yaml.v3" ) -func ReadSchema(yamlFile, jsonFile string) bool { +func ValidateZarfSchema(yamlFile, jsonFile string) bool { yamlBytes, err := os.ReadFile(yamlFile) if err != nil { - log.Fatalf("Error reading YAML file: %s", yamlFile) + log.Fatalf("Error reading file: %s", err) } - jsonBytes, err := os.ReadFile(jsonFile) + zarfSchema, err := os.ReadFile(jsonFile) if err != nil { - log.Fatalf("Error reading YAML file: %s", jsonFile) + log.Fatalf("Error reading file: %s", err) } - var m interface{} - err = yaml.Unmarshal(yamlBytes, &m) + var unmarshalledYaml interface{} + err = yaml.Unmarshal(yamlBytes, &unmarshalledYaml) if err != nil { panic(err) } - mJSON, err := json.Marshal(m) + zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) if err != nil { panic(err) } - yamlJsonReader := bytes.NewReader(mJSON) - schemaReader := bytes.NewReader(jsonBytes) compiler := jsonschema.NewCompiler() + inMemoryZarfSchema := "zarf.schema.json" - if err := compiler.AddResource("schema.json", schemaReader); err != nil { + if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfSchema)); err != nil { panic(err) } - schema, err := compiler.Compile("schema.json") + schema, err := compiler.Compile(inMemoryZarfSchema) if err != nil { panic(err) } - if err := schema.Validate(yamlJsonReader); err != nil { + if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { panic(err) } - fmt.Println("validation successfull") return true } diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index cb243189aa..d97ad3c87e 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -6,23 +6,8 @@ func TestReadSchema(t *testing.T) { t.Run("basic read schema", func(t *testing.T) { want := true - if got := ReadSchema("example.yaml","example-schema.json"); got != want { + if got := ValidateZarfSchema("/home/austin/code/zarf/zarf.yaml", "../../../../zarf.schema.json"); got != want { t.Errorf("AppleSauce() = %v, want %v", got, want) } }) } - - -// func TestIsZarfValid(t *testing.T) { -// t.Run("first test", func(t *testing.T) { -// want := false - -// if got := ZarfYamlIsValid("/home/austin/code/zarf/zarf1.yaml"); got != want { -// t.Errorf("ZarfYamlIsValid = %v, want %v", got, want) -// } -// // if got := ZarfYamlIsValid("/home/austin/code/zarf/zarf.yaml"); got != want { -// // t.Errorf("AppleSauce() = %v, want %v", got, want) -// // } -// }) -// } - From 353ed3f8803c2272583f86bbb9ba4ba5822e61c7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 17:58:52 +0000 Subject: [PATCH 04/87] refactor validate schema to work with unmarshalled yaml --- src/cmd/prepare.go | 17 +++++++----- src/pkg/utils/helpers/validate_schema.go | 16 ++---------- src/pkg/utils/helpers/validate_schema_test.go | 26 +++++++++++++++---- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index a4613ba8e4..48f5544186 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -7,22 +7,21 @@ package cmd import ( "fmt" "io" - "log" "os" + "path/filepath" "strings" "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/spf13/cobra" - "github.com/xeipuuv/gojsonschema" - "sigs.k8s.io/yaml" ) var prepareCmd = &cobra.Command{ @@ -161,12 +160,16 @@ var prepareGenerateConfigFile = &cobra.Command{ } var prepareValidateSchema = &cobra.Command{ - Use: "lint [ FILENAME ] [ FILENAME ]", - Args: cobra.MaximumNArgs(2), - Short: lang.CmdPrepareGenerateConfigShort, - Long: lang.CmdPrepareGenerateConfigLong, + Use: "lint [ FILENAME ] [ FILENAME ]", + Args: cobra.MaximumNArgs(2), + Short: lang.CmdPrepareGenerateConfigShort, + Long: lang.CmdPrepareGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { + pkgClient := packager.NewOrDie(&pkgConfig) + defer pkgClient.ClearTempPaths() + // Need to find where we load in the file + helpers.ValidateZarfSchema() }, } diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index c98b3706dc..14ba0ef9ef 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -7,28 +7,16 @@ import ( "os" "github.com/santhosh-tekuri/jsonschema" - "gopkg.in/yaml.v3" ) -func ValidateZarfSchema(yamlFile, jsonFile string) bool { - - yamlBytes, err := os.ReadFile(yamlFile) - if err != nil { - log.Fatalf("Error reading file: %s", err) - } +func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, jsonFile string) bool { zarfSchema, err := os.ReadFile(jsonFile) if err != nil { log.Fatalf("Error reading file: %s", err) } - var unmarshalledYaml interface{} - err = yaml.Unmarshal(yamlBytes, &unmarshalledYaml) - if err != nil { - panic(err) - } - - zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) + zarfYamlAsJsonBytes, err := json.Marshal(zarfUnmarshaledYaml) if err != nil { panic(err) } diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index d97ad3c87e..e2b62bf77f 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -1,13 +1,29 @@ package helpers -import "testing" +import ( + "os" + "testing" -func TestReadSchema(t *testing.T) { + goyaml "github.com/goccy/go-yaml" +) + +func TestValidateZarfSchema(t *testing.T) { t.Run("basic read schema", func(t *testing.T) { want := true - - if got := ValidateZarfSchema("/home/austin/code/zarf/zarf.yaml", "../../../../zarf.schema.json"); got != want { - t.Errorf("AppleSauce() = %v, want %v", got, want) + var unmarshalledYaml interface{} + readYaml("../../../../zarf.yaml", &unmarshalledYaml) + if got := ValidateZarfSchema(unmarshalledYaml, "../../../../zarf.schema.json"); got != want { + t.Errorf("ValidateZarfSchema = %v, want %v", got, want) } }) } + +func readYaml(path string, destConfig any) error { + file, err := os.ReadFile(path) + + if err != nil { + return err + } + + return goyaml.Unmarshal(file, destConfig) +} From 719bf9a6e671ef47d99c16a21fefcd77eb111214 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 16 Oct 2023 23:13:29 +0000 Subject: [PATCH 05/87] wip --- main.go | 4 +++ src/cmd/prepare.go | 4 +-- src/config/config.go | 5 ++++ src/pkg/packager/lint.go | 26 +++++++++++++++++++ src/pkg/utils/helpers/validate_schema.go | 11 ++------ src/pkg/utils/helpers/validate_schema_test.go | 10 ++++++- 6 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 src/pkg/packager/lint.go diff --git a/main.go b/main.go index b3c621ef93..397b240fc6 100644 --- a/main.go +++ b/main.go @@ -17,8 +17,12 @@ var assets embed.FS //go:embed cosign.pub var cosignPublicKey string +//go:embed zarf.schema.json +var zarfSchema embed.FS + func main() { config.UIAssets = assets config.CosignPublicKey = cosignPublicKey + config.ZarfSchema = zarfSchema cmd.Execute() } diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 48f5544186..e028af5f0d 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -8,14 +8,12 @@ import ( "fmt" "io" "os" - "path/filepath" "strings" "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -169,7 +167,7 @@ var prepareValidateSchema = &cobra.Command{ pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() // Need to find where we load in the file - helpers.ValidateZarfSchema() + //helpers.ValidateZarfSchema() }, } diff --git a/src/config/config.go b/src/config/config.go index 669efd4fb4..a0eca247ac 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -94,6 +94,7 @@ var ( CosignPublicKey string UIAssets embed.FS + ZarfSchema embed.FS // Timestamp of when the CLI was started operationStartTime = time.Now().Unix() @@ -102,6 +103,10 @@ var ( ZarfDefaultCachePath = filepath.Join("~", ".zarf-cache") ) +func GetSchemaFile() ([]byte, error) { + return ZarfSchema.ReadFile("zarf.schema.json") +} + // GetArch returns the arch based on a priority list with options for overriding. func GetArch(archs ...string) string { // List of architecture overrides. diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go new file mode 100644 index 0000000000..5e3b708d4c --- /dev/null +++ b/src/pkg/packager/lint.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import ( + "fmt" + "path/filepath" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" +) + +// Create generates a Zarf package tarball for a given PackageConfig and optional base directory. +func (p *Packager) Lint() (err error) { + + if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + } + data, _ := config.ZarfSchema.ReadFile("zarf.schema.json") + helpers.ValidateZarfSchema(p.cfg.Pkg, data) + + return nil +} diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 14ba0ef9ef..03e1a34d15 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,18 +3,11 @@ package helpers import ( "bytes" "encoding/json" - "log" - "os" "github.com/santhosh-tekuri/jsonschema" ) -func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, jsonFile string) bool { - - zarfSchema, err := os.ReadFile(jsonFile) - if err != nil { - log.Fatalf("Error reading file: %s", err) - } +func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) bool { zarfYamlAsJsonBytes, err := json.Marshal(zarfUnmarshaledYaml) if err != nil { @@ -24,7 +17,7 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, jsonFile string) bool { compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" - if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfSchema)); err != nil { + if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { panic(err) } schema, err := compiler.Compile(inMemoryZarfSchema) diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index e2b62bf77f..cbfbe0d8ea 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -1,6 +1,7 @@ package helpers import ( + "log" "os" "testing" @@ -12,7 +13,14 @@ func TestValidateZarfSchema(t *testing.T) { want := true var unmarshalledYaml interface{} readYaml("../../../../zarf.yaml", &unmarshalledYaml) - if got := ValidateZarfSchema(unmarshalledYaml, "../../../../zarf.schema.json"); got != want { + zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") + if err != nil { + log.Fatalf("Error reading file: %s", err) + } + if err != nil { + panic(err) + } + if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != want { t.Errorf("ValidateZarfSchema = %v, want %v", got, want) } }) From ccf26adeff4d96894262682f72c0cb04d7898c95 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 17 Oct 2023 13:12:51 +0000 Subject: [PATCH 06/87] wip --- src/cmd/prepare.go | 13 ++++++---- src/pkg/packager/lint.go | 2 +- src/pkg/utils/helpers/example-schema.json | 31 ----------------------- src/pkg/utils/helpers/example.yaml | 6 ----- src/pkg/utils/helpers/validate_schema.go | 3 +++ 5 files changed, 12 insertions(+), 43 deletions(-) delete mode 100644 src/pkg/utils/helpers/example-schema.json delete mode 100644 src/pkg/utils/helpers/example.yaml diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index e028af5f0d..4db581a4ac 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -158,16 +158,19 @@ var prepareGenerateConfigFile = &cobra.Command{ } var prepareValidateSchema = &cobra.Command{ - Use: "lint [ FILENAME ] [ FILENAME ]", - Args: cobra.MaximumNArgs(2), + Use: "lint ", + Args: cobra.MaximumNArgs(0), Short: lang.CmdPrepareGenerateConfigShort, Long: lang.CmdPrepareGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { - + cwd, err := os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) + } + pkgConfig.CreateOpts.BaseDir = cwd pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() - // Need to find where we load in the file - //helpers.ValidateZarfSchema() + pkgClient.Lint() }, } diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 5e3b708d4c..f613e6f5f6 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -19,7 +19,7 @@ func (p *Packager) Lint() (err error) { if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } - data, _ := config.ZarfSchema.ReadFile("zarf.schema.json") + data, _ := config.GetSchemaFile() helpers.ValidateZarfSchema(p.cfg.Pkg, data) return nil diff --git a/src/pkg/utils/helpers/example-schema.json b/src/pkg/utils/helpers/example-schema.json deleted file mode 100644 index 513869f972..0000000000 --- a/src/pkg/utils/helpers/example-schema.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$id": "https://example.com/product.schema.json", - "title": "Product", - "description": "A product from Acme's catalog", - "type": "object", - "properties": { - "productId": { - "description": "The unique identifier for a product", - "type": "integer" - }, - "productName": { - "description": "Name of the product", - "type": "string" - }, - "price": { - "description": "The price of the product", - "type": "number", - "exclusiveMinimum": 0 - }, - "tags": { - "description": "Tags for the product", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - } - }, - "required": [ "productId", "productName", "price" ] -} diff --git a/src/pkg/utils/helpers/example.yaml b/src/pkg/utils/helpers/example.yaml deleted file mode 100644 index 5d19224c8c..0000000000 --- a/src/pkg/utils/helpers/example.yaml +++ /dev/null @@ -1,6 +0,0 @@ -productId: 1 -productName: A green door -price: 12.50 -tags: -- home -- green diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 03e1a34d15..4e884b2759 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,6 +3,7 @@ package helpers import ( "bytes" "encoding/json" + "fmt" "github.com/santhosh-tekuri/jsonschema" ) @@ -16,6 +17,7 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" + fmt.Println(string(zarfYamlAsJsonBytes)) if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { panic(err) @@ -27,5 +29,6 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { panic(err) } + fmt.Println("validation succesful") return true } From 610c088da2e29bcad7458fc322497a0ab0e249b9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 15:00:46 +0000 Subject: [PATCH 07/87] WIP have test working with sucessful and unsuccsful example --- src/cmd/prepare.go | 16 +- src/pkg/packager/lint.go | 2 + .../helpers/successful_validation/zarf.yaml | 43 ++++ src/pkg/utils/helpers/validate_schema.go | 16 +- src/pkg/utils/helpers/validate_schema_test.go | 21 +- src/test/e2e/09_lint_test.go | 17 ++ src/test/packages/09-test-lint/README.md | 1 + src/test/packages/09-test-lint/zarf.json | 216 ++++++++++++++++++ src/test/packages/09-test-lint/zarf.yaml | 9 + 9 files changed, 320 insertions(+), 21 deletions(-) create mode 100644 src/pkg/utils/helpers/successful_validation/zarf.yaml create mode 100644 src/test/e2e/09_lint_test.go create mode 100644 src/test/packages/09-test-lint/README.md create mode 100644 src/test/packages/09-test-lint/zarf.json create mode 100644 src/test/packages/09-test-lint/zarf.yaml diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 4db581a4ac..f46b3fcab4 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -158,16 +158,20 @@ var prepareGenerateConfigFile = &cobra.Command{ } var prepareValidateSchema = &cobra.Command{ - Use: "lint ", - Args: cobra.MaximumNArgs(0), + Use: "lint [ DIRECTORY ]", + Args: cobra.MaximumNArgs(1), Short: lang.CmdPrepareGenerateConfigShort, Long: lang.CmdPrepareGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { - cwd, err := os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) + if len(args) > 0 { + pkgConfig.CreateOpts.BaseDir = args[0] + } else { + var err error + pkgConfig.CreateOpts.BaseDir, err = os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + } } - pkgConfig.CreateOpts.BaseDir = cwd pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() pkgClient.Lint() diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index f613e6f5f6..79b0656145 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -19,7 +19,9 @@ func (p *Packager) Lint() (err error) { if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } + fmt.Printf("This is the struct %+v\n", p.cfg.Pkg) data, _ := config.GetSchemaFile() + helpers.ValidateZarfSchema(p.cfg.Pkg, data) return nil diff --git a/src/pkg/utils/helpers/successful_validation/zarf.yaml b/src/pkg/utils/helpers/successful_validation/zarf.yaml new file mode 100644 index 0000000000..859813fde5 --- /dev/null +++ b/src/pkg/utils/helpers/successful_validation/zarf.yaml @@ -0,0 +1,43 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + path: packages/distros/k3s + + # This package moves the injector & registries binaries + - name: zarf-injector + required: true + import: + path: packages/zarf-registry + + # Creates the temporary seed-registry + - name: zarf-seed-registry + required: true + import: + path: packages/zarf-registry + + # Creates the permanent registry + - name: zarf-registry + required: true + import: + path: packages/zarf-registry + + # Creates the pod+git mutating webhook + - name: zarf-agent + required: true + import: + path: packages/zarf-agent + + # (Optional) Adds logging to the cluster + - name: logging + import: + path: packages/logging-pgl + + # (Optional) Adds a git server to the cluster + - name: git-server + import: + path: packages/gitea diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 4e884b2759..355c06642c 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,32 +3,30 @@ package helpers import ( "bytes" "encoding/json" - "fmt" "github.com/santhosh-tekuri/jsonschema" ) -func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) bool { +func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) error { zarfYamlAsJsonBytes, err := json.Marshal(zarfUnmarshaledYaml) if err != nil { - panic(err) + return err } compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" - fmt.Println(string(zarfYamlAsJsonBytes)) if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { - panic(err) + return err } schema, err := compiler.Compile(inMemoryZarfSchema) if err != nil { - panic(err) + return err } if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { - panic(err) + return err } - fmt.Println("validation succesful") - return true + + return nil } diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index cbfbe0d8ea..8d7d1736d8 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -9,19 +9,28 @@ import ( ) func TestValidateZarfSchema(t *testing.T) { - t.Run("basic read schema", func(t *testing.T) { - want := true + t.Run("Read schema success", func(t *testing.T) { var unmarshalledYaml interface{} - readYaml("../../../../zarf.yaml", &unmarshalledYaml) + readYaml("successful_validation/zarf.yaml", &unmarshalledYaml) zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { log.Fatalf("Error reading file: %s", err) } + if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != nil { + t.Errorf("ValidateZarfSchema = %v, want %v", got, nil) + } + }) + + t.Run("Read schema fail", func(t *testing.T) { + var unmarshalledYaml interface{} + readYaml("unsuccessful_validation/bad_zarf.yaml", &unmarshalledYaml) + zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { - panic(err) + log.Fatalf("Error reading file: %s", err) } - if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != want { - t.Errorf("ValidateZarfSchema = %v, want %v", got, want) + err = ValidateZarfSchema(unmarshalledYaml, zarfSchema) + if err == nil { + t.Errorf("ValidateZarfSchema worked on a bad file") } }) } diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go new file mode 100644 index 0000000000..e16ba396c5 --- /dev/null +++ b/src/test/e2e/09_lint_test.go @@ -0,0 +1,17 @@ +package test + +import ( + "path/filepath" + "testing" +) + +func TestLint(t *testing.T) { + t.Log("E2E: Lint") + + t.Run("zarf test lint", func(t *testing.T) { + t.Log("E2E: Test Differential Package Behavior") + + zarfYaml := filepath.Join("src", "test", "packages", "09-lint") + e2e.Zarf("prepare", "lint", zarfYaml) + }) +} diff --git a/src/test/packages/09-test-lint/README.md b/src/test/packages/09-test-lint/README.md new file mode 100644 index 0000000000..e93d9a79da --- /dev/null +++ b/src/test/packages/09-test-lint/README.md @@ -0,0 +1 @@ +# Lint test Package diff --git a/src/test/packages/09-test-lint/zarf.json b/src/test/packages/09-test-lint/zarf.json new file mode 100644 index 0000000000..fcb0df33db --- /dev/null +++ b/src/test/packages/09-test-lint/zarf.json @@ -0,0 +1,216 @@ +{ + "kind": "ZarfInitConfig", + "metadata": { + "name": "init", + "description": "Used to establish a new Zarf cluster" + }, + "build": { + "terminal": "", + "user": "", + "architecture": "", + "timestamp": "", + "version": "" + }, + "components": [ + { + "name": "k3s", + "only": { + "cluster": {} + }, + "import": { + "path": "packages/distros/k3s" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "zarf-injector", + "required": true, + "only": { + "cluster": {} + }, + "import": { + "path": "packages/zarf-registry" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "zarf-seed-registry", + "required": true, + "only": { + "cluster": {} + }, + "import": { + "path": "packages/zarf-registry" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "zarf-registry", + "required": true, + "only": { + "cluster": {} + }, + "import": { + "path": "packages/zarf-registry" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "zarf-agent", + "required": true, + "only": { + "cluster": {} + }, + "import": { + "path": "packages/zarf-agent" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "logging", + "only": { + "cluster": {} + }, + "import": { + "path": "packages/logging-pgl" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + }, + { + "name": "git-server", + "only": { + "cluster": {} + }, + "import": { + "path": "packages/gitea" + }, + "scripts": {}, + "extensions": {}, + "actions": { + "onCreate": { + "defaults": { + "shell": {} + } + }, + "onDeploy": { + "defaults": { + "shell": {} + } + }, + "onRemove": { + "defaults": { + "shell": {} + } + } + } + } + ] +} diff --git a/src/test/packages/09-test-lint/zarf.yaml b/src/test/packages/09-test-lint/zarf.yaml new file mode 100644 index 0000000000..53bef86117 --- /dev/null +++ b/src/test/packages/09-test-lint/zarf.yaml @@ -0,0 +1,9 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + path: packages/distros/k3s From db1c19ab467cf4b27125dc4910e47b1322e35b4e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 15:07:18 +0000 Subject: [PATCH 08/87] adding bad zarf file for testing --- .pre-commit-config.yaml | 2 + .../helpers/unsuccessful_validation/zarf.yaml | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7028b9df9c..81e7aafc71 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,3 +47,5 @@ repos: files: "zarf.yaml" types: [yaml] args: ["--schemafile", "zarf.schema.json"] + exclude: | + (?x)^(src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml) diff --git a/src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml b/src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml new file mode 100644 index 0000000000..7efc69a9a8 --- /dev/null +++ b/src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml @@ -0,0 +1,43 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + pa324234th: packages/distros/k3s + + # This package moves the injector & registries binaries + - name: zarf-injector + required: true + import: + path: packages/zarf-registry + + # Creates the temporary seed-registry + - name: zarf-seed-registry + required: true + import: + path: packages/zarf-registry + + # Creates the permanent registry + - name: zarf-registry + required: true + import: + path: packages/zarf-registry + + # Creates the pod+git mutating webhook + - name: zarf-agent + required: true + import: + path: packages/zarf-agent + + # (Optional) Adds logging to the cluster + - name: logging + import: + path: packages/logging-pgl + + # (Optional) Adds a git server to the cluster + - name: git-server + import: + path: packages/gitea From faebd0a03d1eff09b7fb975254e078c2c752446c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 17:46:47 +0000 Subject: [PATCH 09/87] WIP --- .pre-commit-config.yaml | 5 +- src/pkg/packager/lint.go | 10 +- src/pkg/utils/helpers/validate_schema.go | 3 + src/test/e2e/09_lint_test.go | 2 +- src/test/packages/09-test-lint/zarf.json | 216 ----------------------- src/test/packages/09-test-lint/zarf.yaml | 2 +- 6 files changed, 17 insertions(+), 221 deletions(-) delete mode 100644 src/test/packages/09-test-lint/zarf.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81e7aafc71..230312d079 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,4 +48,7 @@ repos: types: [yaml] args: ["--schemafile", "zarf.schema.json"] exclude: | - (?x)^(src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml) + (?x)^( + src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml| + src/test/packages/09-test-lint/ + ) diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 79b0656145..3d0ed0fa21 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ) @@ -20,9 +21,14 @@ func (p *Packager) Lint() (err error) { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } fmt.Printf("This is the struct %+v\n", p.cfg.Pkg) - data, _ := config.GetSchemaFile() + zarfSchema, _ := config.GetSchemaFile() + var zarfData interface{} - helpers.ValidateZarfSchema(p.cfg.Pkg, data) + if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { + return err + } + + helpers.ValidateZarfSchema(zarfData, zarfSchema) return nil } diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 355c06642c..939aec1a87 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,6 +3,7 @@ package helpers import ( "bytes" "encoding/json" + "fmt" "github.com/santhosh-tekuri/jsonschema" ) @@ -16,6 +17,8 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" + fmt.Println("we are here") + fmt.Println(zarfYamlAsJsonBytes) if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { return err diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index e16ba396c5..418de72ee6 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -9,7 +9,7 @@ func TestLint(t *testing.T) { t.Log("E2E: Lint") t.Run("zarf test lint", func(t *testing.T) { - t.Log("E2E: Test Differential Package Behavior") + t.Log("E2E: Test lint") zarfYaml := filepath.Join("src", "test", "packages", "09-lint") e2e.Zarf("prepare", "lint", zarfYaml) diff --git a/src/test/packages/09-test-lint/zarf.json b/src/test/packages/09-test-lint/zarf.json deleted file mode 100644 index fcb0df33db..0000000000 --- a/src/test/packages/09-test-lint/zarf.json +++ /dev/null @@ -1,216 +0,0 @@ -{ - "kind": "ZarfInitConfig", - "metadata": { - "name": "init", - "description": "Used to establish a new Zarf cluster" - }, - "build": { - "terminal": "", - "user": "", - "architecture": "", - "timestamp": "", - "version": "" - }, - "components": [ - { - "name": "k3s", - "only": { - "cluster": {} - }, - "import": { - "path": "packages/distros/k3s" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "zarf-injector", - "required": true, - "only": { - "cluster": {} - }, - "import": { - "path": "packages/zarf-registry" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "zarf-seed-registry", - "required": true, - "only": { - "cluster": {} - }, - "import": { - "path": "packages/zarf-registry" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "zarf-registry", - "required": true, - "only": { - "cluster": {} - }, - "import": { - "path": "packages/zarf-registry" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "zarf-agent", - "required": true, - "only": { - "cluster": {} - }, - "import": { - "path": "packages/zarf-agent" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "logging", - "only": { - "cluster": {} - }, - "import": { - "path": "packages/logging-pgl" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - }, - { - "name": "git-server", - "only": { - "cluster": {} - }, - "import": { - "path": "packages/gitea" - }, - "scripts": {}, - "extensions": {}, - "actions": { - "onCreate": { - "defaults": { - "shell": {} - } - }, - "onDeploy": { - "defaults": { - "shell": {} - } - }, - "onRemove": { - "defaults": { - "shell": {} - } - } - } - } - ] -} diff --git a/src/test/packages/09-test-lint/zarf.yaml b/src/test/packages/09-test-lint/zarf.yaml index 53bef86117..f30a6b8f9a 100644 --- a/src/test/packages/09-test-lint/zarf.yaml +++ b/src/test/packages/09-test-lint/zarf.yaml @@ -6,4 +6,4 @@ metadata: components: - name: k3s import: - path: packages/distros/k3s + pat12312h: packages/distros/k3s From 063c4383f5c546e54a311eda13a8ced9314b04a3 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 20:03:18 +0000 Subject: [PATCH 10/87] refactor schema unit tests --- src/cmd/prepare.go | 1 + src/pkg/packager/lint.go | 10 ++-- src/pkg/utils/helpers/validate_schema.go | 3 -- src/pkg/utils/helpers/validate_schema_test.go | 50 +++++++++---------- src/test/e2e/09_lint_test.go | 10 ++-- src/test/packages/09-test-lint/README.md | 1 - src/test/packages/09-test-lint/zarf.yaml | 9 ---- 7 files changed, 36 insertions(+), 48 deletions(-) delete mode 100644 src/test/packages/09-test-lint/README.md delete mode 100644 src/test/packages/09-test-lint/zarf.yaml diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index f46b3fcab4..5cdda72c06 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -175,6 +175,7 @@ var prepareValidateSchema = &cobra.Command{ pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() pkgClient.Lint() + }, } diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 3d0ed0fa21..0785c38047 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -16,19 +16,15 @@ import ( // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Lint() (err error) { - - if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - fmt.Printf("This is the struct %+v\n", p.cfg.Pkg) zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { return err } - helpers.ValidateZarfSchema(zarfData, zarfSchema) + if err = helpers.ValidateZarfSchema(zarfData, zarfSchema); err != nil { + return err + } return nil } diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/utils/helpers/validate_schema.go index 939aec1a87..355c06642c 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/utils/helpers/validate_schema.go @@ -3,7 +3,6 @@ package helpers import ( "bytes" "encoding/json" - "fmt" "github.com/santhosh-tekuri/jsonschema" ) @@ -17,8 +16,6 @@ func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) compiler := jsonschema.NewCompiler() inMemoryZarfSchema := "zarf.schema.json" - fmt.Println("we are here") - fmt.Println(zarfYamlAsJsonBytes) if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { return err diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/utils/helpers/validate_schema_test.go index 8d7d1736d8..3000885a6f 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/utils/helpers/validate_schema_test.go @@ -1,7 +1,6 @@ package helpers import ( - "log" "os" "testing" @@ -9,38 +8,39 @@ import ( ) func TestValidateZarfSchema(t *testing.T) { - t.Run("Read schema success", func(t *testing.T) { - var unmarshalledYaml interface{} - readYaml("successful_validation/zarf.yaml", &unmarshalledYaml) + readSchema := func(t *testing.T) []byte { + t.Helper() zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { - log.Fatalf("Error reading file: %s", err) + t.Fatalf("Error reading schema file: %s", err) } - if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != nil { - t.Errorf("ValidateZarfSchema = %v, want %v", got, nil) - } - }) + return zarfSchema + } - t.Run("Read schema fail", func(t *testing.T) { + readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { + t.Helper() var unmarshalledYaml interface{} - readYaml("unsuccessful_validation/bad_zarf.yaml", &unmarshalledYaml) - zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") + file, err := os.ReadFile(path) + goyaml.Unmarshal(file, &unmarshalledYaml) if err != nil { - log.Fatalf("Error reading file: %s", err) + return err } - err = ValidateZarfSchema(unmarshalledYaml, zarfSchema) - if err == nil { - t.Errorf("ValidateZarfSchema worked on a bad file") + return unmarshalledYaml + } + + t.Run("Read schema success", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") + zarfSchema := readSchema(t) + if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != nil { + t.Errorf("Expected successful validation, got error: %v", got) } }) -} -func readYaml(path string, destConfig any) error { - file, err := os.ReadFile(path) - - if err != nil { - return err - } - - return goyaml.Unmarshal(file, destConfig) + t.Run("Read schema fail", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/bad_zarf.yaml") + zarfSchema := readSchema(t) + if err := ValidateZarfSchema(unmarshalledYaml, zarfSchema); err == nil { + t.Errorf("Expected validation to fail, but it succeeded.") + } + }) } diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 418de72ee6..d2666be4ba 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -1,7 +1,7 @@ package test import ( - "path/filepath" + "fmt" "testing" ) @@ -11,7 +11,11 @@ func TestLint(t *testing.T) { t.Run("zarf test lint", func(t *testing.T) { t.Log("E2E: Test lint") - zarfYaml := filepath.Join("src", "test", "packages", "09-lint") - e2e.Zarf("prepare", "lint", zarfYaml) + //zarfYaml := filepath.Join("src", "test", "packages", "09-lint") + path := "src/test/packages/09-lint" + stdOut, stdErr, err := e2e.Zarf("prepare", "lint", path) + fmt.Println("printing stdout", stdOut) + fmt.Println(stdErr) + fmt.Println(err) }) } diff --git a/src/test/packages/09-test-lint/README.md b/src/test/packages/09-test-lint/README.md deleted file mode 100644 index e93d9a79da..0000000000 --- a/src/test/packages/09-test-lint/README.md +++ /dev/null @@ -1 +0,0 @@ -# Lint test Package diff --git a/src/test/packages/09-test-lint/zarf.yaml b/src/test/packages/09-test-lint/zarf.yaml deleted file mode 100644 index f30a6b8f9a..0000000000 --- a/src/test/packages/09-test-lint/zarf.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - pat12312h: packages/distros/k3s From 96efdb81e9b652a226eff937b1530b2233808405 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 20:31:32 +0000 Subject: [PATCH 11/87] moving lint --- .pre-commit-config.yaml | 6 +++--- src/cmd/prepare.go | 4 +--- src/pkg/{packager => lint}/lint.go | 12 +++++------- .../helpers => lint}/successful_validation/zarf.yaml | 0 .../unsuccessful_validation/zarf.yaml | 0 src/pkg/{utils/helpers => lint}/validate_schema.go | 10 +++++----- .../{utils/helpers => lint}/validate_schema_test.go | 8 ++++---- src/test/packages/09-lint/README.md | 1 + src/test/packages/09-lint/zarf.yaml | 9 +++++++++ 9 files changed, 28 insertions(+), 22 deletions(-) rename src/pkg/{packager => lint}/lint.go (61%) rename src/pkg/{utils/helpers => lint}/successful_validation/zarf.yaml (100%) rename src/pkg/{utils/helpers => lint}/unsuccessful_validation/zarf.yaml (100%) rename src/pkg/{utils/helpers => lint}/validate_schema.go (63%) rename src/pkg/{utils/helpers => lint}/validate_schema_test.go (81%) create mode 100644 src/test/packages/09-lint/README.md create mode 100644 src/test/packages/09-lint/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 230312d079..2fb8ee0763 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,6 +49,6 @@ repos: args: ["--schemafile", "zarf.schema.json"] exclude: | (?x)^( - src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml| - src/test/packages/09-test-lint/ - ) + src/pkg/lint/unsuccessful_validation/zarf.yaml| + src/test/packages/09-lint/zarf.yaml + )$ diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 5cdda72c06..83fe2bab78 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -172,9 +172,7 @@ var prepareValidateSchema = &cobra.Command{ message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) } } - pkgClient := packager.NewOrDie(&pkgConfig) - defer pkgClient.ClearTempPaths() - pkgClient.Lint() + Lint.ValidateSchema() }, } diff --git a/src/pkg/packager/lint.go b/src/pkg/lint/lint.go similarity index 61% rename from src/pkg/packager/lint.go rename to src/pkg/lint/lint.go index 0785c38047..fa3a20bbcd 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/lint/lint.go @@ -1,28 +1,26 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager +// Package packager contains functions for interacting with, managing and deploying Zarf packages +package lint import ( - "fmt" "path/filepath" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. -func (p *Packager) Lint() (err error) { +func Lint(baseDir string) (err error) { zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { + if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { return err } - if err = helpers.ValidateZarfSchema(zarfData, zarfSchema); err != nil { + if err = ValidateSchema(zarfData, zarfSchema); err != nil { return err } diff --git a/src/pkg/utils/helpers/successful_validation/zarf.yaml b/src/pkg/lint/successful_validation/zarf.yaml similarity index 100% rename from src/pkg/utils/helpers/successful_validation/zarf.yaml rename to src/pkg/lint/successful_validation/zarf.yaml diff --git a/src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml b/src/pkg/lint/unsuccessful_validation/zarf.yaml similarity index 100% rename from src/pkg/utils/helpers/unsuccessful_validation/zarf.yaml rename to src/pkg/lint/unsuccessful_validation/zarf.yaml diff --git a/src/pkg/utils/helpers/validate_schema.go b/src/pkg/lint/validate_schema.go similarity index 63% rename from src/pkg/utils/helpers/validate_schema.go rename to src/pkg/lint/validate_schema.go index 355c06642c..24e4545e3b 100644 --- a/src/pkg/utils/helpers/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -1,4 +1,4 @@ -package helpers +package lint import ( "bytes" @@ -7,17 +7,17 @@ import ( "github.com/santhosh-tekuri/jsonschema" ) -func ValidateZarfSchema(zarfUnmarshaledYaml interface{}, zarfJsonSchema []byte) error { +func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { - zarfYamlAsJsonBytes, err := json.Marshal(zarfUnmarshaledYaml) + zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) if err != nil { return err } compiler := jsonschema.NewCompiler() - inMemoryZarfSchema := "zarf.schema.json" + inMemoryZarfSchema := "schema.json" - if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(zarfJsonSchema)); err != nil { + if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(jsonSchema)); err != nil { return err } schema, err := compiler.Compile(inMemoryZarfSchema) diff --git a/src/pkg/utils/helpers/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go similarity index 81% rename from src/pkg/utils/helpers/validate_schema_test.go rename to src/pkg/lint/validate_schema_test.go index 3000885a6f..a8b4c729a6 100644 --- a/src/pkg/utils/helpers/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -1,4 +1,4 @@ -package helpers +package lint import ( "os" @@ -10,7 +10,7 @@ import ( func TestValidateZarfSchema(t *testing.T) { readSchema := func(t *testing.T) []byte { t.Helper() - zarfSchema, err := os.ReadFile("../../../../zarf.schema.json") + zarfSchema, err := os.ReadFile("../../../zarf.schema.json") if err != nil { t.Fatalf("Error reading schema file: %s", err) } @@ -31,7 +31,7 @@ func TestValidateZarfSchema(t *testing.T) { t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) - if got := ValidateZarfSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) @@ -39,7 +39,7 @@ func TestValidateZarfSchema(t *testing.T) { t.Run("Read schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/bad_zarf.yaml") zarfSchema := readSchema(t) - if err := ValidateZarfSchema(unmarshalledYaml, zarfSchema); err == nil { + if err := ValidateSchema(unmarshalledYaml, zarfSchema); err == nil { t.Errorf("Expected validation to fail, but it succeeded.") } }) diff --git a/src/test/packages/09-lint/README.md b/src/test/packages/09-lint/README.md new file mode 100644 index 0000000000..e93d9a79da --- /dev/null +++ b/src/test/packages/09-lint/README.md @@ -0,0 +1 @@ +# Lint test Package diff --git a/src/test/packages/09-lint/zarf.yaml b/src/test/packages/09-lint/zarf.yaml new file mode 100644 index 0000000000..f30a6b8f9a --- /dev/null +++ b/src/test/packages/09-lint/zarf.yaml @@ -0,0 +1,9 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + pat12312h: packages/distros/k3s From 6485174e38319c32e876cfa4cd2d9b55c40d0e5b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 20:40:14 +0000 Subject: [PATCH 12/87] WIP --- src/cmd/prepare.go | 8 +++++--- src/pkg/lint/lint.go | 2 +- src/pkg/lint/validate_schema_test.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 83fe2bab78..1894403ca7 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -163,16 +164,17 @@ var prepareValidateSchema = &cobra.Command{ Short: lang.CmdPrepareGenerateConfigShort, Long: lang.CmdPrepareGenerateConfigLong, Run: func(cmd *cobra.Command, args []string) { + baseDir := "" if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] + baseDir = args[0] } else { var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() + baseDir, err = os.Getwd() if err != nil { message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) } } - Lint.ValidateSchema() + lint.ValidateZarfSchema(baseDir) }, } diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index fa3a20bbcd..7c1cabf4db 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -13,7 +13,7 @@ import ( ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. -func Lint(baseDir string) (err error) { +func ValidateZarfSchema(baseDir string) (err error) { zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index a8b4c729a6..e457532315 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -7,7 +7,7 @@ import ( goyaml "github.com/goccy/go-yaml" ) -func TestValidateZarfSchema(t *testing.T) { +func TestValidateSchema(t *testing.T) { readSchema := func(t *testing.T) []byte { t.Helper() zarfSchema, err := os.ReadFile("../../../zarf.schema.json") From 93238ed21b269d6cede4215141ae8d56bdeae115 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 19 Oct 2023 20:49:27 +0000 Subject: [PATCH 13/87] WIP e2e tests functional --- src/cmd/prepare.go | 4 ++-- src/test/e2e/09_lint_test.go | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 1894403ca7..6582b784cd 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -174,8 +174,8 @@ var prepareValidateSchema = &cobra.Command{ message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) } } - lint.ValidateZarfSchema(baseDir) - + err := lint.ValidateZarfSchema(baseDir) + message.Fatal(err, err.Error()) }, } diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index d2666be4ba..7f527f9181 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -2,7 +2,10 @@ package test import ( "fmt" + "path/filepath" "testing" + + "github.com/stretchr/testify/require" ) func TestLint(t *testing.T) { @@ -11,9 +14,9 @@ func TestLint(t *testing.T) { t.Run("zarf test lint", func(t *testing.T) { t.Log("E2E: Test lint") - //zarfYaml := filepath.Join("src", "test", "packages", "09-lint") - path := "src/test/packages/09-lint" + path := filepath.Join("src", "test", "packages", "09-lint") stdOut, stdErr, err := e2e.Zarf("prepare", "lint", path) + require.NoError(t, err, "We don't get an error") fmt.Println("printing stdout", stdOut) fmt.Println(stdErr) fmt.Println(err) From ad05c6b37184b9c49b182b8806c1e287556c80da Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 20 Oct 2023 14:19:17 +0000 Subject: [PATCH 14/87] fix test lint --- src/pkg/lint/validate_schema.go | 3 ++- src/test/e2e/09_lint_test.go | 10 +++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 24e4545e3b..f7fd8c5153 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -3,6 +3,7 @@ package lint import ( "bytes" "encoding/json" + "fmt" "github.com/santhosh-tekuri/jsonschema" ) @@ -25,7 +26,7 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { - return err + return fmt.Errorf("this is an error I'm messing with %s", err) } return nil diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 7f527f9181..bfb02a2c76 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -1,7 +1,6 @@ package test import ( - "fmt" "path/filepath" "testing" @@ -12,13 +11,10 @@ func TestLint(t *testing.T) { t.Log("E2E: Lint") t.Run("zarf test lint", func(t *testing.T) { - t.Log("E2E: Test lint") + t.Log("E2E: Test lint on schema") path := filepath.Join("src", "test", "packages", "09-lint") - stdOut, stdErr, err := e2e.Zarf("prepare", "lint", path) - require.NoError(t, err, "We don't get an error") - fmt.Println("printing stdout", stdOut) - fmt.Println(stdErr) - fmt.Println(err) + _, _, err := e2e.Zarf("prepare", "lint", path) + require.Error(t, err, "We should get an error here because the yaml file is bad") }) } From 5b285773ecbad86d7d353fcc0250d3fd9c9b994b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 20 Oct 2023 14:33:02 +0000 Subject: [PATCH 15/87] lint tests working --- src/cmd/lint.go | 39 ++++++++++++++++++++++++++++++++++++ src/cmd/prepare.go | 23 --------------------- src/config/lang/english.go | 3 +++ src/test/e2e/09_lint_test.go | 2 +- 4 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 src/cmd/lint.go diff --git a/src/cmd/lint.go b/src/cmd/lint.go new file mode 100644 index 0000000000..d1504ace23 --- /dev/null +++ b/src/cmd/lint.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package cmd contains the CLI commands for Zarf. +package cmd + +import ( + "os" + + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/lint" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/spf13/cobra" +) + +var lintCmd = &cobra.Command{ + Use: "lint [ DIRECTORY ]", + Args: cobra.MaximumNArgs(1), + Aliases: []string{"l"}, + Short: lang.CmdLintShort, + Run: func(cmd *cobra.Command, args []string) { + baseDir := "" + if len(args) > 0 { + baseDir = args[0] + } else { + var err error + baseDir, err = os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + } + } + err := lint.ValidateZarfSchema(baseDir) + message.Fatal(err, err.Error()) + }, +} + +func init() { + rootCmd.AddCommand(lintCmd) +} diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 6582b784cd..f53dcdcae6 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -14,7 +14,6 @@ import ( "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -158,27 +157,6 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } -var prepareValidateSchema = &cobra.Command{ - Use: "lint [ DIRECTORY ]", - Args: cobra.MaximumNArgs(1), - Short: lang.CmdPrepareGenerateConfigShort, - Long: lang.CmdPrepareGenerateConfigLong, - Run: func(cmd *cobra.Command, args []string) { - baseDir := "" - if len(args) > 0 { - baseDir = args[0] - } else { - var err error - baseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } - err := lint.ValidateZarfSchema(baseDir) - message.Fatal(err, err.Error()) - }, -} - func init() { v := common.InitViper() @@ -187,7 +165,6 @@ func init() { prepareCmd.AddCommand(prepareComputeFileSha256sum) prepareCmd.AddCommand(prepareFindImages) prepareCmd.AddCommand(prepareGenerateConfigFile) - prepareCmd.AddCommand(prepareValidateSchema) prepareFindImages.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 1642e605d1..9d30058643 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -302,6 +302,9 @@ const ( CmdPackageClusterSourceFallback = "%q does not satisfy any current sources, assuming it is a package deployed to a cluster" CmdPackageInvalidSource = "Unable to identify source from %q: %s" + //zarf lint + CmdLintShort = "Tools for validating schema / ensuring best practices" + // zarf prepare CmdPrepareShort = "Tools to help prepare assets for packaging" diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index bfb02a2c76..9d206e47a5 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -14,7 +14,7 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema") path := filepath.Join("src", "test", "packages", "09-lint") - _, _, err := e2e.Zarf("prepare", "lint", path) + _, _, err := e2e.Zarf("lint", path) require.Error(t, err, "We should get an error here because the yaml file is bad") }) } From bb8eaf6670763ef280e71d9fd1d1f696a6f0244d Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 24 Oct 2023 18:40:15 +0000 Subject: [PATCH 16/87] adding tests, adding success mesage, taking away negative lookahead regex --- src/cmd/lint.go | 4 +++- src/pkg/lint/lint.go | 2 ++ src/pkg/lint/validate_schema.go | 2 +- src/test/e2e/09_lint_test.go | 14 +++++++++++--- src/test/packages/09-lint/README.md | 1 - src/test/packages/09-lint/zarf.yaml | 9 --------- src/types/component.go | 4 ++-- 7 files changed, 19 insertions(+), 17 deletions(-) delete mode 100644 src/test/packages/09-lint/README.md delete mode 100644 src/test/packages/09-lint/zarf.yaml diff --git a/src/cmd/lint.go b/src/cmd/lint.go index d1504ace23..13ee8385af 100644 --- a/src/cmd/lint.go +++ b/src/cmd/lint.go @@ -30,7 +30,9 @@ var lintCmd = &cobra.Command{ } } err := lint.ValidateZarfSchema(baseDir) - message.Fatal(err, err.Error()) + if err != nil { + message.Fatal(err, err.Error()) + } }, } diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 7c1cabf4db..d0e2db596d 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -9,6 +9,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -24,5 +25,6 @@ func ValidateZarfSchema(baseDir string) (err error) { return err } + message.Success("Validation successful") return nil } diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index f7fd8c5153..d8f869255c 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -26,7 +26,7 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { - return fmt.Errorf("this is an error I'm messing with %s", err) + return fmt.Errorf("schema validation error: %s", err) } return nil diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 9d206e47a5..1fdfade530 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -10,11 +10,19 @@ import ( func TestLint(t *testing.T) { t.Log("E2E: Lint") - t.Run("zarf test lint", func(t *testing.T) { + t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema") - path := filepath.Join("src", "test", "packages", "09-lint") + path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") _, _, err := e2e.Zarf("lint", path) - require.Error(t, err, "We should get an error here because the yaml file is bad") + require.Error(t, err, "Expect error here because the yaml file is not following schema") + }) + + t.Run("zarf test lint success", func(t *testing.T) { + t.Log("E2E: Test lint on schema") + + path := filepath.Join("src", "test", "packages", "09-lint", "valid_yaml") + _, _, err := e2e.Zarf("lint", path) + require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) } diff --git a/src/test/packages/09-lint/README.md b/src/test/packages/09-lint/README.md deleted file mode 100644 index e93d9a79da..0000000000 --- a/src/test/packages/09-lint/README.md +++ /dev/null @@ -1 +0,0 @@ -# Lint test Package diff --git a/src/test/packages/09-lint/zarf.yaml b/src/test/packages/09-lint/zarf.yaml deleted file mode 100644 index f30a6b8f9a..0000000000 --- a/src/test/packages/09-lint/zarf.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - pat12312h: packages/distros/k3s diff --git a/src/types/component.go b/src/types/component.go index 89b8e00111..883a852637 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -217,9 +217,9 @@ type ZarfDataInjection struct { type ZarfComponentImport struct { ComponentName string `json:"name,omitempty" jsonschema:"description=The name of the component to import from the referenced zarf.yaml"` // For further explanation see https://regex101.com/r/nxX8vx/1 - Path string `json:"path,omitempty" jsonschema:"description=The relative path to a directory containing a zarf.yaml to import from,pattern=^(?!.*###ZARF_PKG_TMPL_).*$"` + Path string `json:"path,omitempty" jsonschema:"description=The relative path to a directory containing a zarf.yaml to import from"` // For further explanation see https://regex101.com/r/nxX8vx/1 - URL string `json:"url,omitempty" jsonschema:"description=[beta] The URL to a Zarf package to import via OCI,pattern=^oci://(?!.*###ZARF_PKG_TMPL_).*$"` + URL string `json:"url,omitempty" jsonschema:"description=[beta] The URL to a Zarf package to import via OCI,pattern=^oci://.*$"` } // IsEmpty returns if the components fields (other than the fields we were told to ignore) are empty or set to the types zero-value From e28be8e87131b2a3e0f20414ac90674dfee4db67 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 24 Oct 2023 18:55:02 +0000 Subject: [PATCH 17/87] add yamls for tests --- .pre-commit-config.yaml | 2 +- .../packages/09-lint/invalid_yaml/zarf.yaml | 9 ++++ .../packages/09-lint/valid_yaml/zarf.yaml | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/test/packages/09-lint/invalid_yaml/zarf.yaml create mode 100644 src/test/packages/09-lint/valid_yaml/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fb8ee0763..9784951ef6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,5 +50,5 @@ repos: exclude: | (?x)^( src/pkg/lint/unsuccessful_validation/zarf.yaml| - src/test/packages/09-lint/zarf.yaml + src/test/packages/09-lint/.* )$ diff --git a/src/test/packages/09-lint/invalid_yaml/zarf.yaml b/src/test/packages/09-lint/invalid_yaml/zarf.yaml new file mode 100644 index 0000000000..f30a6b8f9a --- /dev/null +++ b/src/test/packages/09-lint/invalid_yaml/zarf.yaml @@ -0,0 +1,9 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + pat12312h: packages/distros/k3s diff --git a/src/test/packages/09-lint/valid_yaml/zarf.yaml b/src/test/packages/09-lint/valid_yaml/zarf.yaml new file mode 100644 index 0000000000..859813fde5 --- /dev/null +++ b/src/test/packages/09-lint/valid_yaml/zarf.yaml @@ -0,0 +1,43 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + path: packages/distros/k3s + + # This package moves the injector & registries binaries + - name: zarf-injector + required: true + import: + path: packages/zarf-registry + + # Creates the temporary seed-registry + - name: zarf-seed-registry + required: true + import: + path: packages/zarf-registry + + # Creates the permanent registry + - name: zarf-registry + required: true + import: + path: packages/zarf-registry + + # Creates the pod+git mutating webhook + - name: zarf-agent + required: true + import: + path: packages/zarf-agent + + # (Optional) Adds logging to the cluster + - name: logging + import: + path: packages/logging-pgl + + # (Optional) Adds a git server to the cluster + - name: git-server + import: + path: packages/gitea From e2a632da57a5839a41ee980187bce891077b9e84 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 1 Nov 2023 01:14:04 +0000 Subject: [PATCH 18/87] yaml extension partially tested --- .pre-commit-config.yaml | 1 + src/pkg/lint/validate_schema_test.go | 10 +++++++++- src/pkg/lint/yaml-extension/zarf.yaml | 24 ++++++++++++++++++++++++ src/test/e2e/09_lint_test.go | 1 + 4 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 src/pkg/lint/yaml-extension/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9784951ef6..cf0e67c06b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,5 +50,6 @@ repos: exclude: | (?x)^( src/pkg/lint/unsuccessful_validation/zarf.yaml| + src/pkg/lint/yaml-extension/zarf.yaml| src/test/packages/09-lint/.* )$ diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index e457532315..e098d71896 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -37,10 +37,18 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Read schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/bad_zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") zarfSchema := readSchema(t) if err := ValidateSchema(unmarshalledYaml, zarfSchema); err == nil { t.Errorf("Expected validation to fail, but it succeeded.") } }) + + t.Run("Read schema yaml extension", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") + zarfSchema := readSchema(t) + if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + t.Errorf("Expected successful validation, got error: %v", got) + } + }) } diff --git a/src/pkg/lint/yaml-extension/zarf.yaml b/src/pkg/lint/yaml-extension/zarf.yaml new file mode 100644 index 0000000000..c55c123e75 --- /dev/null +++ b/src/pkg/lint/yaml-extension/zarf.yaml @@ -0,0 +1,24 @@ +# Plan to have the schema or the lint code allow these soon +# x-env: &env +# environment: +# - CONFIG_KEY +# - EXAMPLE_KEY + +# x-version: &version +# version: 1.0.0 + +kind: ZarfPackageConfig +metadata: + name: hello + <<: &version 1.0.0 + +components: + - name: c1 + required: true + charts: + - name: redis + namespace: redis-system + version: *version + localPath: ./redis + images: + - &hello_image docker.io/library/redis:latest diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 1fdfade530..49641c3750 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -16,6 +16,7 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") _, _, err := e2e.Zarf("lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") + //Require contains to make sure the messaging is roughly how we want }) t.Run("zarf test lint success", func(t *testing.T) { From 09d1c07f3471096ac85cdd777ab5f8c4df792c3c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 1 Nov 2023 01:28:10 +0000 Subject: [PATCH 19/87] moving lint back to prepare --- src/cmd/lint.go | 41 ------------------------------------ src/cmd/prepare.go | 25 ++++++++++++++++++++++ src/test/e2e/09_lint_test.go | 4 ++-- 3 files changed, 27 insertions(+), 43 deletions(-) delete mode 100644 src/cmd/lint.go diff --git a/src/cmd/lint.go b/src/cmd/lint.go deleted file mode 100644 index 13ee8385af..0000000000 --- a/src/cmd/lint.go +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package cmd contains the CLI commands for Zarf. -package cmd - -import ( - "os" - - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/lint" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/spf13/cobra" -) - -var lintCmd = &cobra.Command{ - Use: "lint [ DIRECTORY ]", - Args: cobra.MaximumNArgs(1), - Aliases: []string{"l"}, - Short: lang.CmdLintShort, - Run: func(cmd *cobra.Command, args []string) { - baseDir := "" - if len(args) > 0 { - baseDir = args[0] - } else { - var err error - baseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } - err := lint.ValidateZarfSchema(baseDir) - if err != nil { - message.Fatal(err, err.Error()) - } - }, -} - -func init() { - rootCmd.AddCommand(lintCmd) -} diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index f53dcdcae6..c71f056938 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -157,6 +158,29 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } +var lintCmd = &cobra.Command{ + Use: "lint [ DIRECTORY ]", + Args: cobra.MaximumNArgs(1), + Aliases: []string{"l"}, + Short: lang.CmdLintShort, + Run: func(cmd *cobra.Command, args []string) { + baseDir := "" + if len(args) > 0 { + baseDir = args[0] + } else { + var err error + baseDir, err = os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + } + } + err := lint.ValidateZarfSchema(baseDir) + if err != nil { + message.Fatal(err, err.Error()) + } + }, +} + func init() { v := common.InitViper() @@ -165,6 +189,7 @@ func init() { prepareCmd.AddCommand(prepareComputeFileSha256sum) prepareCmd.AddCommand(prepareFindImages) prepareCmd.AddCommand(prepareGenerateConfigFile) + prepareCmd.AddCommand(lintCmd) prepareFindImages.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 49641c3750..4ac54f88ae 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -14,7 +14,7 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema") path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") - _, _, err := e2e.Zarf("lint", path) + _, _, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") //Require contains to make sure the messaging is roughly how we want }) @@ -23,7 +23,7 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema") path := filepath.Join("src", "test", "packages", "09-lint", "valid_yaml") - _, _, err := e2e.Zarf("lint", path) + _, _, err := e2e.Zarf("prepare", "lint", path) require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) } From 61e8f60ad2d2ea2ad4790b7a225693d2b491c940 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 6 Nov 2023 17:10:24 +0000 Subject: [PATCH 20/87] WIP test working --- src/pkg/lint/lint.go | 16 ++++++--- .../lint/unsuccessful_validation/zarf.yaml | 34 ------------------- src/pkg/lint/validate_schema.go | 30 +++++++++++++++- src/pkg/lint/validate_schema_test.go | 26 ++++++++++++-- 4 files changed, 64 insertions(+), 42 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index d0e2db596d..f545f92d28 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -7,7 +7,6 @@ package lint import ( "path/filepath" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -15,15 +14,22 @@ import ( // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func ValidateZarfSchema(baseDir string) (err error) { - zarfSchema, _ := config.GetSchemaFile() + // zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { return err } - if err = ValidateSchema(zarfData, zarfSchema); err != nil { - return err - } + // result := ValidateSchema(zarfData, zarfSchema) + + // if result.Valid() { + // message.Success("The document is valid\n") + // } else { + // fmt.Printf("The document is not valid. see errors :\n") + // for _, desc := range result.Errors() { + // fmt.Printf("- %s\n", desc) + // } + // } message.Success("Validation successful") return nil diff --git a/src/pkg/lint/unsuccessful_validation/zarf.yaml b/src/pkg/lint/unsuccessful_validation/zarf.yaml index 7efc69a9a8..b46d66c4e0 100644 --- a/src/pkg/lint/unsuccessful_validation/zarf.yaml +++ b/src/pkg/lint/unsuccessful_validation/zarf.yaml @@ -7,37 +7,3 @@ components: - name: k3s import: pa324234th: packages/distros/k3s - - # This package moves the injector & registries binaries - - name: zarf-injector - required: true - import: - path: packages/zarf-registry - - # Creates the temporary seed-registry - - name: zarf-seed-registry - required: true - import: - path: packages/zarf-registry - - # Creates the permanent registry - - name: zarf-registry - required: true - import: - path: packages/zarf-registry - - # Creates the pod+git mutating webhook - - name: zarf-agent - required: true - import: - path: packages/zarf-agent - - # (Optional) Adds logging to the cluster - - name: logging - import: - path: packages/logging-pgl - - # (Optional) Adds a git server to the cluster - - name: git-server - import: - path: packages/gitea diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index d8f869255c..3bccfa3ffd 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -3,12 +3,15 @@ package lint import ( "bytes" "encoding/json" + "errors" "fmt" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/santhosh-tekuri/jsonschema" + "github.com/xeipuuv/gojsonschema" ) -func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func ValidateSchema2(unmarshalledYaml interface{}, jsonSchema []byte) error { zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) if err != nil { @@ -31,3 +34,28 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return nil } + +func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { + schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) + documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) + + result, err := gojsonschema.Validate(schemaLoader, documentLoader) + if err != nil { + return err + } + + if result.Valid() { + message.Success("The document is valid\n") + } else { + errorMessage := "The document is not valid: " + for i, desc := range result.Errors() { + if i != 0 { + errorMessage = errorMessage + ", " + } + errorMessage = errorMessage + desc.String() + } + err = errors.New(errorMessage) + } + + return err +} diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index e098d71896..8c6bff2486 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -5,6 +5,7 @@ import ( "testing" goyaml "github.com/goccy/go-yaml" + "github.com/stretchr/testify/require" ) func TestValidateSchema(t *testing.T) { @@ -28,6 +29,16 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } + unmarshallStringYaml := func(t *testing.T, str string) interface{} { + t.Helper() + var unmarshalledYaml interface{} + err := goyaml.Unmarshal([]byte(str), &unmarshalledYaml) + if err != nil { + return err + } + return unmarshalledYaml + } + t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) @@ -37,9 +48,20 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Read schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") + var yamlContents = ` +kind: ZarfInitConfig +metadata: + name: init +components: + - name: k3s + import: + pa324234th: test +` + unmarshalledYaml := unmarshallStringYaml(t, yamlContents) zarfSchema := readSchema(t) - if err := ValidateSchema(unmarshalledYaml, zarfSchema); err == nil { + err := ValidateSchema(unmarshalledYaml, zarfSchema) + require.Contains(t, err.Error(), "components.0.import: Additional property pa324234th is not allowed") + if err == nil { t.Errorf("Expected validation to fail, but it succeeded.") } }) From 35c8e09f5464f6c989343ed71dbc930ace342e21 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 13:47:49 +0000 Subject: [PATCH 21/87] lint feature is working --- src/pkg/lint/lint.go | 16 ++--- .../lint/unsuccessful_validation/zarf.yaml | 10 ++- src/pkg/lint/validate_schema.go | 14 ++--- src/pkg/lint/validate_schema_test.go | 62 ++++++++++++------- src/test/e2e/09_lint_test.go | 8 +-- 5 files changed, 60 insertions(+), 50 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index f545f92d28..46a6d1b3eb 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -7,6 +7,7 @@ package lint import ( "path/filepath" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -14,22 +15,17 @@ import ( // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func ValidateZarfSchema(baseDir string) (err error) { - // zarfSchema, _ := config.GetSchemaFile() + zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { return err } - // result := ValidateSchema(zarfData, zarfSchema) + err = ValidateSchema(zarfData, zarfSchema) - // if result.Valid() { - // message.Success("The document is valid\n") - // } else { - // fmt.Printf("The document is not valid. see errors :\n") - // for _, desc := range result.Errors() { - // fmt.Printf("- %s\n", desc) - // } - // } + if err != nil { + return err + } message.Success("Validation successful") return nil diff --git a/src/pkg/lint/unsuccessful_validation/zarf.yaml b/src/pkg/lint/unsuccessful_validation/zarf.yaml index b46d66c4e0..6ebb779c23 100644 --- a/src/pkg/lint/unsuccessful_validation/zarf.yaml +++ b/src/pkg/lint/unsuccessful_validation/zarf.yaml @@ -1,9 +1,13 @@ kind: ZarfInitConfig metadata: name: init - description: Used to establish a new Zarf cluster + description: Testing bad yaml components: - - name: k3s + - name: first-test-component import: - pa324234th: packages/distros/k3s + not-path: packages/distros/k3s + + - name: second-test-component + not-import: + path: packages/distros/k3s diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 3bccfa3ffd..348007871d 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/santhosh-tekuri/jsonschema" "github.com/xeipuuv/gojsonschema" ) @@ -44,15 +43,10 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } - if result.Valid() { - message.Success("The document is valid\n") - } else { - errorMessage := "The document is not valid: " - for i, desc := range result.Errors() { - if i != 0 { - errorMessage = errorMessage + ", " - } - errorMessage = errorMessage + desc.String() + if !result.Valid() { + errorMessage := "The document is not valid:" + for _, desc := range result.Errors() { + errorMessage = errorMessage + "\n - " + desc.String() } err = errors.New(errorMessage) } diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 8c6bff2486..41ece0f9d0 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -29,46 +29,62 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } - unmarshallStringYaml := func(t *testing.T, str string) interface{} { - t.Helper() - var unmarshalledYaml interface{} - err := goyaml.Unmarshal([]byte(str), &unmarshalledYaml) - if err != nil { - return err - } - return unmarshalledYaml - } + // unmarshallStringYaml := func(t *testing.T, str string) (interface{}, error) { + // t.Helper() + // var unmarshalledYaml interface{} + // err := goyaml.Unmarshal([]byte(str), &unmarshalledYaml) + // return unmarshalledYaml, err + // } t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) + err := ValidateSchema(unmarshalledYaml, zarfSchema) + require.NoError(t, err) if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) t.Run("Read schema fail", func(t *testing.T) { - var yamlContents = ` -kind: ZarfInitConfig -metadata: - name: init -components: - - name: k3s - import: - pa324234th: test -` - unmarshalledYaml := unmarshallStringYaml(t, yamlContents) + // var yamlContents = ` + // kind: ZarfInitConfig + // metadata: + // name: init + // components: + // - name: k3s + // import: + // pa324234th: test + // - name: test + // import: + // pa3th: test + // ` + // unmarshalledYaml,_ := unmarshallStringYaml(t, yamlContents) zarfSchema := readSchema(t) + unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") err := ValidateSchema(unmarshalledYaml, zarfSchema) - require.Contains(t, err.Error(), "components.0.import: Additional property pa324234th is not allowed") - if err == nil { - t.Errorf("Expected validation to fail, but it succeeded.") - } + errorMessage := `The document is not valid: + - components.0.import: Additional property not-path is not allowed + - components.1: Additional property not-import is not allowed` + require.EqualError(t, err, errorMessage) }) + // t.Run("bad yaml", func(t *testing.T) { + // var yamlContents = "unquoted_string_with_colons: key: value" + // unmarshalledYaml,err := unmarshallStringYaml(t, yamlContents) + // zarfSchema := readSchema(t) + // err = ValidateSchema(unmarshalledYaml, zarfSchema) + // require.Contains(t, err.Error(), "components.0.import: Additional property pa324234th is not allowed") + // if err == nil { + // t.Errorf("Expected validation to fail, but it succeeded.") + // } + // }) + t.Run("Read schema yaml extension", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") zarfSchema := readSchema(t) + err := ValidateSchema(unmarshalledYaml, zarfSchema) + require.NoError(t, err) if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/09_lint_test.go index 4ac54f88ae..f7fa1da8c7 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/09_lint_test.go @@ -11,16 +11,16 @@ func TestLint(t *testing.T) { t.Log("E2E: Lint") t.Run("zarf test lint fail", func(t *testing.T) { - t.Log("E2E: Test lint on schema") + t.Log("E2E: Test lint on schema fail") path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") - _, _, err := e2e.Zarf("prepare", "lint", path) + _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") - //Require contains to make sure the messaging is roughly how we want + require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") }) t.Run("zarf test lint success", func(t *testing.T) { - t.Log("E2E: Test lint on schema") + t.Log("E2E: Test lint on schema success") path := filepath.Join("src", "test", "packages", "09-lint", "valid_yaml") _, _, err := e2e.Zarf("prepare", "lint", path) From fa03f3529c6ca6faad8f3b35bf3ff8549c11999e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 13:48:13 +0000 Subject: [PATCH 22/87] refactoring --- src/pkg/lint/validate_schema.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 348007871d..a614400b2d 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -1,39 +1,11 @@ package lint import ( - "bytes" - "encoding/json" "errors" - "fmt" - "github.com/santhosh-tekuri/jsonschema" "github.com/xeipuuv/gojsonschema" ) -func ValidateSchema2(unmarshalledYaml interface{}, jsonSchema []byte) error { - - zarfYamlAsJsonBytes, err := json.Marshal(unmarshalledYaml) - if err != nil { - return err - } - - compiler := jsonschema.NewCompiler() - inMemoryZarfSchema := "schema.json" - - if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(jsonSchema)); err != nil { - return err - } - schema, err := compiler.Compile(inMemoryZarfSchema) - if err != nil { - return err - } - if err := schema.Validate(bytes.NewReader(zarfYamlAsJsonBytes)); err != nil { - return fmt.Errorf("schema validation error: %s", err) - } - - return nil -} - func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) From a0c307dd09fbfa64581386589e66b681ecf0a0ce Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 13:54:01 +0000 Subject: [PATCH 23/87] adding comment to exported function --- src/config/config.go | 1 + src/pkg/lint/lint.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.go b/src/config/config.go index b71950386a..f5d24b2e05 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -102,6 +102,7 @@ var ( ZarfDefaultCachePath = filepath.Join("~", ".zarf-cache") ) +// GetSchemaFile returns the zarf schema file func GetSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 46a6d1b3eb..00029836e6 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -7,7 +7,6 @@ package lint import ( "path/filepath" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" From 3376a3eed0b9aed8d4dac1ad45ca0d0ee2226a12 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 14:03:43 +0000 Subject: [PATCH 24/87] unexporting validate schema --- src/pkg/lint/lint.go | 5 +++-- src/pkg/lint/validate_schema.go | 2 +- src/pkg/lint/validate_schema_test.go | 10 +++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 00029836e6..5f66e05a71 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -7,12 +7,13 @@ package lint import ( "path/filepath" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" ) -// Create generates a Zarf package tarball for a given PackageConfig and optional base directory. +// Validates a zarf file against the zarf schema, returns an error if the file is invalid func ValidateZarfSchema(baseDir string) (err error) { zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} @@ -20,7 +21,7 @@ func ValidateZarfSchema(baseDir string) (err error) { return err } - err = ValidateSchema(zarfData, zarfSchema) + err = validateSchema(zarfData, zarfSchema) if err != nil { return err diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index a614400b2d..9b2e99a60d 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -6,7 +6,7 @@ import ( "github.com/xeipuuv/gojsonschema" ) -func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 41ece0f9d0..aeed602a0a 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -39,9 +39,9 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) @@ -62,7 +62,7 @@ func TestValidateSchema(t *testing.T) { // unmarshalledYaml,_ := unmarshallStringYaml(t, yamlContents) zarfSchema := readSchema(t) unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := `The document is not valid: - components.0.import: Additional property not-path is not allowed - components.1: Additional property not-import is not allowed` @@ -83,9 +83,9 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema yaml extension", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) From 73d4de80aa941fd4d3341f4cb278b798be9085cd Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 14:11:12 +0000 Subject: [PATCH 25/87] moving lint to test 11 --- .pre-commit-config.yaml | 2 +- src/test/e2e/{09_lint_test.go => 11_lint_test.go} | 4 ++-- src/test/packages/{09-lint => 11-lint}/invalid_yaml/zarf.yaml | 0 src/test/packages/{09-lint => 11-lint}/valid_yaml/zarf.yaml | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/test/e2e/{09_lint_test.go => 11_lint_test.go} (85%) rename src/test/packages/{09-lint => 11-lint}/invalid_yaml/zarf.yaml (100%) rename src/test/packages/{09-lint => 11-lint}/valid_yaml/zarf.yaml (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0e67c06b..ec3a9f6a58 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,5 +51,5 @@ repos: (?x)^( src/pkg/lint/unsuccessful_validation/zarf.yaml| src/pkg/lint/yaml-extension/zarf.yaml| - src/test/packages/09-lint/.* + src/test/packages/11-lint/.* )$ diff --git a/src/test/e2e/09_lint_test.go b/src/test/e2e/11_lint_test.go similarity index 85% rename from src/test/e2e/09_lint_test.go rename to src/test/e2e/11_lint_test.go index f7fa1da8c7..c4578a2442 100644 --- a/src/test/e2e/09_lint_test.go +++ b/src/test/e2e/11_lint_test.go @@ -13,7 +13,7 @@ func TestLint(t *testing.T) { t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema fail") - path := filepath.Join("src", "test", "packages", "09-lint", "invalid_yaml") + path := filepath.Join("src", "test", "packages", "11-lint", "invalid_yaml") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") @@ -22,7 +22,7 @@ func TestLint(t *testing.T) { t.Run("zarf test lint success", func(t *testing.T) { t.Log("E2E: Test lint on schema success") - path := filepath.Join("src", "test", "packages", "09-lint", "valid_yaml") + path := filepath.Join("src", "test", "packages", "11-lint", "valid_yaml") _, _, err := e2e.Zarf("prepare", "lint", path) require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) diff --git a/src/test/packages/09-lint/invalid_yaml/zarf.yaml b/src/test/packages/11-lint/invalid_yaml/zarf.yaml similarity index 100% rename from src/test/packages/09-lint/invalid_yaml/zarf.yaml rename to src/test/packages/11-lint/invalid_yaml/zarf.yaml diff --git a/src/test/packages/09-lint/valid_yaml/zarf.yaml b/src/test/packages/11-lint/valid_yaml/zarf.yaml similarity index 100% rename from src/test/packages/09-lint/valid_yaml/zarf.yaml rename to src/test/packages/11-lint/valid_yaml/zarf.yaml From d6dc0b5abf51b3ad23f1208b3fb4996950a39a8f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 14:39:44 +0000 Subject: [PATCH 26/87] tests working --- src/pkg/lint/lint.go | 2 +- src/pkg/lint/validate_schema.go | 2 +- src/pkg/lint/validate_schema_test.go | 43 ++++------------------------ 3 files changed, 8 insertions(+), 39 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 5f66e05a71..83db6cba36 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -21,7 +21,7 @@ func ValidateZarfSchema(baseDir string) (err error) { return err } - err = validateSchema(zarfData, zarfSchema) + err = ValidateSchema(zarfData, zarfSchema) if err != nil { return err diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 9b2e99a60d..a614400b2d 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -6,7 +6,7 @@ import ( "github.com/xeipuuv/gojsonschema" ) -func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index aeed602a0a..0c1cd93aa0 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -29,63 +29,32 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } - // unmarshallStringYaml := func(t *testing.T, str string) (interface{}, error) { - // t.Helper() - // var unmarshalledYaml interface{} - // err := goyaml.Unmarshal([]byte(str), &unmarshalledYaml) - // return unmarshalledYaml, err - // } - t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) + err := ValidateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) t.Run("Read schema fail", func(t *testing.T) { - // var yamlContents = ` - // kind: ZarfInitConfig - // metadata: - // name: init - // components: - // - name: k3s - // import: - // pa324234th: test - // - name: test - // import: - // pa3th: test - // ` - // unmarshalledYaml,_ := unmarshallStringYaml(t, yamlContents) - zarfSchema := readSchema(t) unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") - err := validateSchema(unmarshalledYaml, zarfSchema) + zarfSchema := readSchema(t) + err := ValidateSchema(unmarshalledYaml, zarfSchema) errorMessage := `The document is not valid: - components.0.import: Additional property not-path is not allowed - components.1: Additional property not-import is not allowed` require.EqualError(t, err, errorMessage) }) - // t.Run("bad yaml", func(t *testing.T) { - // var yamlContents = "unquoted_string_with_colons: key: value" - // unmarshalledYaml,err := unmarshallStringYaml(t, yamlContents) - // zarfSchema := readSchema(t) - // err = ValidateSchema(unmarshalledYaml, zarfSchema) - // require.Contains(t, err.Error(), "components.0.import: Additional property pa324234th is not allowed") - // if err == nil { - // t.Errorf("Expected validation to fail, but it succeeded.") - // } - // }) - t.Run("Read schema yaml extension", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") zarfSchema := readSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) + err := ValidateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) From 0d414eeb8726f81661fae841f835cea945c05b68 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 18:01:56 +0000 Subject: [PATCH 27/87] warning if someone has zarf_pkg_tmpl somewhere --- src/pkg/lint/lint.go | 11 +++++++ .../lint/unsuccessful_validation/zarf.yaml | 8 +++++ src/pkg/lint/validate_schema.go | 32 ++++++++++++++++++- src/pkg/lint/validate_schema_test.go | 22 ++++++++++++- .../packages/11-lint/invalid_yaml/zarf.yaml | 7 ++++ 5 files changed, 78 insertions(+), 2 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 83db6cba36..1fc550b68d 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -11,10 +11,21 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" ) // Validates a zarf file against the zarf schema, returns an error if the file is invalid func ValidateZarfSchema(baseDir string) (err error) { + var typedZarfData types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &typedZarfData); err != nil { + return err + } + + err = checkForVarInComponentImport(typedZarfData) + if err != nil { + message.Warn(err.Error()) + } + zarfSchema, _ := config.GetSchemaFile() var zarfData interface{} if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { diff --git a/src/pkg/lint/unsuccessful_validation/zarf.yaml b/src/pkg/lint/unsuccessful_validation/zarf.yaml index 6ebb779c23..639f1cf462 100644 --- a/src/pkg/lint/unsuccessful_validation/zarf.yaml +++ b/src/pkg/lint/unsuccessful_validation/zarf.yaml @@ -11,3 +11,11 @@ components: - name: second-test-component not-import: path: packages/distros/k3s + + - name: import-test + import: + path: "###ZARF_PKG_TMPL_ZEBRA###" + + - name: import-url + import: + url: "oci://###ZARF_PKG_TMPL_ZEBRA###" diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index a614400b2d..950a426439 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -2,10 +2,18 @@ package lint import ( "errors" + "strconv" + "strings" + "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) +const ( + zarfInvalidPrefix = "zarf.yaml is not valid:" + zarfWarningPrefix = "zarf schema warning:" +) + func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) @@ -16,7 +24,7 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { } if !result.Valid() { - errorMessage := "The document is not valid:" + errorMessage := zarfInvalidPrefix for _, desc := range result.Errors() { errorMessage = errorMessage + "\n - " + desc.String() } @@ -25,3 +33,25 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } + +func checkForVarInComponentImport(unmarshalledYaml types.ZarfPackage) error { + valid := true + errorMessage := zarfWarningPrefix + componentWarningStart := "\n - component." + for i, component := range unmarshalledYaml.Components { + if strings.Contains(component.Import.Path, "###ZARF_PKG_TMPL_") { + errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + + ".import.path will not resolve ZARF_PKG_TMPL_* variables" + valid = false + } + if strings.Contains(component.Import.URL, "###ZARF_PKG_TMPL_") { + errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + + ".import.url will not resolve ZARF_PKG_TMPL_* variables" + valid = false + } + } + if valid { + return nil + } + return errors.New(errorMessage) +} diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 0c1cd93aa0..02a1062921 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -4,6 +4,7 @@ import ( "os" "testing" + "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" ) @@ -43,7 +44,7 @@ func TestValidateSchema(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") zarfSchema := readSchema(t) err := ValidateSchema(unmarshalledYaml, zarfSchema) - errorMessage := `The document is not valid: + errorMessage := zarfInvalidPrefix + ` - components.0.import: Additional property not-path is not allowed - components.1: Additional property not-import is not allowed` require.EqualError(t, err, errorMessage) @@ -58,4 +59,23 @@ func TestValidateSchema(t *testing.T) { t.Errorf("Expected successful validation, got error: %v", got) } }) + + t.Run("Read schema yaml extension", func(t *testing.T) { + var unmarshalledYaml types.ZarfPackage + file, _ := os.ReadFile("yaml-extension/zarf.yaml") + goyaml.Unmarshal(file, &unmarshalledYaml) + err := checkForVarInComponentImport(unmarshalledYaml) + require.NoError(t, err) + }) + + t.Run("Read schema import-error", func(t *testing.T) { + var unmarshalledYaml types.ZarfPackage + file, _ := os.ReadFile("unsuccessful_validation/zarf.yaml") + goyaml.Unmarshal(file, &unmarshalledYaml) + err := checkForVarInComponentImport(unmarshalledYaml) + errorMessage := zarfWarningPrefix + ` + - component.2.import.path will not resolve ZARF_PKG_TMPL_* variables + - component.3.import.url will not resolve ZARF_PKG_TMPL_* variables` + require.EqualError(t, err, errorMessage) + }) } diff --git a/src/test/packages/11-lint/invalid_yaml/zarf.yaml b/src/test/packages/11-lint/invalid_yaml/zarf.yaml index f30a6b8f9a..39f1b899da 100644 --- a/src/test/packages/11-lint/invalid_yaml/zarf.yaml +++ b/src/test/packages/11-lint/invalid_yaml/zarf.yaml @@ -7,3 +7,10 @@ components: - name: k3s import: pat12312h: packages/distros/k3s + - name: import-test + import: + path: "###ZARF_PKG_TMPL_ZEBRA###" + + - name: import-url + import: + url: "oci://###ZARF_PKG_TMPL_ZEBRA###" From 14e9804bd1b6416e571939946c4b7639cb5b23a7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 18:47:34 +0000 Subject: [PATCH 28/87] refactor --- src/pkg/lint/validate_schema.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 950a426439..6ccc0ea5e2 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -12,6 +12,7 @@ import ( const ( zarfInvalidPrefix = "zarf.yaml is not valid:" zarfWarningPrefix = "zarf schema warning:" + zarfTemplateVar = "###ZARF_PKG_TMPL_" ) func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { @@ -39,12 +40,12 @@ func checkForVarInComponentImport(unmarshalledYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "\n - component." for i, component := range unmarshalledYaml.Components { - if strings.Contains(component.Import.Path, "###ZARF_PKG_TMPL_") { + if strings.Contains(component.Import.Path, zarfTemplateVar) { errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + ".import.path will not resolve ZARF_PKG_TMPL_* variables" valid = false } - if strings.Contains(component.Import.URL, "###ZARF_PKG_TMPL_") { + if strings.Contains(component.Import.URL, zarfTemplateVar) { errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + ".import.url will not resolve ZARF_PKG_TMPL_* variables" valid = false From 78b76f042a790b333f4ef87f303919796fe1460c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 19:26:11 +0000 Subject: [PATCH 29/87] changing format of message --- src/pkg/lint/lint.go | 2 +- src/pkg/lint/validate_schema.go | 12 ++++++------ src/pkg/lint/validate_schema_test.go | 10 +++++----- src/test/e2e/11_lint_test.go | 1 + 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 1fc550b68d..092d1b04f9 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -32,7 +32,7 @@ func ValidateZarfSchema(baseDir string) (err error) { return err } - err = ValidateSchema(zarfData, zarfSchema) + err = validateSchema(zarfData, zarfSchema) if err != nil { return err diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 6ccc0ea5e2..a728f00e34 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -15,7 +15,7 @@ const ( zarfTemplateVar = "###ZARF_PKG_TMPL_" ) -func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) @@ -35,19 +35,19 @@ func ValidateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return err } -func checkForVarInComponentImport(unmarshalledYaml types.ZarfPackage) error { +func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := "\n - component." - for i, component := range unmarshalledYaml.Components { + componentWarningStart := " component." + for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + - ".import.path will not resolve ZARF_PKG_TMPL_* variables" + ".import.path will not resolve ZARF_PKG_TMPL_* variables." valid = false } if strings.Contains(component.Import.URL, zarfTemplateVar) { errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + - ".import.url will not resolve ZARF_PKG_TMPL_* variables" + ".import.url will not resolve ZARF_PKG_TMPL_* variables." valid = false } } diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 02a1062921..2b71b2fae4 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -33,9 +33,9 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) @@ -43,7 +43,7 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` - components.0.import: Additional property not-path is not allowed - components.1: Additional property not-import is not allowed` @@ -53,9 +53,9 @@ func TestValidateSchema(t *testing.T) { t.Run("Read schema yaml extension", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") zarfSchema := readSchema(t) - err := ValidateSchema(unmarshalledYaml, zarfSchema) + err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := ValidateSchema(unmarshalledYaml, zarfSchema); got != nil { + if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { t.Errorf("Expected successful validation, got error: %v", got) } }) diff --git a/src/test/e2e/11_lint_test.go b/src/test/e2e/11_lint_test.go index c4578a2442..14b0865040 100644 --- a/src/test/e2e/11_lint_test.go +++ b/src/test/e2e/11_lint_test.go @@ -17,6 +17,7 @@ func TestLint(t *testing.T) { _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") + require.Contains(t, stderr, "component.1.import.path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { From 03ea4459b030cfc127718ca31a2e1f69247de159 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 19:28:09 +0000 Subject: [PATCH 30/87] fixing comment --- src/pkg/lint/lint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 092d1b04f9..febc4f902f 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// Validates a zarf file against the zarf schema, returns an error if the file is invalid +// ValidatesZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid func ValidateZarfSchema(baseDir string) (err error) { var typedZarfData types.ZarfPackage if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &typedZarfData); err != nil { From ec1eb04a4a6e7efc031e79045c0d31b570623fde Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 19:35:40 +0000 Subject: [PATCH 31/87] fix comment --- src/pkg/lint/lint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index febc4f902f..277020fcd8 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// ValidatesZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid +// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid func ValidateZarfSchema(baseDir string) (err error) { var typedZarfData types.ZarfPackage if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &typedZarfData); err != nil { From f5f484f8088179b6b05b8ab7814cc8e441b538b3 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 19:36:16 +0000 Subject: [PATCH 32/87] comment --- src/pkg/lint/lint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go index 277020fcd8..ee119ac663 100644 --- a/src/pkg/lint/lint.go +++ b/src/pkg/lint/lint.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying Zarf packages +// Package lint contains functions for linting the zarf.yaml package lint import ( From b82f857b110437c7b00a4411931cd6c75f3c485e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 7 Nov 2023 20:10:10 +0000 Subject: [PATCH 33/87] refactor tests --- .pre-commit-config.yaml | 1 - src/pkg/lint/successful_validation/zarf.yaml | 59 +++++++------------- src/pkg/lint/validate_schema_test.go | 44 ++++++--------- src/pkg/lint/yaml-extension/zarf.yaml | 24 -------- 4 files changed, 37 insertions(+), 91 deletions(-) delete mode 100644 src/pkg/lint/yaml-extension/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec3a9f6a58..564e34c92a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,6 +50,5 @@ repos: exclude: | (?x)^( src/pkg/lint/unsuccessful_validation/zarf.yaml| - src/pkg/lint/yaml-extension/zarf.yaml| src/test/packages/11-lint/.* )$ diff --git a/src/pkg/lint/successful_validation/zarf.yaml b/src/pkg/lint/successful_validation/zarf.yaml index 859813fde5..a8640d1b4c 100644 --- a/src/pkg/lint/successful_validation/zarf.yaml +++ b/src/pkg/lint/successful_validation/zarf.yaml @@ -1,43 +1,24 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - path: packages/distros/k3s +# Plan to have the schema or the lint code allow these soon +# x-env: &env +# environment: +# - CONFIG_KEY +# - EXAMPLE_KEY - # This package moves the injector & registries binaries - - name: zarf-injector - required: true - import: - path: packages/zarf-registry - - # Creates the temporary seed-registry - - name: zarf-seed-registry - required: true - import: - path: packages/zarf-registry +# x-version: &version +# version: 1.0.0 - # Creates the permanent registry - - name: zarf-registry - required: true - import: - path: packages/zarf-registry +kind: ZarfPackageConfig +metadata: + name: hello + version: &version 1.0.0 - # Creates the pod+git mutating webhook - - name: zarf-agent +components: + - name: c1 required: true - import: - path: packages/zarf-agent - - # (Optional) Adds logging to the cluster - - name: logging - import: - path: packages/logging-pgl - - # (Optional) Adds a git server to the cluster - - name: git-server - import: - path: packages/gitea + charts: + - name: redis + namespace: redis-system + version: *version + localPath: ./redis + images: + - &hello_image docker.io/library/redis:latest diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index 2b71b2fae4..ee4641fa79 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -22,15 +22,20 @@ func TestValidateSchema(t *testing.T) { readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { t.Helper() var unmarshalledYaml interface{} - file, err := os.ReadFile(path) + file, _ := os.ReadFile(path) + goyaml.Unmarshal(file, &unmarshalledYaml) + return unmarshalledYaml + } + + readAndUnmarshallZarf := func(t *testing.T, path string) types.ZarfPackage { + t.Helper() + var unmarshalledYaml types.ZarfPackage + file, _ := os.ReadFile(path) goyaml.Unmarshal(file, &unmarshalledYaml) - if err != nil { - return err - } return unmarshalledYaml } - t.Run("Read schema success", func(t *testing.T) { + t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) @@ -40,7 +45,7 @@ func TestValidateSchema(t *testing.T) { } }) - t.Run("Read schema fail", func(t *testing.T) { + t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) @@ -50,32 +55,17 @@ func TestValidateSchema(t *testing.T) { require.EqualError(t, err, errorMessage) }) - t.Run("Read schema yaml extension", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "yaml-extension/zarf.yaml") - zarfSchema := readSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) - require.NoError(t, err) - if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { - t.Errorf("Expected successful validation, got error: %v", got) - } - }) - - t.Run("Read schema yaml extension", func(t *testing.T) { - var unmarshalledYaml types.ZarfPackage - file, _ := os.ReadFile("yaml-extension/zarf.yaml") - goyaml.Unmarshal(file, &unmarshalledYaml) + t.Run("Template in component import success", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshallZarf(t, "successful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) - t.Run("Read schema import-error", func(t *testing.T) { - var unmarshalledYaml types.ZarfPackage - file, _ := os.ReadFile("unsuccessful_validation/zarf.yaml") - goyaml.Unmarshal(file, &unmarshalledYaml) + t.Run("Template in component import failure", func(t *testing.T) { + unmarshalledYaml := readAndUnmarshallZarf(t, "unsuccessful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + ` - - component.2.import.path will not resolve ZARF_PKG_TMPL_* variables - - component.3.import.url will not resolve ZARF_PKG_TMPL_* variables` + errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." require.EqualError(t, err, errorMessage) }) } diff --git a/src/pkg/lint/yaml-extension/zarf.yaml b/src/pkg/lint/yaml-extension/zarf.yaml deleted file mode 100644 index c55c123e75..0000000000 --- a/src/pkg/lint/yaml-extension/zarf.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Plan to have the schema or the lint code allow these soon -# x-env: &env -# environment: -# - CONFIG_KEY -# - EXAMPLE_KEY - -# x-version: &version -# version: 1.0.0 - -kind: ZarfPackageConfig -metadata: - name: hello - <<: &version 1.0.0 - -components: - - name: c1 - required: true - charts: - - name: redis - namespace: redis-system - version: *version - localPath: ./redis - images: - - &hello_image docker.io/library/redis:latest From 347a8d1435e7cfeb5ac77cb96d45e48fe2c36cfb Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 14:09:58 +0000 Subject: [PATCH 34/87] refactor tests --- go.mod | 5 +--- go.sum | 4 ---- main.go | 1 - src/pkg/lint/successful_validation/zarf.yaml | 24 -------------------- src/pkg/lint/validate_schema_test.go | 11 ++++----- 5 files changed, 5 insertions(+), 40 deletions(-) delete mode 100644 src/pkg/lint/successful_validation/zarf.yaml diff --git a/go.mod b/go.mod index 229e3b8b51..958a9962e2 100644 --- a/go.mod +++ b/go.mod @@ -59,8 +59,6 @@ require ( sigs.k8s.io/yaml v1.3.0 ) -require github.com/santhosh-tekuri/jsonschema v1.2.4 - require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect @@ -373,7 +371,6 @@ require ( github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sassoftware/go-rpmutils v0.2.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect @@ -458,7 +455,7 @@ require ( gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 + gopkg.in/yaml.v3 v3.0.1 // indirect inet.af/netaddr v0.0.0-20220811202034-502d2d690317 // indirect k8s.io/apiextensions-apiserver v0.28.2 // indirect k8s.io/apiserver v0.28.2 // indirect diff --git a/go.sum b/go.sum index efb574297d..848584069a 100644 --- a/go.sum +++ b/go.sum @@ -1425,10 +1425,6 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= -github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= -github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= diff --git a/main.go b/main.go index e71065a19b..651ba68a26 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ package main import ( "embed" - _ "embed" "github.com/defenseunicorns/zarf/src/cmd" "github.com/defenseunicorns/zarf/src/config" diff --git a/src/pkg/lint/successful_validation/zarf.yaml b/src/pkg/lint/successful_validation/zarf.yaml deleted file mode 100644 index a8640d1b4c..0000000000 --- a/src/pkg/lint/successful_validation/zarf.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Plan to have the schema or the lint code allow these soon -# x-env: &env -# environment: -# - CONFIG_KEY -# - EXAMPLE_KEY - -# x-version: &version -# version: 1.0.0 - -kind: ZarfPackageConfig -metadata: - name: hello - version: &version 1.0.0 - -components: - - name: c1 - required: true - charts: - - name: redis - namespace: redis-system - version: *version - localPath: ./redis - images: - - &hello_image docker.io/library/redis:latest diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/lint/validate_schema_test.go index ee4641fa79..8c914557d1 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/lint/validate_schema_test.go @@ -27,7 +27,7 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } - readAndUnmarshallZarf := func(t *testing.T, path string) types.ZarfPackage { + readAndUnmarshallZarfPackage := func(t *testing.T, path string) types.ZarfPackage { t.Helper() var unmarshalledYaml types.ZarfPackage file, _ := os.ReadFile(path) @@ -36,13 +36,10 @@ func TestValidateSchema(t *testing.T) { } t.Run("validate schema success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "successful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml(t, "../../../zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) - if got := validateSchema(unmarshalledYaml, zarfSchema); got != nil { - t.Errorf("Expected successful validation, got error: %v", got) - } }) t.Run("validate schema fail", func(t *testing.T) { @@ -56,13 +53,13 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarf(t, "successful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "successful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarf(t, "unsuccessful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "unsuccessful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." From fba2afed609c97c355ecc8a722b7dded4bb76e42 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 14:31:24 +0000 Subject: [PATCH 35/87] refactor e2e, make docs and schema --- .../100-cli-commands/zarf_prepare.md | 1 + .../100-cli-commands/zarf_prepare_lint.md | 31 +++++++++++++ .../100-cli-commands/zarf_tools_helm.md | 1 + .../zarf_tools_helm_cm-push.md | 42 ++++++++++++++++++ docs/3-create-a-zarf-package/4-zarf-schema.md | 10 ++--- src/test/e2e/11_lint_test.go | 4 +- .../packages/11-lint/invalid_yaml/zarf.yaml | 1 + .../packages/11-lint/valid_yaml/zarf.yaml | 43 ------------------- 8 files changed, 81 insertions(+), 52 deletions(-) create mode 100644 docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md create mode 100644 docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md delete mode 100644 src/test/packages/11-lint/valid_yaml/zarf.yaml diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md index d0c4598e7c..c8b2ecd222 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md @@ -27,6 +27,7 @@ Tools to help prepare assets for packaging * [zarf](zarf.md) - DevSecOps for Airgap * [zarf prepare find-images](zarf_prepare_find-images.md) - Evaluates components in a zarf file to identify images specified in their helm charts and manifests * [zarf prepare generate-config](zarf_prepare_generate-config.md) - Generates a config file for Zarf +* [zarf prepare lint](zarf_prepare_lint.md) - Tools for validating schema / ensuring best practices * [zarf prepare patch-git](zarf_prepare_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf prepare sha256sum](zarf_prepare_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md new file mode 100644 index 0000000000..a114cba47f --- /dev/null +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -0,0 +1,31 @@ +# zarf prepare lint + + +Tools for validating schema / ensuring best practices + +``` +zarf prepare lint [ DIRECTORY ] [flags] +``` + +## Options + +``` + -h, --help help for lint +``` + +## Options inherited from parent commands + +``` + -a, --architecture string Architecture for OCI images and Zarf packages + --insecure Allow access to insecure registries and disable other recommended security enforcements such as package checksum and signature validation. This flag should only be used if you have a specific reason and accept the reduced security posture. + -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") + --no-color Disable colors in output + --no-log-file Disable log file creation + --no-progress Disable fancy UI progress bars, spinners, logos, etc + --tmpdir string Specify the temporary directory to use for intermediate files + --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") +``` + +## SEE ALSO + +* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md index a0c5884216..cea6f81d37 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md @@ -31,5 +31,6 @@ Subset of the Helm CLI that includes the repo and dependency commands for managi ## SEE ALSO * [zarf tools](zarf_tools.md) - Collection of additional tools to make airgap easier +* [zarf tools helm cm-push](zarf_tools_helm_cm-push.md) - Please see https://github.com/chartmuseum/helm-push for usage * [zarf tools helm dependency](zarf_tools_helm_dependency.md) - manage a chart's dependencies * [zarf tools helm repo](zarf_tools_helm_repo.md) - add, list, remove, update, and index chart repositories diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md new file mode 100644 index 0000000000..c82bd946ab --- /dev/null +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md @@ -0,0 +1,42 @@ +# zarf tools helm cm-push + + +Please see https://github.com/chartmuseum/helm-push for usage + +## Synopsis + +Push chart package to ChartMuseum + +``` +zarf tools helm cm-push [flags] +``` + +## Options + +``` + -h, --help help for cm-push +``` + +## Options inherited from parent commands + +``` + --burst-limit int client-side default throttling limit (default 100) + --debug enable verbose output + --kube-apiserver string the address and the port for the Kubernetes API server + --kube-as-group stringArray group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --kube-as-user string username to impersonate for the operation + --kube-ca-file string the certificate authority file for the Kubernetes API server connection + --kube-context string name of the kubeconfig context to use + --kube-insecure-skip-tls-verify if true, the Kubernetes API server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kube-tls-server-name string server name to use for Kubernetes API server certificate validation. If it is not provided, the hostname used to contact the server is used + --kube-token string bearer token used for authentication + --kubeconfig string path to the kubeconfig file + -n, --namespace string namespace scope for this request + --registry-config string path to the registry config file + --repository-cache string path to the file containing cached repository indexes + --repository-config string path to the file containing repository names and URLs +``` + +## SEE ALSO + +* [zarf tools helm](zarf_tools_helm.md) - Subset of the Helm CLI included with Zarf to help manage helm charts. diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 241382d6f6..3156977bce 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -783,10 +783,6 @@ Must be one of: | -------- | -------- | | **Type** | `string` | -| Restrictions | | -| --------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | -| **Must match regular expression** | ```^(?!.*###ZARF_PKG_TMPL_).*$``` [Test](https://regex101.com/?regex=%5E%28%3F%21.%2A%23%23%23ZARF_PKG_TMPL_%29.%2A%24) | - @@ -803,9 +799,9 @@ Must be one of: | -------- | -------- | | **Type** | `string` | -| Restrictions | | -| --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| **Must match regular expression** | ```^oci://(?!.*###ZARF_PKG_TMPL_).*$``` [Test](https://regex101.com/?regex=%5Eoci%3A%2F%2F%28%3F%21.%2A%23%23%23ZARF_PKG_TMPL_%29.%2A%24) | +| Restrictions | | +| --------------------------------- | --------------------------------------------------------------------------- | +| **Must match regular expression** | ```^oci://.*$``` [Test](https://regex101.com/?regex=%5Eoci%3A%2F%2F.%2A%24) | diff --git a/src/test/e2e/11_lint_test.go b/src/test/e2e/11_lint_test.go index 14b0865040..96e9cba750 100644 --- a/src/test/e2e/11_lint_test.go +++ b/src/test/e2e/11_lint_test.go @@ -23,8 +23,8 @@ func TestLint(t *testing.T) { t.Run("zarf test lint success", func(t *testing.T) { t.Log("E2E: Test lint on schema success") - path := filepath.Join("src", "test", "packages", "11-lint", "valid_yaml") - _, _, err := e2e.Zarf("prepare", "lint", path) + // This runs lint on the zarf.yaml in the base directory of the repo + _, _, err := e2e.Zarf("prepare", "lint") require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) } diff --git a/src/test/packages/11-lint/invalid_yaml/zarf.yaml b/src/test/packages/11-lint/invalid_yaml/zarf.yaml index 39f1b899da..af38a32cb6 100644 --- a/src/test/packages/11-lint/invalid_yaml/zarf.yaml +++ b/src/test/packages/11-lint/invalid_yaml/zarf.yaml @@ -7,6 +7,7 @@ components: - name: k3s import: pat12312h: packages/distros/k3s + - name: import-test import: path: "###ZARF_PKG_TMPL_ZEBRA###" diff --git a/src/test/packages/11-lint/valid_yaml/zarf.yaml b/src/test/packages/11-lint/valid_yaml/zarf.yaml deleted file mode 100644 index 859813fde5..0000000000 --- a/src/test/packages/11-lint/valid_yaml/zarf.yaml +++ /dev/null @@ -1,43 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - path: packages/distros/k3s - - # This package moves the injector & registries binaries - - name: zarf-injector - required: true - import: - path: packages/zarf-registry - - # Creates the temporary seed-registry - - name: zarf-seed-registry - required: true - import: - path: packages/zarf-registry - - # Creates the permanent registry - - name: zarf-registry - required: true - import: - path: packages/zarf-registry - - # Creates the pod+git mutating webhook - - name: zarf-agent - required: true - import: - path: packages/zarf-agent - - # (Optional) Adds logging to the cluster - - name: logging - import: - path: packages/logging-pgl - - # (Optional) Adds a git server to the cluster - - name: git-server - import: - path: packages/gitea From 84e82c3bffd5cee077460eb15f5aef3d5b967216 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 14:32:45 +0000 Subject: [PATCH 36/87] refactor e2e --- src/test/e2e/11_lint_test.go | 2 +- src/test/packages/11-lint/{invalid_yaml => }/zarf.yaml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/test/packages/11-lint/{invalid_yaml => }/zarf.yaml (100%) diff --git a/src/test/e2e/11_lint_test.go b/src/test/e2e/11_lint_test.go index 96e9cba750..6232a06315 100644 --- a/src/test/e2e/11_lint_test.go +++ b/src/test/e2e/11_lint_test.go @@ -13,7 +13,7 @@ func TestLint(t *testing.T) { t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema fail") - path := filepath.Join("src", "test", "packages", "11-lint", "invalid_yaml") + path := filepath.Join("src", "test", "packages", "11-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") diff --git a/src/test/packages/11-lint/invalid_yaml/zarf.yaml b/src/test/packages/11-lint/zarf.yaml similarity index 100% rename from src/test/packages/11-lint/invalid_yaml/zarf.yaml rename to src/test/packages/11-lint/zarf.yaml From 094a192d7dc66998575a1c9a775577f2f650cfd4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 14:55:02 +0000 Subject: [PATCH 37/87] refactor to use sprintf --- src/pkg/lint/validate_schema.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index a728f00e34..24016553cf 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -2,7 +2,7 @@ package lint import ( "errors" - "strconv" + "fmt" "strings" "github.com/defenseunicorns/zarf/src/types" @@ -27,7 +27,7 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { if !result.Valid() { errorMessage := zarfInvalidPrefix for _, desc := range result.Errors() { - errorMessage = errorMessage + "\n - " + desc.String() + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, desc.String()) } err = errors.New(errorMessage) } @@ -38,16 +38,16 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := " component." + componentWarningStart := "component." for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { - errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + - ".import.path will not resolve ZARF_PKG_TMPL_* variables." + errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage, componentWarningStart, i) valid = false } if strings.Contains(component.Import.URL, zarfTemplateVar) { - errorMessage = errorMessage + componentWarningStart + strconv.Itoa(i) + - ".import.url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage, componentWarningStart, i) valid = false } } From 3b3aa75eae7ed126ab15e1b4589dd28d4037353b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 15:08:24 +0000 Subject: [PATCH 38/87] exporting template var string --- src/pkg/lint/validate_schema.go | 6 +++--- src/pkg/packager/variables.go | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/lint/validate_schema.go index 24016553cf..0b1d92179e 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/lint/validate_schema.go @@ -12,7 +12,7 @@ import ( const ( zarfInvalidPrefix = "zarf.yaml is not valid:" zarfWarningPrefix = "zarf schema warning:" - zarfTemplateVar = "###ZARF_PKG_TMPL_" + ZarfTemplateVar = "###ZARF_PKG_TMPL_" ) func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { @@ -40,12 +40,12 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "component." for i, component := range zarfYaml.Components { - if strings.Contains(component.Import.Path, zarfTemplateVar) { + if strings.Contains(component.Import.Path, ZarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } - if strings.Contains(component.Import.URL, zarfTemplateVar) { + if strings.Contains(component.Import.URL, ZarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index f8f9dd20b4..b9e417ed6a 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" + "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -59,7 +60,7 @@ func (p *Packager) fillActiveTemplate() error { return err } - if err := promptAndSetTemplate("###ZARF_PKG_TMPL_", false); err != nil { + if err := promptAndSetTemplate(lint.ZarfTemplateVar, false); err != nil { return err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility From fa6ff162adbdfa2cb430735b74aa49c50ae6757c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 17:28:46 +0000 Subject: [PATCH 39/87] moved lint into packager --- .pre-commit-config.yaml | 2 +- src/cmd/prepare.go | 14 +++--- src/pkg/lint/lint.go | 43 ------------------- .../validate_schema.go => packager/lint.go} | 41 ++++++++++++++++-- .../lint_test.go} | 6 ++- .../unsuccessful_validation/zarf.yaml | 0 src/pkg/packager/variables.go | 3 +- 7 files changed, 51 insertions(+), 58 deletions(-) delete mode 100644 src/pkg/lint/lint.go rename src/pkg/{lint/validate_schema.go => packager/lint.go} (50%) rename src/pkg/{lint/validate_schema_test.go => packager/lint_test.go} (91%) rename src/pkg/{lint => packager}/unsuccessful_validation/zarf.yaml (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 564e34c92a..a17e0ad46a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,6 +49,6 @@ repos: args: ["--schemafile", "zarf.schema.json"] exclude: | (?x)^( - src/pkg/lint/unsuccessful_validation/zarf.yaml| + src/pkg/packager/unsuccessful_validation/zarf.yaml| src/test/packages/11-lint/.* )$ diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 2a9aab6358..1c5e658eb6 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -15,7 +15,6 @@ import ( "github.com/defenseunicorns/zarf/src/cmd/common" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -212,17 +211,18 @@ var lintCmd = &cobra.Command{ Aliases: []string{"l"}, Short: lang.CmdLintShort, Run: func(cmd *cobra.Command, args []string) { - baseDir := "" if len(args) > 0 { - baseDir = args[0] + pkgConfig.CreateOpts.BaseDir = args[0] } else { - var err error - baseDir, err = os.Getwd() + cwd, err := os.Getwd() if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) } + pkgConfig.CreateOpts.BaseDir = cwd } - err := lint.ValidateZarfSchema(baseDir) + pkgClient := packager.NewOrDie(&pkgConfig) + defer pkgClient.ClearTempPaths() + err := pkgClient.ValidateZarfSchema() if err != nil { message.Fatal(err, err.Error()) } diff --git a/src/pkg/lint/lint.go b/src/pkg/lint/lint.go deleted file mode 100644 index ee119ac663..0000000000 --- a/src/pkg/lint/lint.go +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package lint contains functions for linting the zarf.yaml -package lint - -import ( - "path/filepath" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" -) - -// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid -func ValidateZarfSchema(baseDir string) (err error) { - var typedZarfData types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &typedZarfData); err != nil { - return err - } - - err = checkForVarInComponentImport(typedZarfData) - if err != nil { - message.Warn(err.Error()) - } - - zarfSchema, _ := config.GetSchemaFile() - var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(baseDir, layout.ZarfYAML), &zarfData); err != nil { - return err - } - - err = validateSchema(zarfData, zarfSchema) - - if err != nil { - return err - } - - message.Success("Validation successful") - return nil -} diff --git a/src/pkg/lint/validate_schema.go b/src/pkg/packager/lint.go similarity index 50% rename from src/pkg/lint/validate_schema.go rename to src/pkg/packager/lint.go index 0b1d92179e..939b3b1193 100644 --- a/src/pkg/lint/validate_schema.go +++ b/src/pkg/packager/lint.go @@ -1,10 +1,19 @@ -package lint +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for linting the zarf.yaml +package packager import ( "errors" "fmt" + "path/filepath" "strings" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) @@ -12,9 +21,33 @@ import ( const ( zarfInvalidPrefix = "zarf.yaml is not valid:" zarfWarningPrefix = "zarf schema warning:" - ZarfTemplateVar = "###ZARF_PKG_TMPL_" + zarfTemplateVar = "###ZARF_PKG_TMPL_" ) +// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid +func (p *Packager) ValidateZarfSchema() (err error) { + if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + } + + if err := checkForVarInComponentImport(p.cfg.Pkg); err != nil { + message.Warn(err.Error()) + } + + zarfSchema, _ := config.GetSchemaFile() + var zarfData interface{} + if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { + return err + } + + if err = validateSchema(zarfData, zarfSchema); err != nil { + return err + } + + message.Success("Validation successful") + return nil +} + func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) @@ -40,12 +73,12 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "component." for i, component := range zarfYaml.Components { - if strings.Contains(component.Import.Path, ZarfTemplateVar) { + if strings.Contains(component.Import.Path, zarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } - if strings.Contains(component.Import.URL, ZarfTemplateVar) { + if strings.Contains(component.Import.URL, zarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false diff --git a/src/pkg/lint/validate_schema_test.go b/src/pkg/packager/lint_test.go similarity index 91% rename from src/pkg/lint/validate_schema_test.go rename to src/pkg/packager/lint_test.go index 8c914557d1..1b3320b3c9 100644 --- a/src/pkg/lint/validate_schema_test.go +++ b/src/pkg/packager/lint_test.go @@ -1,4 +1,8 @@ -package lint +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager import ( "os" diff --git a/src/pkg/lint/unsuccessful_validation/zarf.yaml b/src/pkg/packager/unsuccessful_validation/zarf.yaml similarity index 100% rename from src/pkg/lint/unsuccessful_validation/zarf.yaml rename to src/pkg/packager/unsuccessful_validation/zarf.yaml diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index b9e417ed6a..2a07944c1c 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" - "github.com/defenseunicorns/zarf/src/pkg/lint" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -60,7 +59,7 @@ func (p *Packager) fillActiveTemplate() error { return err } - if err := promptAndSetTemplate(lint.ZarfTemplateVar, false); err != nil { + if err := promptAndSetTemplate(zarfTemplateVar, false); err != nil { return err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility From 02a39b737d77b8dbe0af9dde9d4834b7b70ca1f9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 17:36:35 +0000 Subject: [PATCH 40/87] move 11 to 12 --- .pre-commit-config.yaml | 2 +- src/test/e2e/11_lint_test.go | 30 ----------------------------- src/test/packages/11-lint/zarf.yaml | 17 ---------------- 3 files changed, 1 insertion(+), 48 deletions(-) delete mode 100644 src/test/e2e/11_lint_test.go delete mode 100644 src/test/packages/11-lint/zarf.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a17e0ad46a..3217685488 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,5 +50,5 @@ repos: exclude: | (?x)^( src/pkg/packager/unsuccessful_validation/zarf.yaml| - src/test/packages/11-lint/.* + src/test/packages/12-lint/.* )$ diff --git a/src/test/e2e/11_lint_test.go b/src/test/e2e/11_lint_test.go deleted file mode 100644 index 6232a06315..0000000000 --- a/src/test/e2e/11_lint_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package test - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestLint(t *testing.T) { - t.Log("E2E: Lint") - - t.Run("zarf test lint fail", func(t *testing.T) { - t.Log("E2E: Test lint on schema fail") - - path := filepath.Join("src", "test", "packages", "11-lint") - _, stderr, err := e2e.Zarf("prepare", "lint", path) - require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") - require.Contains(t, stderr, "component.1.import.path will not resolve ZARF_PKG_TMPL_* variables") - }) - - t.Run("zarf test lint success", func(t *testing.T) { - t.Log("E2E: Test lint on schema success") - - // This runs lint on the zarf.yaml in the base directory of the repo - _, _, err := e2e.Zarf("prepare", "lint") - require.NoError(t, err, "Expect no error here because the yaml file is following schema") - }) -} diff --git a/src/test/packages/11-lint/zarf.yaml b/src/test/packages/11-lint/zarf.yaml deleted file mode 100644 index af38a32cb6..0000000000 --- a/src/test/packages/11-lint/zarf.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Used to establish a new Zarf cluster - -components: - - name: k3s - import: - pat12312h: packages/distros/k3s - - - name: import-test - import: - path: "###ZARF_PKG_TMPL_ZEBRA###" - - - name: import-url - import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" From 3bd8a7d7810889afe320ac78a3f16b956082db65 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 8 Nov 2023 17:38:13 +0000 Subject: [PATCH 41/87] with files this time --- src/test/e2e/12_lint_test.go | 30 +++++++++++++++++++++++++++++ src/test/packages/12-lint/zarf.yaml | 17 ++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 src/test/e2e/12_lint_test.go create mode 100644 src/test/packages/12-lint/zarf.yaml diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go new file mode 100644 index 0000000000..d0252a82f6 --- /dev/null +++ b/src/test/e2e/12_lint_test.go @@ -0,0 +1,30 @@ +package test + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLint(t *testing.T) { + t.Log("E2E: Lint") + + t.Run("zarf test lint fail", func(t *testing.T) { + t.Log("E2E: Test lint on schema fail") + + path := filepath.Join("src", "test", "packages", "12-lint") + _, stderr, err := e2e.Zarf("prepare", "lint", path) + require.Error(t, err, "Expect error here because the yaml file is not following schema") + require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") + require.Contains(t, stderr, "component.1.import.path will not resolve ZARF_PKG_TMPL_* variables") + }) + + t.Run("zarf test lint success", func(t *testing.T) { + t.Log("E2E: Test lint on schema success") + + // This runs lint on the zarf.yaml in the base directory of the repo + _, _, err := e2e.Zarf("prepare", "lint") + require.NoError(t, err, "Expect no error here because the yaml file is following schema") + }) +} diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml new file mode 100644 index 0000000000..af38a32cb6 --- /dev/null +++ b/src/test/packages/12-lint/zarf.yaml @@ -0,0 +1,17 @@ +kind: ZarfInitConfig +metadata: + name: init + description: Used to establish a new Zarf cluster + +components: + - name: k3s + import: + pat12312h: packages/distros/k3s + + - name: import-test + import: + path: "###ZARF_PKG_TMPL_ZEBRA###" + + - name: import-url + import: + url: "oci://###ZARF_PKG_TMPL_ZEBRA###" From 5013c9f4b06ff789700806f2de7e290e333ad21e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 15:17:52 +0000 Subject: [PATCH 42/87] changing to use more maintained json package --- go.mod | 1 + go.sum | 2 + src/pkg/packager/lint.go | 57 +++++++++++++++++-- src/pkg/packager/lint_test.go | 8 +-- .../unsuccessful_validation/zarf.yaml | 6 +- src/test/e2e/12_lint_test.go | 4 +- 6 files changed, 64 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 958a9962e2..e56edc2bd9 100644 --- a/go.mod +++ b/go.mod @@ -371,6 +371,7 @@ require ( github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/sassoftware/go-rpmutils v0.2.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect diff --git a/go.sum b/go.sum index 848584069a..59465d5175 100644 --- a/go.sum +++ b/go.sum @@ -1425,6 +1425,8 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 939b3b1193..811e43ad07 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -5,6 +5,7 @@ package packager import ( + "bytes" "errors" "fmt" "path/filepath" @@ -15,11 +16,12 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" + "github.com/santhosh-tekuri/jsonschema/v5" "github.com/xeipuuv/gojsonschema" ) const ( - zarfInvalidPrefix = "zarf.yaml is not valid:" + zarfInvalidPrefix = "schema is invalid:" zarfWarningPrefix = "zarf schema warning:" zarfTemplateVar = "###ZARF_PKG_TMPL_" ) @@ -48,7 +50,7 @@ func (p *Packager) ValidateZarfSchema() (err error) { return nil } -func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func validateSchema2(unmarshalledYaml interface{}, jsonSchema []byte) error { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) @@ -71,15 +73,15 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := "component." + componentWarningStart := "component/" for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { - errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage = fmt.Sprintf("%s %s%d/import/path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } if strings.Contains(component.Import.URL, zarfTemplateVar) { - errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage = fmt.Sprintf("%s %s%d/import/url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } @@ -89,3 +91,48 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { } return errors.New(errorMessage) } + +func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { + compiler := jsonschema.NewCompiler() + inMemoryZarfSchema := "schema.json" + + if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(jsonSchema)); err != nil { + return err + } + + schema, err := compiler.Compile(inMemoryZarfSchema) + if err != nil { + return err + } + + if err := schema.Validate(unmarshalledYaml); err != nil { + if validationError, ok := err.(*jsonschema.ValidationError); ok { + allSchemaErrors := printAllCauses(validationError, []error{}) + var errMessage strings.Builder + errMessage.WriteString(zarfInvalidPrefix) + for _, err := range allSchemaErrors { + errMessage.WriteString("\n") + errMessage.WriteString(err.Error()) + } + return errors.New(errMessage.String()) + } + return err + } + + return nil +} + +func printAllCauses(validationErr *jsonschema.ValidationError, errToUser []error) []error { + if validationErr == nil { + return errToUser + } + if validationErr.Causes == nil { + errMessage := fmt.Sprintf(" - %s: %s", validationErr.InstanceLocation, validationErr.Message) + return append(errToUser, fmt.Errorf("%s", errMessage)) + } + + for _, subCause := range validationErr.Causes { + errToUser = printAllCauses(subCause, errToUser) + } + return errToUser +} diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index 1b3320b3c9..0fec409b31 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -51,8 +51,8 @@ func TestValidateSchema(t *testing.T) { zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` - - components.0.import: Additional property not-path is not allowed - - components.1: Additional property not-import is not allowed` + - /components/0/import: additionalProperties 'not-path' not allowed + - /components/1/import/path: expected string, but got number` require.EqualError(t, err, errorMessage) }) @@ -65,8 +65,8 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshallZarfPackage(t, "unsuccessful_validation/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + - "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage := zarfWarningPrefix + " component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + + "component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." require.EqualError(t, err, errorMessage) }) } diff --git a/src/pkg/packager/unsuccessful_validation/zarf.yaml b/src/pkg/packager/unsuccessful_validation/zarf.yaml index 639f1cf462..3c2d75c816 100644 --- a/src/pkg/packager/unsuccessful_validation/zarf.yaml +++ b/src/pkg/packager/unsuccessful_validation/zarf.yaml @@ -8,9 +8,9 @@ components: import: not-path: packages/distros/k3s - - name: second-test-component - not-import: - path: packages/distros/k3s + - name: import-test + import: + path: 123123 - name: import-test import: diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index d0252a82f6..598480a62c 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -16,8 +16,8 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "12-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- components.0.import: Additional property pat12312h is not allowed") - require.Contains(t, stderr, "component.1.import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "- /components/0/import: additionalProperties 'pat12312h' not allowed") + require.Contains(t, stderr, "component/1/import/path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { From 5b67deabf35f9a488d0f2c2d04b8462b73123b1e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 15:19:07 +0000 Subject: [PATCH 43/87] removing old jsonschema package --- go.mod | 4 ++-- src/pkg/packager/lint.go | 21 --------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index e56edc2bd9..e0eb0e7732 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/pterm/pterm v0.12.69 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sergi/go-diff v1.3.1 github.com/sigstore/cosign/v2 v2.2.0 github.com/sigstore/sigstore/pkg/signature/kms/aws v1.7.4 @@ -41,7 +42,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 - github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.14.0 golang.org/x/sync v0.4.0 golang.org/x/term v0.13.0 @@ -371,7 +371,6 @@ require ( github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/sassoftware/go-rpmutils v0.2.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect @@ -416,6 +415,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 811e43ad07..87a80ba05a 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -17,7 +17,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/santhosh-tekuri/jsonschema/v5" - "github.com/xeipuuv/gojsonschema" ) const ( @@ -50,26 +49,6 @@ func (p *Packager) ValidateZarfSchema() (err error) { return nil } -func validateSchema2(unmarshalledYaml interface{}, jsonSchema []byte) error { - schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) - documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) - - result, err := gojsonschema.Validate(schemaLoader, documentLoader) - if err != nil { - return err - } - - if !result.Valid() { - errorMessage := zarfInvalidPrefix - for _, desc := range result.Errors() { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, desc.String()) - } - err = errors.New(errorMessage) - } - - return err -} - func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix From 1e998459c97b1657a457db6f0f4b55c9ef10265e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 15:31:26 +0000 Subject: [PATCH 44/87] changed language --- src/cmd/prepare.go | 2 +- src/config/lang/english.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 1c5e658eb6..6154249036 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -209,7 +209,7 @@ var lintCmd = &cobra.Command{ Use: "lint [ DIRECTORY ]", Args: cobra.MaximumNArgs(1), Aliases: []string{"l"}, - Short: lang.CmdLintShort, + Short: lang.CmdPrepareLintShort, Run: func(cmd *cobra.Command, args []string) { if len(args) > 0 { pkgConfig.CreateOpts.BaseDir = args[0] diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 91edf55868..296b9ad419 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -305,9 +305,6 @@ const ( CmdPackageClusterSourceFallback = "%q does not satisfy any current sources, assuming it is a package deployed to a cluster" CmdPackageInvalidSource = "Unable to identify source from %q: %s" - //zarf lint - CmdLintShort = "Tools for validating schema / ensuring best practices" - // zarf prepare CmdPrepareShort = "Tools to help prepare assets for packaging" @@ -340,6 +337,8 @@ const ( CmdPrepareFlagGitAccount = "User or organization name for the git account that the repos are created under." CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" + CmdPrepareLintShort = "Verifies the package schema" + // zarf tools CmdToolsShort = "Collection of additional tools to make airgap easier" From 94fad1cf693c3ed7fefe79f41d0224508a1dbfa9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 15:32:08 +0000 Subject: [PATCH 45/87] make docs --- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md | 2 +- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md index c8b2ecd222..837e4c03ba 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md @@ -27,7 +27,7 @@ Tools to help prepare assets for packaging * [zarf](zarf.md) - DevSecOps for Airgap * [zarf prepare find-images](zarf_prepare_find-images.md) - Evaluates components in a zarf file to identify images specified in their helm charts and manifests * [zarf prepare generate-config](zarf_prepare_generate-config.md) - Generates a config file for Zarf -* [zarf prepare lint](zarf_prepare_lint.md) - Tools for validating schema / ensuring best practices +* [zarf prepare lint](zarf_prepare_lint.md) - Verifies the package schema * [zarf prepare patch-git](zarf_prepare_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. * [zarf prepare sha256sum](zarf_prepare_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index a114cba47f..4de015b12d 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -1,7 +1,7 @@ # zarf prepare lint -Tools for validating schema / ensuring best practices +Verifies the package schema ``` zarf prepare lint [ DIRECTORY ] [flags] From 4750a1acb73f4adfe20fe78eaafa58b330ae5780 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 16:59:09 +0000 Subject: [PATCH 46/87] refactor tests --- src/pkg/packager/lint_test.go | 34 ++++++++++++------- .../unsuccessful_validation/zarf.yaml | 21 ------------ src/test/e2e/12_lint_test.go | 4 +-- src/test/packages/12-lint/zarf.yaml | 10 ++++-- 4 files changed, 31 insertions(+), 38 deletions(-) delete mode 100644 src/pkg/packager/unsuccessful_validation/zarf.yaml diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index 0fec409b31..cb1012e44f 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -14,28 +14,38 @@ import ( ) func TestValidateSchema(t *testing.T) { - readSchema := func(t *testing.T) []byte { - t.Helper() - zarfSchema, err := os.ReadFile("../../../zarf.schema.json") + readFileFailFatally := func(t *testing.T, path string) []byte { + file, err := os.ReadFile(path) if err != nil { - t.Fatalf("Error reading schema file: %s", err) + t.Errorf("error reading file: %s", err) } - return zarfSchema + return file + } + + readSchema := func(t *testing.T) []byte { + t.Helper() + return readFileFailFatally(t, "../../../zarf.schema.json") } readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { t.Helper() var unmarshalledYaml interface{} - file, _ := os.ReadFile(path) - goyaml.Unmarshal(file, &unmarshalledYaml) + file := readFileFailFatally(t, path) + err := goyaml.Unmarshal(file, &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml %s", err) + } return unmarshalledYaml } readAndUnmarshallZarfPackage := func(t *testing.T, path string) types.ZarfPackage { t.Helper() var unmarshalledYaml types.ZarfPackage - file, _ := os.ReadFile(path) - goyaml.Unmarshal(file, &unmarshalledYaml) + file := readFileFailFatally(t, path) + err := goyaml.Unmarshal(file, &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml %s", err) + } return unmarshalledYaml } @@ -47,7 +57,7 @@ func TestValidateSchema(t *testing.T) { }) t.Run("validate schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "unsuccessful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml(t, "../../test/packages/12-lint/zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` @@ -57,13 +67,13 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "successful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../../zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "unsuccessful_validation/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../test/packages/12-lint/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) errorMessage := zarfWarningPrefix + " component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + "component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." diff --git a/src/pkg/packager/unsuccessful_validation/zarf.yaml b/src/pkg/packager/unsuccessful_validation/zarf.yaml deleted file mode 100644 index 3c2d75c816..0000000000 --- a/src/pkg/packager/unsuccessful_validation/zarf.yaml +++ /dev/null @@ -1,21 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: Testing bad yaml - -components: - - name: first-test-component - import: - not-path: packages/distros/k3s - - - name: import-test - import: - path: 123123 - - - name: import-test - import: - path: "###ZARF_PKG_TMPL_ZEBRA###" - - - name: import-url - import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 598480a62c..c88f10b664 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -16,8 +16,8 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "12-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- /components/0/import: additionalProperties 'pat12312h' not allowed") - require.Contains(t, stderr, "component/1/import/path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "- /components/0/import: additionalProperties 'not-path' not allowed") + require.Contains(t, stderr, "component/2/import/path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index af38a32cb6..3c2d75c816 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -1,12 +1,16 @@ kind: ZarfInitConfig metadata: name: init - description: Used to establish a new Zarf cluster + description: Testing bad yaml components: - - name: k3s + - name: first-test-component import: - pat12312h: packages/distros/k3s + not-path: packages/distros/k3s + + - name: import-test + import: + path: 123123 - name: import-test import: From 167a78dada35c83db7d6a52e6d3c4a3dcbee20c2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 17:37:42 +0000 Subject: [PATCH 47/87] refactor func name --- src/pkg/packager/lint.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 87a80ba05a..edd513762f 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -86,7 +86,7 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { if err := schema.Validate(unmarshalledYaml); err != nil { if validationError, ok := err.(*jsonschema.ValidationError); ok { - allSchemaErrors := printAllCauses(validationError, []error{}) + allSchemaErrors := getChildCauses(validationError, []error{}) var errMessage strings.Builder errMessage.WriteString(zarfInvalidPrefix) for _, err := range allSchemaErrors { @@ -101,7 +101,7 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return nil } -func printAllCauses(validationErr *jsonschema.ValidationError, errToUser []error) []error { +func getChildCauses(validationErr *jsonschema.ValidationError, errToUser []error) []error { if validationErr == nil { return errToUser } @@ -111,7 +111,7 @@ func printAllCauses(validationErr *jsonschema.ValidationError, errToUser []error } for _, subCause := range validationErr.Causes { - errToUser = printAllCauses(subCause, errToUser) + errToUser = getChildCauses(subCause, errToUser) } return errToUser } From 762c34e37a7a54326004af121ee2c42369dbbd7b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 9 Nov 2023 17:48:58 +0000 Subject: [PATCH 48/87] refactor lint --- src/pkg/packager/lint.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index edd513762f..b0853f4d28 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -101,17 +101,17 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { return nil } -func getChildCauses(validationErr *jsonschema.ValidationError, errToUser []error) []error { +func getChildCauses(validationErr *jsonschema.ValidationError, childCauses []error) []error { if validationErr == nil { - return errToUser + return childCauses } if validationErr.Causes == nil { - errMessage := fmt.Sprintf(" - %s: %s", validationErr.InstanceLocation, validationErr.Message) - return append(errToUser, fmt.Errorf("%s", errMessage)) + errMessage := fmt.Errorf(" - %s: %s", validationErr.InstanceLocation, validationErr.Message) + return append(childCauses, errMessage) } for _, subCause := range validationErr.Causes { - errToUser = getChildCauses(subCause, errToUser) + childCauses = getChildCauses(subCause, childCauses) } - return errToUser + return childCauses } From c8a66e281394c601f8da7a4b40db197b63178cf4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 14 Nov 2023 17:48:29 +0000 Subject: [PATCH 49/87] matching style guide better --- src/pkg/packager/lint.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index b0853f4d28..6377c4ac1a 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -85,7 +85,8 @@ func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { } if err := schema.Validate(unmarshalledYaml); err != nil { - if validationError, ok := err.(*jsonschema.ValidationError); ok { + var validationError *jsonschema.ValidationError + if errors.As(err, &validationError) { allSchemaErrors := getChildCauses(validationError, []error{}) var errMessage strings.Builder errMessage.WriteString(zarfInvalidPrefix) From 527c26b13ed371e155e4bfebad54a4fcc0322142 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 14 Nov 2023 19:51:08 +0000 Subject: [PATCH 50/87] removing local helm plugin so docs are correct --- .../100-cli-commands/zarf_tools_helm.md | 1 - .../zarf_tools_helm_cm-push.md | 42 ------------------- src/pkg/packager/lint.go | 4 +- src/pkg/packager/lint_test.go | 4 +- 4 files changed, 5 insertions(+), 46 deletions(-) delete mode 100644 docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md index cea6f81d37..a0c5884216 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm.md @@ -31,6 +31,5 @@ Subset of the Helm CLI that includes the repo and dependency commands for managi ## SEE ALSO * [zarf tools](zarf_tools.md) - Collection of additional tools to make airgap easier -* [zarf tools helm cm-push](zarf_tools_helm_cm-push.md) - Please see https://github.com/chartmuseum/helm-push for usage * [zarf tools helm dependency](zarf_tools_helm_dependency.md) - manage a chart's dependencies * [zarf tools helm repo](zarf_tools_helm_repo.md) - add, list, remove, update, and index chart repositories diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md deleted file mode 100644 index c82bd946ab..0000000000 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_helm_cm-push.md +++ /dev/null @@ -1,42 +0,0 @@ -# zarf tools helm cm-push - - -Please see https://github.com/chartmuseum/helm-push for usage - -## Synopsis - -Push chart package to ChartMuseum - -``` -zarf tools helm cm-push [flags] -``` - -## Options - -``` - -h, --help help for cm-push -``` - -## Options inherited from parent commands - -``` - --burst-limit int client-side default throttling limit (default 100) - --debug enable verbose output - --kube-apiserver string the address and the port for the Kubernetes API server - --kube-as-group stringArray group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --kube-as-user string username to impersonate for the operation - --kube-ca-file string the certificate authority file for the Kubernetes API server connection - --kube-context string name of the kubeconfig context to use - --kube-insecure-skip-tls-verify if true, the Kubernetes API server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kube-tls-server-name string server name to use for Kubernetes API server certificate validation. If it is not provided, the hostname used to contact the server is used - --kube-token string bearer token used for authentication - --kubeconfig string path to the kubeconfig file - -n, --namespace string namespace scope for this request - --registry-config string path to the registry config file - --repository-cache string path to the file containing cached repository indexes - --repository-config string path to the file containing repository names and URLs -``` - -## SEE ALSO - -* [zarf tools helm](zarf_tools_helm.md) - Subset of the Helm CLI included with Zarf to help manage helm charts. diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 6377c4ac1a..0d5aa7e1ac 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -26,6 +26,8 @@ const ( ) // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid + +//Move this out to a validater package func (p *Packager) ValidateZarfSchema() (err error) { if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) @@ -52,7 +54,7 @@ func (p *Packager) ValidateZarfSchema() (err error) { func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := "component/" + componentWarningStart := "/component/" for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d/import/path will not resolve ZARF_PKG_TMPL_* variables.", diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index cb1012e44f..252ca1caff 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -75,8 +75,8 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../test/packages/12-lint/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + - "component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage := zarfWarningPrefix + " /component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + + "/component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." require.EqualError(t, err, errorMessage) }) } From 251b5e8a95595fda14ad75540e23ebc0ade500ca Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 15 Nov 2023 16:24:46 +0000 Subject: [PATCH 51/87] changing back to xeipuuv/gojsonschema --- go.mod | 3 +- go.sum | 2 -- src/pkg/packager/lint.go | 55 +++++++++-------------------------- src/pkg/packager/lint_test.go | 8 ++--- src/test/e2e/12_lint_test.go | 4 +-- 5 files changed, 21 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index a157de6255..2709b50399 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/pterm/pterm v0.12.69 - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sergi/go-diff v1.3.1 github.com/sigstore/cosign/v2 v2.2.1 github.com/sigstore/sigstore/pkg/signature/kms/aws v1.7.5 @@ -42,6 +41,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 + github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.14.0 golang.org/x/sync v0.5.0 golang.org/x/term v0.13.0 @@ -414,7 +414,6 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index d48f22130d..f0d06523ef 100644 --- a/go.sum +++ b/go.sum @@ -1352,8 +1352,6 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 0d5aa7e1ac..15891922aa 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -5,7 +5,6 @@ package packager import ( - "bytes" "errors" "fmt" "path/filepath" @@ -16,7 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" - "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/xeipuuv/gojsonschema" ) const ( @@ -27,7 +26,7 @@ const ( // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid -//Move this out to a validater package +// Move this out to a validater package func (p *Packager) ValidateZarfSchema() (err error) { if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) @@ -54,15 +53,15 @@ func (p *Packager) ValidateZarfSchema() (err error) { func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { valid := true errorMessage := zarfWarningPrefix - componentWarningStart := "/component/" + componentWarningStart := "component." for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, zarfTemplateVar) { - errorMessage = fmt.Sprintf("%s %s%d/import/path will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } if strings.Contains(component.Import.URL, zarfTemplateVar) { - errorMessage = fmt.Sprintf("%s %s%d/import/url will not resolve ZARF_PKG_TMPL_* variables.", + errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } @@ -74,47 +73,21 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { } func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { - compiler := jsonschema.NewCompiler() - inMemoryZarfSchema := "schema.json" + schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) + documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) - if err := compiler.AddResource(inMemoryZarfSchema, bytes.NewReader(jsonSchema)); err != nil { - return err - } - - schema, err := compiler.Compile(inMemoryZarfSchema) + result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { return err } - if err := schema.Validate(unmarshalledYaml); err != nil { - var validationError *jsonschema.ValidationError - if errors.As(err, &validationError) { - allSchemaErrors := getChildCauses(validationError, []error{}) - var errMessage strings.Builder - errMessage.WriteString(zarfInvalidPrefix) - for _, err := range allSchemaErrors { - errMessage.WriteString("\n") - errMessage.WriteString(err.Error()) - } - return errors.New(errMessage.String()) + if !result.Valid() { + errorMessage := zarfInvalidPrefix + for _, desc := range result.Errors() { + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, desc.String()) } - return err + err = errors.New(errorMessage) } - return nil -} - -func getChildCauses(validationErr *jsonschema.ValidationError, childCauses []error) []error { - if validationErr == nil { - return childCauses - } - if validationErr.Causes == nil { - errMessage := fmt.Errorf(" - %s: %s", validationErr.InstanceLocation, validationErr.Message) - return append(childCauses, errMessage) - } - - for _, subCause := range validationErr.Causes { - childCauses = getChildCauses(subCause, childCauses) - } - return childCauses + return err } diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index 252ca1caff..21ab56c059 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -61,8 +61,8 @@ func TestValidateSchema(t *testing.T) { zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` - - /components/0/import: additionalProperties 'not-path' not allowed - - /components/1/import/path: expected string, but got number` + - components.0.import: Additional property not-path is not allowed + - components.1.import.path: Invalid type. Expected: string, given: integer` require.EqualError(t, err, errorMessage) }) @@ -75,8 +75,8 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../test/packages/12-lint/zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " /component/2/import/path will not resolve ZARF_PKG_TMPL_* variables. " + - "/component/3/import/url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." require.EqualError(t, err, errorMessage) }) } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index c88f10b664..0a3c8dc60d 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -16,8 +16,8 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "12-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- /components/0/import: additionalProperties 'not-path' not allowed") - require.Contains(t, stderr, "component/2/import/path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "- components.0.import: Additional property not-path is not allowed") + require.Contains(t, stderr, "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { From 509e71902ec2757ce0477bef8208a3bc034cb72f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 15 Nov 2023 19:37:48 +0000 Subject: [PATCH 52/87] changing lint tests to be from yaml string rather than file --- go.mod | 8 +++--- go.sum | 2 ++ src/pkg/packager/lint.go | 2 +- src/pkg/packager/lint_test.go | 49 ++++++++++++++++++++++++++++++++--- 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 2709b50399..78f2456d85 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/defenseunicorns/zarf -go 1.21.0 +go 1.21.1 -toolchain go1.21.3 +toolchain go1.21.4 require ( cuelang.org/go v0.6.0 @@ -11,6 +11,7 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/anchore/stereoscope v0.0.0-20231027135531-5909e353ee88 github.com/anchore/syft v0.84.1 + github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7 github.com/derailed/k9s v0.27.4 github.com/distribution/reference v0.5.0 github.com/docker/cli v24.0.7+incompatible @@ -41,7 +42,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 - github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.14.0 golang.org/x/sync v0.5.0 golang.org/x/term v0.13.0 @@ -57,6 +57,7 @@ require ( sigs.k8s.io/kustomize/api v0.14.0 sigs.k8s.io/kustomize/kyaml v0.14.3 sigs.k8s.io/yaml v1.4.0 + ) require ( @@ -414,6 +415,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index f0d06523ef..2fb41f61e1 100644 --- a/go.sum +++ b/go.sum @@ -445,6 +445,8 @@ github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hR github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7 h1:PrAGq11REmGDdhHwERlPu7HmQP5rPu8iHiIevlUgiUs= +github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7/go.mod h1:QJUPfuJ1T3ag4AYZr3Cq1VXct3rv94PZOKH+y/zkw80= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY= diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/lint.go index 15891922aa..9a23fda652 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/lint.go @@ -10,12 +10,12 @@ import ( "path/filepath" "strings" + "github.com/defenseunicorns/gojsonschema" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" - "github.com/xeipuuv/gojsonschema" ) const ( diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/lint_test.go index 21ab56c059..62130f5654 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/lint_test.go @@ -13,6 +13,29 @@ import ( "github.com/stretchr/testify/require" ) +const brokenSchemaZarfPackage = ` +kind: ZarfInitConfig +metadata: + name: init + description: Testing bad yaml + +components: +- name: first-test-component + import: + not-path: packages/distros/k3s +- name: import-test + import: + path: 123123 + +- name: import-test + import: + path: "###ZARF_PKG_TMPL_ZEBRA###" + +- name: import-url + import: + url: "oci://###ZARF_PKG_TMPL_ZEBRA###" +` + func TestValidateSchema(t *testing.T) { readFileFailFatally := func(t *testing.T, path string) []byte { file, err := os.ReadFile(path) @@ -33,7 +56,17 @@ func TestValidateSchema(t *testing.T) { file := readFileFailFatally(t, path) err := goyaml.Unmarshal(file, &unmarshalledYaml) if err != nil { - t.Errorf("error unmarshalling yaml %s", err) + t.Errorf("error unmarshalling yaml %v", err) + } + return unmarshalledYaml + } + + readAndUnmarshallYamlString := func(t *testing.T, yamlString string) interface{} { + t.Helper() + var unmarshalledYaml interface{} + err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml string %v", err) } return unmarshalledYaml } @@ -49,6 +82,16 @@ func TestValidateSchema(t *testing.T) { return unmarshalledYaml } + readAndUnmarshallZarfPackageString := func(t *testing.T, yamlString string) types.ZarfPackage { + t.Helper() + var unmarshalledYaml types.ZarfPackage + err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml %v", err) + } + return unmarshalledYaml + } + t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml(t, "../../../zarf.yaml") zarfSchema := readSchema(t) @@ -57,7 +100,7 @@ func TestValidateSchema(t *testing.T) { }) t.Run("validate schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "../../test/packages/12-lint/zarf.yaml") + unmarshalledYaml := readAndUnmarshallYamlString(t, brokenSchemaZarfPackage) zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` @@ -73,7 +116,7 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../test/packages/12-lint/zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackageString(t, brokenSchemaZarfPackage) err := checkForVarInComponentImport(unmarshalledYaml) errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." From 5a42ffa9971ffba2cabe74a2b6c12553940038f9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 16:41:37 +0000 Subject: [PATCH 53/87] adding validator, moving to defenseunicorns/jsonschema --- go.mod | 6 ++--- go.sum | 7 ++---- src/cmd/prepare.go | 12 +++++----- .../validater_test.go} | 4 ++-- .../{lint.go => validator/validator.go} | 24 ++++++++++--------- src/pkg/packager/variables.go | 3 ++- 6 files changed, 28 insertions(+), 28 deletions(-) rename src/pkg/packager/{lint_test.go => validator/validater_test.go} (96%) rename src/pkg/packager/{lint.go => validator/validator.go} (74%) diff --git a/go.mod b/go.mod index 78f2456d85..3931c1ca09 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.21.1 toolchain go1.21.4 +replace github.com/xeipuuv/gojsonschema => github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 + require ( cuelang.org/go v0.6.0 github.com/AlecAivazis/survey/v2 v2.3.7 @@ -11,7 +13,6 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/anchore/stereoscope v0.0.0-20231027135531-5909e353ee88 github.com/anchore/syft v0.84.1 - github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7 github.com/derailed/k9s v0.27.4 github.com/distribution/reference v0.5.0 github.com/docker/cli v24.0.7+incompatible @@ -42,6 +43,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 + github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.14.0 golang.org/x/sync v0.5.0 golang.org/x/term v0.13.0 @@ -57,7 +59,6 @@ require ( sigs.k8s.io/kustomize/api v0.14.0 sigs.k8s.io/kustomize/kyaml v0.14.3 sigs.k8s.io/yaml v1.4.0 - ) require ( @@ -415,7 +416,6 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index 2fb41f61e1..279b0e1f91 100644 --- a/go.sum +++ b/go.sum @@ -445,8 +445,8 @@ github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hR github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7 h1:PrAGq11REmGDdhHwERlPu7HmQP5rPu8iHiIevlUgiUs= -github.com/defenseunicorns/gojsonschema v0.0.0-20231115180840-cc7edfd762d7/go.mod h1:QJUPfuJ1T3ag4AYZr3Cq1VXct3rv94PZOKH+y/zkw80= +github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 h1:gwevOZ0fxT2nzM9hrtdPbsiOHjFqDRIYMzJHba3/G6Q= +github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6/go.mod h1:StKLYMmPj1R5yIs6CK49EkcW1TvUYuw5Vri+LRk7Dy8= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY= @@ -1528,13 +1528,10 @@ github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3k github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 6154249036..9403e820c3 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" + validator "github.com/defenseunicorns/zarf/src/pkg/packager/validator" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -211,18 +212,17 @@ var lintCmd = &cobra.Command{ Aliases: []string{"l"}, Short: lang.CmdPrepareLintShort, Run: func(cmd *cobra.Command, args []string) { + baseDir := "" if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] + baseDir = args[0] } else { - cwd, err := os.Getwd() + var err error + baseDir, err = os.Getwd() if err != nil { message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) } - pkgConfig.CreateOpts.BaseDir = cwd } - pkgClient := packager.NewOrDie(&pkgConfig) - defer pkgClient.ClearTempPaths() - err := pkgClient.ValidateZarfSchema() + err := validator.ValidateZarfSchema(baseDir) if err != nil { message.Fatal(err, err.Error()) } diff --git a/src/pkg/packager/lint_test.go b/src/pkg/packager/validator/validater_test.go similarity index 96% rename from src/pkg/packager/lint_test.go rename to src/pkg/packager/validator/validater_test.go index 62130f5654..1a82ef7bf8 100644 --- a/src/pkg/packager/lint_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager +// Package validator contains functions for interacting with, managing and deploying Zarf packages. +package validator import ( "os" diff --git a/src/pkg/packager/lint.go b/src/pkg/packager/validator/validator.go similarity index 74% rename from src/pkg/packager/lint.go rename to src/pkg/packager/validator/validator.go index 9a23fda652..c3dc525770 100644 --- a/src/pkg/packager/lint.go +++ b/src/pkg/packager/validator/validator.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for linting the zarf.yaml -package packager +// Package validator contains functions for linting the zarf.yaml +package validator import ( "errors" @@ -10,35 +10,37 @@ import ( "path/filepath" "strings" - "github.com/defenseunicorns/gojsonschema" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" + "github.com/xeipuuv/gojsonschema" ) const ( zarfInvalidPrefix = "schema is invalid:" zarfWarningPrefix = "zarf schema warning:" - zarfTemplateVar = "###ZARF_PKG_TMPL_" + ZarfTemplateVar = "###ZARF_PKG_TMPL_" ) // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid // Move this out to a validater package -func (p *Packager) ValidateZarfSchema() (err error) { - if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) +func ValidateZarfSchema(path string) (err error) { + var zarfTypedData types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { + return err } - if err := checkForVarInComponentImport(p.cfg.Pkg); err != nil { + if err := checkForVarInComponentImport(zarfTypedData); err != nil { message.Warn(err.Error()) } zarfSchema, _ := config.GetSchemaFile() + var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML), &zarfData); err != nil { + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { return err } @@ -55,12 +57,12 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "component." for i, component := range zarfYaml.Components { - if strings.Contains(component.Import.Path, zarfTemplateVar) { + if strings.Contains(component.Import.Path, ZarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } - if strings.Contains(component.Import.URL, zarfTemplateVar) { + if strings.Contains(component.Import.URL, ZarfTemplateVar) { errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 2a07944c1c..0a7c411fbf 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" + validator "github.com/defenseunicorns/zarf/src/pkg/packager/validator" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -59,7 +60,7 @@ func (p *Packager) fillActiveTemplate() error { return err } - if err := promptAndSetTemplate(zarfTemplateVar, false); err != nil { + if err := promptAndSetTemplate(validator.ZarfTemplateVar, false); err != nil { return err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility From 48acf97c4403ec06a0e4d2166f9f469bc3e94f9f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 16:50:00 +0000 Subject: [PATCH 54/87] fix file path unit tests --- src/pkg/packager/validator/validater_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 1a82ef7bf8..c2f7dda613 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -47,7 +47,7 @@ func TestValidateSchema(t *testing.T) { readSchema := func(t *testing.T) []byte { t.Helper() - return readFileFailFatally(t, "../../../zarf.schema.json") + return readFileFailFatally(t, "../../../../zarf.schema.json") } readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { @@ -93,7 +93,7 @@ func TestValidateSchema(t *testing.T) { } t.Run("validate schema success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "../../../zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml(t, "../../../../zarf.yaml") zarfSchema := readSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) @@ -110,7 +110,7 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../../zarf.yaml") + unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../../../zarf.yaml") err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) From 7d695f90c5aef19a06a07c5e379c2ff04efbec69 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 16:59:28 +0000 Subject: [PATCH 55/87] moving some magic strings into types --- src/pkg/packager/validator/validator.go | 5 ++--- src/pkg/packager/variables.go | 9 ++++----- src/types/runtime.go | 5 +++++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index c3dc525770..e290c1cf34 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -21,7 +21,6 @@ import ( const ( zarfInvalidPrefix = "schema is invalid:" zarfWarningPrefix = "zarf schema warning:" - ZarfTemplateVar = "###ZARF_PKG_TMPL_" ) // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid @@ -57,12 +56,12 @@ func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { errorMessage := zarfWarningPrefix componentWarningStart := "component." for i, component := range zarfYaml.Components { - if strings.Contains(component.Import.Path, ZarfTemplateVar) { + if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false } - if strings.Contains(component.Import.URL, ZarfTemplateVar) { + if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", errorMessage, componentWarningStart, i) valid = false diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 0a7c411fbf..2caa135907 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" - validator "github.com/defenseunicorns/zarf/src/pkg/packager/validator" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -60,16 +59,16 @@ func (p *Packager) fillActiveTemplate() error { return err } - if err := promptAndSetTemplate(validator.ZarfTemplateVar, false); err != nil { + if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { return err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility - if err := promptAndSetTemplate("###ZARF_PKG_VAR_", true); err != nil { + if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { return err } // Add special variable for the current package architecture - templateMap["###ZARF_PKG_ARCH###"] = p.arch + templateMap[types.ZarfPackageArch] = p.arch return utils.ReloadYamlTemplate(&p.cfg.Pkg, templateMap) } @@ -132,7 +131,7 @@ func (p *Packager) findComponentTemplatesAndReload() error { // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value for i, component := range p.cfg.Pkg.Components { mappings := map[string]string{} - mappings["###ZARF_COMPONENT_NAME###"] = component.Name + mappings[types.ZarfComponentName] = component.Name err := utils.ReloadYamlTemplate(&p.cfg.Pkg.Components[i], mappings) if err != nil { return err diff --git a/src/types/runtime.go b/src/types/runtime.go index b9546c1658..b4a0653033 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -9,6 +9,11 @@ const ( RawVariableType VariableType = "raw" // FileVariableType is a type for a Zarf package variable that loads its contents from a file FileVariableType VariableType = "file" + + ZarfPackageTemplatePrefix = "###ZARF_PKG_TMPL_" + ZarfPackageVariablePrefix = "###ZARF_PKG_VAR_" + ZarfPackageArch = "###ZARF_PKG_ARCH###" + ZarfComponentName = "###ZARF_COMPONENT_NAME###" ) // VariableType represents a type of a Zarf package variable From 18069573bb50255d85b1f5dd106b09fa2f64f667 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 17:00:45 +0000 Subject: [PATCH 56/87] fix comment --- src/pkg/packager/validator/validator.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index e290c1cf34..929e1980cf 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -24,8 +24,6 @@ const ( ) // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid - -// Move this out to a validater package func ValidateZarfSchema(path string) (err error) { var zarfTypedData types.ZarfPackage if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { From 269c118fccb8c54a74a1d6700604c7ec405726ef Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 21:25:44 +0000 Subject: [PATCH 57/87] refactoring --- .pre-commit-config.yaml | 1 - src/cmd/prepare.go | 2 +- src/pkg/packager/validator/validater_test.go | 84 +++++++------------- src/pkg/packager/validator/validator.go | 4 +- src/types/runtime.go | 3 + 5 files changed, 35 insertions(+), 59 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3217685488..9b3caa47f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,6 +49,5 @@ repos: args: ["--schemafile", "zarf.schema.json"] exclude: | (?x)^( - src/pkg/packager/unsuccessful_validation/zarf.yaml| src/test/packages/12-lint/.* )$ diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 9403e820c3..03e9cabfd0 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -17,7 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" - validator "github.com/defenseunicorns/zarf/src/pkg/packager/validator" + "github.com/defenseunicorns/zarf/src/pkg/packager/validator" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index c2f7dda613..021ce772c8 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package validator contains functions for interacting with, managing and deploying Zarf packages. +// Package validator contains functions for verifying zarf yaml files are valid package validator import ( @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" ) -const brokenSchemaZarfPackage = ` +const badZarfPackage = ` kind: ZarfInitConfig metadata: name: init @@ -36,72 +36,46 @@ components: url: "oci://###ZARF_PKG_TMPL_ZEBRA###" ` -func TestValidateSchema(t *testing.T) { - readFileFailFatally := func(t *testing.T, path string) []byte { - file, err := os.ReadFile(path) - if err != nil { - t.Errorf("error reading file: %s", err) - } - return file - } - - readSchema := func(t *testing.T) []byte { - t.Helper() - return readFileFailFatally(t, "../../../../zarf.schema.json") - } - - readAndUnmarshalYaml := func(t *testing.T, path string) interface{} { - t.Helper() - var unmarshalledYaml interface{} - file := readFileFailFatally(t, path) - err := goyaml.Unmarshal(file, &unmarshalledYaml) - if err != nil { - t.Errorf("error unmarshalling yaml %v", err) - } - return unmarshalledYaml - } +const goodZarfPackage = ` +kind: ZarfPackageConfig +metadata: + name: good-zarf-package - readAndUnmarshallYamlString := func(t *testing.T, yamlString string) interface{} { - t.Helper() - var unmarshalledYaml interface{} - err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) - if err != nil { - t.Errorf("error unmarshalling yaml string %v", err) - } - return unmarshalledYaml - } +components: + - name: baseline + required: true +` - readAndUnmarshallZarfPackage := func(t *testing.T, path string) types.ZarfPackage { - t.Helper() - var unmarshalledYaml types.ZarfPackage - file := readFileFailFatally(t, path) - err := goyaml.Unmarshal(file, &unmarshalledYaml) - if err != nil { - t.Errorf("error unmarshalling yaml %s", err) - } - return unmarshalledYaml +func readAndUnmarshalYaml[T interface{}](t *testing.T, yamlString string) T { + t.Helper() + var unmarshalledYaml T + err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) + if err != nil { + t.Errorf("error unmarshalling yaml %v", err) } + return unmarshalledYaml +} - readAndUnmarshallZarfPackageString := func(t *testing.T, yamlString string) types.ZarfPackage { +func TestValidateSchema(t *testing.T) { + getZarfSchema := func(t *testing.T) []byte { t.Helper() - var unmarshalledYaml types.ZarfPackage - err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) + file, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { - t.Errorf("error unmarshalling yaml %v", err) + t.Errorf("error reading file: %s", err) } - return unmarshalledYaml + return file } t.Run("validate schema success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml(t, "../../../../zarf.yaml") - zarfSchema := readSchema(t) + unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) + zarfSchema := getZarfSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) require.NoError(t, err) }) t.Run("validate schema fail", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallYamlString(t, brokenSchemaZarfPackage) - zarfSchema := readSchema(t) + unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) + zarfSchema := getZarfSchema(t) err := validateSchema(unmarshalledYaml, zarfSchema) errorMessage := zarfInvalidPrefix + ` - components.0.import: Additional property not-path is not allowed @@ -110,13 +84,13 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Template in component import success", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackage(t, "../../../../zarf.yaml") + unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) err := checkForVarInComponentImport(unmarshalledYaml) require.NoError(t, err) }) t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshallZarfPackageString(t, brokenSchemaZarfPackage) + unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) err := checkForVarInComponentImport(unmarshalledYaml) errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 929e1980cf..d4e1eacaa2 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package validator contains functions for linting the zarf.yaml +// Package validator contains functions for verifying zarf yaml files are valid package validator import ( @@ -45,7 +45,7 @@ func ValidateZarfSchema(path string) (err error) { return err } - message.Success("Validation successful") + message.Success(fmt.Sprintf("Schema validation successful for %q", zarfTypedData.Metadata.Name)) return nil } diff --git a/src/types/runtime.go b/src/types/runtime.go index b4a0653033..0cd53e18a0 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -9,7 +9,10 @@ const ( RawVariableType VariableType = "raw" // FileVariableType is a type for a Zarf package variable that loads its contents from a file FileVariableType VariableType = "file" +) +// Zarf looks for these strings in zarf.yaml to make dynamic changes +const ( ZarfPackageTemplatePrefix = "###ZARF_PKG_TMPL_" ZarfPackageVariablePrefix = "###ZARF_PKG_VAR_" ZarfPackageArch = "###ZARF_PKG_ARCH###" From 2e785cf42bdcdfc1ba45035803c49f350fafc2c9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Nov 2023 21:33:18 +0000 Subject: [PATCH 58/87] refactor validator --- src/pkg/packager/validator/validater_test.go | 4 ++-- src/pkg/packager/validator/validator.go | 20 ++++++++------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 021ce772c8..6c07a8df93 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -92,8 +92,8 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables. " + - "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables." + errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables, " + + "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables" require.EqualError(t, err, errorMessage) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index d4e1eacaa2..0dd998b107 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -50,25 +50,21 @@ func ValidateZarfSchema(path string) (err error) { } func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { - valid := true - errorMessage := zarfWarningPrefix - componentWarningStart := "component." + var errorMessages []string for i, component := range zarfYaml.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - errorMessage = fmt.Sprintf("%s %s%d.import.path will not resolve ZARF_PKG_TMPL_* variables.", - errorMessage, componentWarningStart, i) - valid = false + errorMessages = append(errorMessages, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - errorMessage = fmt.Sprintf("%s %s%d.import.url will not resolve ZARF_PKG_TMPL_* variables.", - errorMessage, componentWarningStart, i) - valid = false + errorMessages = append(errorMessages, fmt.Sprintf("component.%d.import.url will not resolve ZARF_PKG_TMPL_* variables", i)) } } - if valid { - return nil + + if len(errorMessages) > 0 { + return fmt.Errorf("%s %s", zarfWarningPrefix, strings.Join(errorMessages, ", ")) } - return errors.New(errorMessage) + + return nil } func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { From 743ecd715327a642940587ecfbca898977671be2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 14:26:01 +0000 Subject: [PATCH 59/87] WIP introducing validator struct --- src/pkg/packager/validator/validater_test.go | 49 ++++++++--- src/pkg/packager/validator/validator.go | 91 +++++++++++--------- 2 files changed, 88 insertions(+), 52 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 6c07a8df93..3d21b7f0f2 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -5,6 +5,7 @@ package validator import ( + "fmt" "os" "testing" @@ -69,31 +70,55 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) zarfSchema := getZarfSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) + validator := Validator{} + validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) require.NoError(t, err) + require.Empty(t, validator.errors) }) t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) zarfSchema := getZarfSchema(t) - err := validateSchema(unmarshalledYaml, zarfSchema) - errorMessage := zarfInvalidPrefix + ` - - components.0.import: Additional property not-path is not allowed - - components.1.import.path: Invalid type. Expected: string, given: integer` - require.EqualError(t, err, errorMessage) + validator := Validator{} + validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) + // errorMessage := zarfInvalidPrefix + ` + // - components.0.import: Additional property not-path is not allowed + // - components.1.import.path: Invalid type. Expected: string, given: integer` + // require.EqualError(t, err, errorMessage) + require.NoError(t, err) + require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") + require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") }) t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) - err := checkForVarInComponentImport(unmarshalledYaml) - require.NoError(t, err) + validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) + require.Empty(t, validator) }) t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - err := checkForVarInComponentImport(unmarshalledYaml) - errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables, " + - "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables" - require.EqualError(t, err, errorMessage) + validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) + // errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables, " + + // "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables" + // require.EqualError(t, err, errorMessage) + require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") + }) + + t.Run("Validator Error formatting", func(t *testing.T) { + error1 := "components.0.import: Additional property not-path is not allowed" + error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" + validator := Validator{errors: []string{error1, error2}} + errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) + require.EqualError(t, validator.getFormmatedError(), errorMessage) + }) + + t.Run("Validator Warning formatting", func(t *testing.T) { + warning1 := "components.0.import: Additional property not-path is not allowed" + warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" + validator := Validator{warnings: []string{warning1, warning2}} + message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) + require.Equal(t, validator.getFormmatedWarning(), message) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 0dd998b107..8848d3e67c 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -7,82 +7,93 @@ package validator import ( "errors" "fmt" - "path/filepath" "strings" - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) const ( - zarfInvalidPrefix = "schema is invalid:" - zarfWarningPrefix = "zarf schema warning:" + validatorInvalidPrefix = "schema is invalid:" + validatorWarningPrefix = "zarf schema warning:" ) -// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid -func ValidateZarfSchema(path string) (err error) { - var zarfTypedData types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { - return err - } +type Validator struct { + warnings []string + errors []string +} - if err := checkForVarInComponentImport(zarfTypedData); err != nil { - message.Warn(err.Error()) +func (v Validator) getFormmatedError() error { + if len(v.errors) == 0 { + return nil } - - zarfSchema, _ := config.GetSchemaFile() - - var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { - return err + errorMessage := validatorInvalidPrefix + for _, errorStr := range v.errors { + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr) } + return errors.New(errorMessage) +} - if err = validateSchema(zarfData, zarfSchema); err != nil { - return err +func (v Validator) getFormmatedWarning() string { + if len(v.warnings) == 0 { + return "" } + return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) +} - message.Success(fmt.Sprintf("Schema validation successful for %q", zarfTypedData.Metadata.Name)) +// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid +func ValidateZarfSchema(path string) (err error) { + // var zarfTypedData types.ZarfPackage + // if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { + // return err + // } + + // if err := checkForVarInComponentImport(zarfTypedData); err != nil { + // message.Warn("") + // } + + // zarfSchema, _ := config.GetSchemaFile() + + // var zarfData interface{} + // if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { + // return err + // } + // var validator Validator + // if validator, err = validateSchema(nil,zarfData, zarfSchema); err != nil { + // return err + // } + + // message.Success(fmt.Sprintf("Schema validation successful for %q", zarfTypedData.Metadata.Name)) return nil } -func checkForVarInComponentImport(zarfYaml types.ZarfPackage) error { - var errorMessages []string - for i, component := range zarfYaml.Components { +func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPackage) Validator { + for i, component := range zarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - errorMessages = append(errorMessages, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - errorMessages = append(errorMessages, fmt.Sprintf("component.%d.import.url will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.url will not resolve ZARF_PKG_TMPL_* variables", i)) } } - if len(errorMessages) > 0 { - return fmt.Errorf("%s %s", zarfWarningPrefix, strings.Join(errorMessages, ", ")) - } - - return nil + return validator } -func validateSchema(unmarshalledYaml interface{}, jsonSchema []byte) error { +func validateSchema(validator Validator, unmarshalledYaml interface{}, jsonSchema []byte) (Validator, error) { schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { - return err + return validator, err } if !result.Valid() { - errorMessage := zarfInvalidPrefix for _, desc := range result.Errors() { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, desc.String()) + validator.errors = append(validator.errors, desc.String()) } - err = errors.New(errorMessage) } - return err + return validator, err } From d240e86a7a9d89e702b9bb5f603ec539f3078ea0 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 15:54:59 +0000 Subject: [PATCH 60/87] tests working with refactored validator, still more refactoring todo --- src/cmd/prepare.go | 12 +++- src/pkg/packager/validator/validater_test.go | 11 +-- src/pkg/packager/validator/validator.go | 70 ++++++++++++-------- 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 03e9cabfd0..60db3cf4f4 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -222,10 +222,20 @@ var lintCmd = &cobra.Command{ message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) } } - err := validator.ValidateZarfSchema(baseDir) + validator, err := validator.ValidateZarfSchema(baseDir) if err != nil { message.Fatal(err, err.Error()) } + if validator.HasWarnings() { + message.Warn(validator.GetFormmatedWarning()) + } + if validator.HasErrors() { + message.Fatal(validator.GetFormmatedError(), validator.GetFormmatedError().Error()) + } + if !validator.HasWarnings() && !validator.HasErrors() { + message.Success(validator.GetFormmatedSuccess()) + } + }, } diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 3d21b7f0f2..297b68ea26 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -81,10 +81,6 @@ func TestValidateSchema(t *testing.T) { zarfSchema := getZarfSchema(t) validator := Validator{} validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) - // errorMessage := zarfInvalidPrefix + ` - // - components.0.import: Additional property not-path is not allowed - // - components.1.import.path: Invalid type. Expected: string, given: integer` - // require.EqualError(t, err, errorMessage) require.NoError(t, err) require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") @@ -99,9 +95,6 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) - // errorMessage := zarfWarningPrefix + " component.2.import.path will not resolve ZARF_PKG_TMPL_* variables, " + - // "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables" - // require.EqualError(t, err, errorMessage) require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") }) @@ -111,7 +104,7 @@ func TestValidateSchema(t *testing.T) { error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{errors: []string{error1, error2}} errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) - require.EqualError(t, validator.getFormmatedError(), errorMessage) + require.EqualError(t, validator.GetFormmatedError(), errorMessage) }) t.Run("Validator Warning formatting", func(t *testing.T) { @@ -119,6 +112,6 @@ func TestValidateSchema(t *testing.T) { warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{warnings: []string{warning1, warning2}} message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.getFormmatedWarning(), message) + require.Equal(t, validator.GetFormmatedWarning(), message) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 8848d3e67c..60a5eb3f98 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -7,8 +7,12 @@ package validator import ( "errors" "fmt" + "path/filepath" "strings" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) @@ -19,11 +23,12 @@ const ( ) type Validator struct { - warnings []string - errors []string + warnings []string + errors []string + jsonSchema []byte } -func (v Validator) getFormmatedError() error { +func (v Validator) GetFormmatedError() error { if len(v.errors) == 0 { return nil } @@ -34,37 +39,48 @@ func (v Validator) getFormmatedError() error { return errors.New(errorMessage) } -func (v Validator) getFormmatedWarning() string { +func (v Validator) GetFormmatedWarning() string { if len(v.warnings) == 0 { return "" } return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) } +func (v Validator) GetFormmatedSuccess() string { + return fmt.Sprintf("Schema validation successful for %q", "init") +} + +func (v Validator) HasWarnings() bool { + return !(len(v.warnings) == 0) +} + +func (v Validator) HasErrors() bool { + return !(len(v.errors) == 0) +} + // ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid -func ValidateZarfSchema(path string) (err error) { - // var zarfTypedData types.ZarfPackage - // if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { - // return err - // } - - // if err := checkForVarInComponentImport(zarfTypedData); err != nil { - // message.Warn("") - // } - - // zarfSchema, _ := config.GetSchemaFile() - - // var zarfData interface{} - // if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { - // return err - // } - // var validator Validator - // if validator, err = validateSchema(nil,zarfData, zarfSchema); err != nil { - // return err - // } - - // message.Success(fmt.Sprintf("Schema validation successful for %q", zarfTypedData.Metadata.Name)) - return nil +func ValidateZarfSchema(path string) (Validator, error) { + validator := Validator{} + var err error + var zarfTypedData types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { + return validator, err + } + + validator = checkForVarInComponentImport(validator, zarfTypedData) + + zarfSchema, _ := config.GetSchemaFile() + + var zarfData interface{} + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { + return validator, err + } + + if validator, err = validateSchema(validator, zarfData, zarfSchema); err != nil { + return validator, err + } + + return validator, nil } func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPackage) Validator { From a02647a8eee8b8cb8c813d99687c76947ae45648 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 15:58:58 +0000 Subject: [PATCH 61/87] refactoring --- src/pkg/packager/validator/validater_test.go | 10 ++++------ src/pkg/packager/validator/validator.go | 12 +++++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 297b68ea26..4a427e109a 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -69,18 +69,16 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) - zarfSchema := getZarfSchema(t) - validator := Validator{} - validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) + validator := Validator{jsonSchema: getZarfSchema(t)} + validator, err := validateSchema(validator, unmarshalledYaml) require.NoError(t, err) require.Empty(t, validator.errors) }) t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) - zarfSchema := getZarfSchema(t) - validator := Validator{} - validator, err := validateSchema(validator, unmarshalledYaml, zarfSchema) + validator := Validator{jsonSchema: getZarfSchema(t)} + validator, err := validateSchema(validator, unmarshalledYaml) require.NoError(t, err) require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 60a5eb3f98..dc9da9c2ed 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -58,7 +58,7 @@ func (v Validator) HasErrors() bool { return !(len(v.errors) == 0) } -// ValidateZarfSchema a zarf file against the zarf schema, returns an error if the file is invalid +// ValidateZarfSchema validates a zarf file against the zarf schema, returns a validator with warnings or errors if they exist func ValidateZarfSchema(path string) (Validator, error) { validator := Validator{} var err error @@ -69,14 +69,16 @@ func ValidateZarfSchema(path string) (Validator, error) { validator = checkForVarInComponentImport(validator, zarfTypedData) - zarfSchema, _ := config.GetSchemaFile() + if validator.jsonSchema, err = config.GetSchemaFile(); err != nil { + return validator, err + } var zarfData interface{} if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { return validator, err } - if validator, err = validateSchema(validator, zarfData, zarfSchema); err != nil { + if validator, err = validateSchema(validator, zarfData); err != nil { return validator, err } @@ -96,8 +98,8 @@ func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPac return validator } -func validateSchema(validator Validator, unmarshalledYaml interface{}, jsonSchema []byte) (Validator, error) { - schemaLoader := gojsonschema.NewBytesLoader(jsonSchema) +func validateSchema(validator Validator, unmarshalledYaml interface{}) (Validator, error) { + schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) result, err := gojsonschema.Validate(schemaLoader, documentLoader) From a9eb84f9510e7c3edbc03c44d8583706134a6ee7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 16:48:14 +0000 Subject: [PATCH 62/87] validator refactor --- src/cmd/prepare.go | 23 +++++----- src/pkg/packager/validator/validater_test.go | 18 ++++---- src/pkg/packager/validator/validator.go | 46 +++++++++++--------- 3 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 60db3cf4f4..cb3a476b7f 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -206,6 +206,18 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } +func displayFormattedMessage(validator validator.Validator) { + if validator.HasWarnings() { + message.Warn(validator.GetFormatedWarning()) + } + if validator.HasErrors() { + message.Fatal(validator.GetFormatedError(), validator.GetFormatedError().Error()) + } + if validator.IsSuccess() { + message.Success(validator.GetFormatedSuccess()) + } +} + var lintCmd = &cobra.Command{ Use: "lint [ DIRECTORY ]", Args: cobra.MaximumNArgs(1), @@ -226,16 +238,7 @@ var lintCmd = &cobra.Command{ if err != nil { message.Fatal(err, err.Error()) } - if validator.HasWarnings() { - message.Warn(validator.GetFormmatedWarning()) - } - if validator.HasErrors() { - message.Fatal(validator.GetFormmatedError(), validator.GetFormmatedError().Error()) - } - if !validator.HasWarnings() && !validator.HasErrors() { - message.Success(validator.GetFormmatedSuccess()) - } - + displayFormattedMessage(validator) }, } diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 4a427e109a..5965dbd0b0 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -69,16 +69,16 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) - validator := Validator{jsonSchema: getZarfSchema(t)} - validator, err := validateSchema(validator, unmarshalledYaml) + validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} + validator, err := validateSchema(validator) require.NoError(t, err) require.Empty(t, validator.errors) }) t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) - validator := Validator{jsonSchema: getZarfSchema(t)} - validator, err := validateSchema(validator, unmarshalledYaml) + validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} + validator, err := validateSchema(validator) require.NoError(t, err) require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") @@ -86,13 +86,13 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) - validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) - require.Empty(t, validator) + validator := checkForVarInComponentImport(Validator{typedZarfPackage: unmarshalledYaml}) + require.Empty(t, validator.warnings) }) t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := checkForVarInComponentImport(Validator{}, unmarshalledYaml) + validator := checkForVarInComponentImport(Validator{typedZarfPackage: unmarshalledYaml}) require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") }) @@ -102,7 +102,7 @@ func TestValidateSchema(t *testing.T) { error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{errors: []string{error1, error2}} errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) - require.EqualError(t, validator.GetFormmatedError(), errorMessage) + require.EqualError(t, validator.GetFormatedError(), errorMessage) }) t.Run("Validator Warning formatting", func(t *testing.T) { @@ -110,6 +110,6 @@ func TestValidateSchema(t *testing.T) { warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{warnings: []string{warning1, warning2}} message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.GetFormmatedWarning(), message) + require.Equal(t, validator.GetFormatedWarning(), message) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index dc9da9c2ed..ac823ea2c8 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -23,13 +23,15 @@ const ( ) type Validator struct { - warnings []string - errors []string - jsonSchema []byte + warnings []string + errors []string + jsonSchema []byte + typedZarfPackage types.ZarfPackage + untypedZarfPackage interface{} } -func (v Validator) GetFormmatedError() error { - if len(v.errors) == 0 { +func (v Validator) GetFormatedError() error { + if !v.HasErrors() { return nil } errorMessage := validatorInvalidPrefix @@ -39,54 +41,56 @@ func (v Validator) GetFormmatedError() error { return errors.New(errorMessage) } -func (v Validator) GetFormmatedWarning() string { - if len(v.warnings) == 0 { +func (v Validator) GetFormatedWarning() string { + if !v.HasWarnings() { return "" } return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) } -func (v Validator) GetFormmatedSuccess() string { - return fmt.Sprintf("Schema validation successful for %q", "init") +func (v Validator) GetFormatedSuccess() string { + return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) } func (v Validator) HasWarnings() bool { - return !(len(v.warnings) == 0) + return len(v.warnings) > 0 } func (v Validator) HasErrors() bool { - return !(len(v.errors) == 0) + return len(v.errors) > 0 +} + +func (v Validator) IsSuccess() bool { + return !v.HasWarnings() && !v.HasErrors() } // ValidateZarfSchema validates a zarf file against the zarf schema, returns a validator with warnings or errors if they exist func ValidateZarfSchema(path string) (Validator, error) { validator := Validator{} var err error - var zarfTypedData types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfTypedData); err != nil { + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { return validator, err } - validator = checkForVarInComponentImport(validator, zarfTypedData) + validator = checkForVarInComponentImport(validator) if validator.jsonSchema, err = config.GetSchemaFile(); err != nil { return validator, err } - var zarfData interface{} - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &zarfData); err != nil { + if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { return validator, err } - if validator, err = validateSchema(validator, zarfData); err != nil { + if validator, err = validateSchema(validator); err != nil { return validator, err } return validator, nil } -func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPackage) Validator { - for i, component := range zarfPackage.Components { +func checkForVarInComponentImport(validator Validator) Validator { + for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } @@ -98,9 +102,9 @@ func checkForVarInComponentImport(validator Validator, zarfPackage types.ZarfPac return validator } -func validateSchema(validator Validator, unmarshalledYaml interface{}) (Validator, error) { +func validateSchema(validator Validator) (Validator, error) { schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) - documentLoader := gojsonschema.NewGoLoader(unmarshalledYaml) + documentLoader := gojsonschema.NewGoLoader(validator.untypedZarfPackage) result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { From 50e85a02abdf6aa1cec8c4e2f3c0369286d7a5c9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 17:02:03 +0000 Subject: [PATCH 63/87] refactoring validator --- src/cmd/prepare.go | 14 +----- src/pkg/packager/validator/validater_test.go | 14 +++--- src/pkg/packager/validator/validator.go | 45 +++++++++++++------- 3 files changed, 38 insertions(+), 35 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index cb3a476b7f..c5404e4de8 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -206,18 +206,6 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } -func displayFormattedMessage(validator validator.Validator) { - if validator.HasWarnings() { - message.Warn(validator.GetFormatedWarning()) - } - if validator.HasErrors() { - message.Fatal(validator.GetFormatedError(), validator.GetFormatedError().Error()) - } - if validator.IsSuccess() { - message.Success(validator.GetFormatedSuccess()) - } -} - var lintCmd = &cobra.Command{ Use: "lint [ DIRECTORY ]", Args: cobra.MaximumNArgs(1), @@ -238,7 +226,7 @@ var lintCmd = &cobra.Command{ if err != nil { message.Fatal(err, err.Error()) } - displayFormattedMessage(validator) + validator.DisplayFormattedMessage() }, } diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 5965dbd0b0..6e0d22ab81 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -70,7 +70,7 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, goodZarfPackage) validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} - validator, err := validateSchema(validator) + err := validateSchema(&validator) require.NoError(t, err) require.Empty(t, validator.errors) }) @@ -78,7 +78,7 @@ func TestValidateSchema(t *testing.T) { t.Run("validate schema fail", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[interface{}](t, badZarfPackage) validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} - validator, err := validateSchema(validator) + err := validateSchema(&validator) require.NoError(t, err) require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") @@ -86,13 +86,15 @@ func TestValidateSchema(t *testing.T) { t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) - validator := checkForVarInComponentImport(Validator{typedZarfPackage: unmarshalledYaml}) + validator := Validator{typedZarfPackage: unmarshalledYaml} + checkForVarInComponentImport(&validator) require.Empty(t, validator.warnings) }) t.Run("Template in component import failure", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := checkForVarInComponentImport(Validator{typedZarfPackage: unmarshalledYaml}) + validator := Validator{typedZarfPackage: unmarshalledYaml} + checkForVarInComponentImport(&validator) require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") }) @@ -102,7 +104,7 @@ func TestValidateSchema(t *testing.T) { error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{errors: []string{error1, error2}} errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) - require.EqualError(t, validator.GetFormatedError(), errorMessage) + require.EqualError(t, validator.getFormatedError(), errorMessage) }) t.Run("Validator Warning formatting", func(t *testing.T) { @@ -110,6 +112,6 @@ func TestValidateSchema(t *testing.T) { warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" validator := Validator{warnings: []string{warning1, warning2}} message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.GetFormatedWarning(), message) + require.Equal(t, validator.getFormatedWarning(), message) }) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index ac823ea2c8..8d63c79611 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" @@ -22,6 +23,7 @@ const ( validatorWarningPrefix = "zarf schema warning:" ) +// Validator is used by other packages to check their zarf packages for errors/warnings type Validator struct { warnings []string errors []string @@ -30,8 +32,8 @@ type Validator struct { untypedZarfPackage interface{} } -func (v Validator) GetFormatedError() error { - if !v.HasErrors() { +func (v Validator) getFormatedError() error { + if !v.hasErrors() { return nil } errorMessage := validatorInvalidPrefix @@ -41,27 +43,39 @@ func (v Validator) GetFormatedError() error { return errors.New(errorMessage) } -func (v Validator) GetFormatedWarning() string { - if !v.HasWarnings() { +func (v Validator) getFormatedWarning() string { + if !v.hasWarnings() { return "" } return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) } -func (v Validator) GetFormatedSuccess() string { +func (v Validator) getFormatedSuccess() string { return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) } -func (v Validator) HasWarnings() bool { +func (v Validator) hasWarnings() bool { return len(v.warnings) > 0 } -func (v Validator) HasErrors() bool { +func (v Validator) hasErrors() bool { return len(v.errors) > 0 } -func (v Validator) IsSuccess() bool { - return !v.HasWarnings() && !v.HasErrors() +func (v Validator) isSuccess() bool { + return !v.hasWarnings() && !v.hasErrors() +} + +func (v Validator) DisplayFormattedMessage() { + if v.hasWarnings() { + message.Warn(v.getFormatedWarning()) + } + if v.hasErrors() { + message.Fatal(v.getFormatedError(), v.getFormatedError().Error()) + } + if v.isSuccess() { + message.Success(v.getFormatedSuccess()) + } } // ValidateZarfSchema validates a zarf file against the zarf schema, returns a validator with warnings or errors if they exist @@ -72,7 +86,7 @@ func ValidateZarfSchema(path string) (Validator, error) { return validator, err } - validator = checkForVarInComponentImport(validator) + checkForVarInComponentImport(&validator) if validator.jsonSchema, err = config.GetSchemaFile(); err != nil { return validator, err @@ -82,14 +96,14 @@ func ValidateZarfSchema(path string) (Validator, error) { return validator, err } - if validator, err = validateSchema(validator); err != nil { + if err = validateSchema(&validator); err != nil { return validator, err } return validator, nil } -func checkForVarInComponentImport(validator Validator) Validator { +func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) @@ -99,16 +113,15 @@ func checkForVarInComponentImport(validator Validator) Validator { } } - return validator } -func validateSchema(validator Validator) (Validator, error) { +func validateSchema(validator *Validator) error { schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) documentLoader := gojsonschema.NewGoLoader(validator.untypedZarfPackage) result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { - return validator, err + return err } if !result.Valid() { @@ -117,5 +130,5 @@ func validateSchema(validator Validator) (Validator, error) { } } - return validator, err + return err } From acbf48ea78846ce805867095576415b971c24cc2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 17:08:41 +0000 Subject: [PATCH 64/87] change commment --- src/pkg/packager/validator/validator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 8d63c79611..154405240b 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -23,7 +23,7 @@ const ( validatorWarningPrefix = "zarf schema warning:" ) -// Validator is used by other packages to check their zarf packages for errors/warnings +// Validator holds the warnings/errors and messaging that we get from validation type Validator struct { warnings []string errors []string From 739aaa9223c2ccfcdec66f98fc5c73fd3ec2e46b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 17:16:58 +0000 Subject: [PATCH 65/87] moving errors to type error --- src/pkg/packager/validator/validater_test.go | 13 +++++++------ src/pkg/packager/validator/validator.go | 8 +++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/validator/validater_test.go index 6e0d22ab81..dc6acb26d1 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/validator/validater_test.go @@ -5,6 +5,7 @@ package validator import ( + "errors" "fmt" "os" "testing" @@ -80,8 +81,8 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.Equal(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") - require.Equal(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") + require.EqualError(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") + require.EqualError(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") }) t.Run("Template in component import success", func(t *testing.T) { @@ -100,10 +101,10 @@ func TestValidateSchema(t *testing.T) { }) t.Run("Validator Error formatting", func(t *testing.T) { - error1 := "components.0.import: Additional property not-path is not allowed" - error2 := "components.1.import.path: Invalid type. Expected: string, given: integer" - validator := Validator{errors: []string{error1, error2}} - errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1, error2) + error1 := errors.New("components.0.import: Additional property not-path is not allowed") + error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") + validator := Validator{errors: []error{error1, error2}} + errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) require.EqualError(t, validator.getFormatedError(), errorMessage) }) diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/validator/validator.go index 154405240b..839c887e3e 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/validator/validator.go @@ -26,7 +26,7 @@ const ( // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { warnings []string - errors []string + errors []error jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} @@ -38,7 +38,7 @@ func (v Validator) getFormatedError() error { } errorMessage := validatorInvalidPrefix for _, errorStr := range v.errors { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr) + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) } return errors.New(errorMessage) } @@ -66,6 +66,8 @@ func (v Validator) isSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } +// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success +// Will exit if there are errors func (v Validator) DisplayFormattedMessage() { if v.hasWarnings() { message.Warn(v.getFormatedWarning()) @@ -126,7 +128,7 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - validator.errors = append(validator.errors, desc.String()) + validator.errors = append(validator.errors, errors.New(desc.String())) } } From 48eaf4ba814b3c3821fff82049fd7b0759c4b459 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 20:29:25 +0000 Subject: [PATCH 66/87] renaming package to lint to make it easier to different object and package --- src/cmd/prepare.go | 4 ++-- src/pkg/packager/{validator/validator.go => lint/lint.go} | 2 +- .../{validator/validater_test.go => lint/lint_test.go} | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/pkg/packager/{validator/validator.go => lint/lint.go} (99%) rename src/pkg/packager/{validator/validater_test.go => lint/lint_test.go} (97%) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index c5404e4de8..6433af1280 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -17,7 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" - "github.com/defenseunicorns/zarf/src/pkg/packager/validator" + "github.com/defenseunicorns/zarf/src/pkg/packager/lint" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -222,7 +222,7 @@ var lintCmd = &cobra.Command{ message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) } } - validator, err := validator.ValidateZarfSchema(baseDir) + validator, err := lint.ValidateZarfSchema(baseDir) if err != nil { message.Fatal(err, err.Error()) } diff --git a/src/pkg/packager/validator/validator.go b/src/pkg/packager/lint/lint.go similarity index 99% rename from src/pkg/packager/validator/validator.go rename to src/pkg/packager/lint/lint.go index 839c887e3e..100f48523d 100644 --- a/src/pkg/packager/validator/validator.go +++ b/src/pkg/packager/lint/lint.go @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: 2021-Present The Zarf Authors // Package validator contains functions for verifying zarf yaml files are valid -package validator +package lint import ( "errors" diff --git a/src/pkg/packager/validator/validater_test.go b/src/pkg/packager/lint/lint_test.go similarity index 97% rename from src/pkg/packager/validator/validater_test.go rename to src/pkg/packager/lint/lint_test.go index dc6acb26d1..313dcfcb16 100644 --- a/src/pkg/packager/validator/validater_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package validator contains functions for verifying zarf yaml files are valid -package validator +// Package lint contains functions for verifying zarf yaml files are valid +package lint import ( "errors" From 6b9d3f970c7e237f21f96e17c0f8e8488306e094 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 20:38:23 +0000 Subject: [PATCH 67/87] moving get schema file inside of lint --- main.go | 3 ++- src/config/config.go | 5 ----- src/pkg/packager/lint/lint.go | 12 ++++++++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 651ba68a26..b9f58f6e44 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "github.com/defenseunicorns/zarf/src/cmd" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/packager/lint" ) //go:embed cosign.pub @@ -19,6 +20,6 @@ var zarfSchema embed.FS func main() { config.CosignPublicKey = cosignPublicKey - config.ZarfSchema = zarfSchema + lint.ZarfSchema = zarfSchema cmd.Execute() } diff --git a/src/config/config.go b/src/config/config.go index f5d24b2e05..071d160fe8 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -102,11 +102,6 @@ var ( ZarfDefaultCachePath = filepath.Join("~", ".zarf-cache") ) -// GetSchemaFile returns the zarf schema file -func GetSchemaFile() ([]byte, error) { - return ZarfSchema.ReadFile("zarf.schema.json") -} - // GetArch returns the arch based on a priority list with options for overriding. func GetArch(archs ...string) string { // List of architecture overrides. diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 100f48523d..2b3a913d24 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -5,12 +5,12 @@ package lint import ( + "embed" "errors" "fmt" "path/filepath" "strings" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -23,6 +23,14 @@ const ( validatorWarningPrefix = "zarf schema warning:" ) +var ( + ZarfSchema embed.FS +) + +func getSchemaFile() ([]byte, error) { + return ZarfSchema.ReadFile("zarf.schema.json") +} + // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { warnings []string @@ -90,7 +98,7 @@ func ValidateZarfSchema(path string) (Validator, error) { checkForVarInComponentImport(&validator) - if validator.jsonSchema, err = config.GetSchemaFile(); err != nil { + if validator.jsonSchema, err = getSchemaFile(); err != nil { return validator, err } From bf588687ca4ee932b921e35537c2a538b628ab68 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 17 Nov 2023 20:59:48 +0000 Subject: [PATCH 68/87] making warnings with numbers work with yq --- src/pkg/packager/lint/lint.go | 14 +++++++++++--- src/pkg/packager/lint/lint_test.go | 4 ++-- src/test/e2e/12_lint_test.go | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 2b3a913d24..92f33dfaf0 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -74,6 +74,14 @@ func (v Validator) isSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } +func (v *Validator) addWarning(warning string) { + v.warnings = append(v.warnings, warning) +} + +func (v *Validator) addError(err error) { + v.errors = append(v.errors, err) +} + // DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success // Will exit if there are errors func (v Validator) DisplayFormattedMessage() { @@ -116,10 +124,10 @@ func ValidateZarfSchema(path string) (Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.path will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf("component.[%d].import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.warnings = append(validator.warnings, fmt.Sprintf("component.%d.import.url will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf("component.[%d].import.url will not resolve ZARF_PKG_TMPL_* variables", i)) } } @@ -136,7 +144,7 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - validator.errors = append(validator.errors, errors.New(desc.String())) + validator.addError(errors.New(desc.String())) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 313dcfcb16..dd668a9449 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -96,8 +96,8 @@ func TestValidateSchema(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} checkForVarInComponentImport(&validator) - require.Equal(t, validator.warnings[0], "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1], "component.3.import.url will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[0], "component.[2].import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[1], "component.[3].import.url will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("Validator Error formatting", func(t *testing.T) { diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 0a3c8dc60d..6abd4eb706 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -17,7 +17,7 @@ func TestLint(t *testing.T) { _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property not-path is not allowed") - require.Contains(t, stderr, "component.2.import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "component.[2].import.path will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("zarf test lint success", func(t *testing.T) { From bc4eb7b4294e8e8eab3d42507998c8a90354d348 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 15:24:03 +0000 Subject: [PATCH 69/87] changign validatezarfschema to return a pointer --- src/pkg/packager/lint/lint.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 92f33dfaf0..ef9a36a4c2 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -96,29 +96,30 @@ func (v Validator) DisplayFormattedMessage() { } } -// ValidateZarfSchema validates a zarf file against the zarf schema, returns a validator with warnings or errors if they exist -func ValidateZarfSchema(path string) (Validator, error) { +// ValidateZarfSchema validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist +// along with an error if the validation itself failed +func ValidateZarfSchema(path string) (*Validator, error) { validator := Validator{} var err error if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { - return validator, err + return &validator, err } checkForVarInComponentImport(&validator) if validator.jsonSchema, err = getSchemaFile(); err != nil { - return validator, err + return &validator, err } if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { - return validator, err + return &validator, err } if err = validateSchema(&validator); err != nil { - return validator, err + return &validator, err } - return validator, nil + return &validator, nil } func checkForVarInComponentImport(validator *Validator) { From ee5cce065d0b74ce4a5b7840f2f967d5fe3758cb Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 15:51:24 +0000 Subject: [PATCH 70/87] separating validator object into it's own file --- src/pkg/packager/lint/lint.go | 53 +------------------- src/pkg/packager/lint/lint_test.go | 17 ------- src/pkg/packager/lint/validator.go | 64 +++++++++++++++++++++++++ src/pkg/packager/lint/validator_test.go | 31 ++++++++++++ 4 files changed, 96 insertions(+), 69 deletions(-) create mode 100644 src/pkg/packager/lint/validator.go create mode 100644 src/pkg/packager/lint/validator_test.go diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index ef9a36a4c2..05cda78353 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package validator contains functions for verifying zarf yaml files are valid +// Package lint contains functions for verifying zarf yaml files are valid package lint import ( @@ -31,57 +31,6 @@ func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } -// Validator holds the warnings/errors and messaging that we get from validation -type Validator struct { - warnings []string - errors []error - jsonSchema []byte - typedZarfPackage types.ZarfPackage - untypedZarfPackage interface{} -} - -func (v Validator) getFormatedError() error { - if !v.hasErrors() { - return nil - } - errorMessage := validatorInvalidPrefix - for _, errorStr := range v.errors { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) - } - return errors.New(errorMessage) -} - -func (v Validator) getFormatedWarning() string { - if !v.hasWarnings() { - return "" - } - return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) -} - -func (v Validator) getFormatedSuccess() string { - return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) -} - -func (v Validator) hasWarnings() bool { - return len(v.warnings) > 0 -} - -func (v Validator) hasErrors() bool { - return len(v.errors) > 0 -} - -func (v Validator) isSuccess() bool { - return !v.hasWarnings() && !v.hasErrors() -} - -func (v *Validator) addWarning(warning string) { - v.warnings = append(v.warnings, warning) -} - -func (v *Validator) addError(err error) { - v.errors = append(v.errors, err) -} - // DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success // Will exit if there are errors func (v Validator) DisplayFormattedMessage() { diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index dd668a9449..da8f2c5b36 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -5,8 +5,6 @@ package lint import ( - "errors" - "fmt" "os" "testing" @@ -100,19 +98,4 @@ func TestValidateSchema(t *testing.T) { require.Equal(t, validator.warnings[1], "component.[3].import.url will not resolve ZARF_PKG_TMPL_* variables") }) - t.Run("Validator Error formatting", func(t *testing.T) { - error1 := errors.New("components.0.import: Additional property not-path is not allowed") - error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") - validator := Validator{errors: []error{error1, error2}} - errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) - require.EqualError(t, validator.getFormatedError(), errorMessage) - }) - - t.Run("Validator Warning formatting", func(t *testing.T) { - warning1 := "components.0.import: Additional property not-path is not allowed" - warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" - validator := Validator{warnings: []string{warning1, warning2}} - message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.getFormatedWarning(), message) - }) } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go new file mode 100644 index 0000000000..850ca7b0eb --- /dev/null +++ b/src/pkg/packager/lint/validator.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package lint contains functions for verifying zarf yaml files are valid +package lint + +import ( + "errors" + "fmt" + "strings" + + "github.com/defenseunicorns/zarf/src/types" +) + +// Validator holds the warnings/errors and messaging that we get from validation +type Validator struct { + warnings []string + errors []error + jsonSchema []byte + typedZarfPackage types.ZarfPackage + untypedZarfPackage interface{} +} + +func (v Validator) getFormatedError() error { + if !v.hasErrors() { + return nil + } + errorMessage := validatorInvalidPrefix + for _, errorStr := range v.errors { + errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) + } + return errors.New(errorMessage) +} + +func (v Validator) getFormatedWarning() string { + if !v.hasWarnings() { + return "" + } + return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) +} + +func (v Validator) getFormatedSuccess() string { + return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) +} + +func (v Validator) hasWarnings() bool { + return len(v.warnings) > 0 +} + +func (v Validator) hasErrors() bool { + return len(v.errors) > 0 +} + +func (v Validator) isSuccess() bool { + return !v.hasWarnings() && !v.hasErrors() +} + +func (v *Validator) addWarning(warning string) { + v.warnings = append(v.warnings, warning) +} + +func (v *Validator) addError(err error) { + v.errors = append(v.errors, err) +} diff --git a/src/pkg/packager/lint/validator_test.go b/src/pkg/packager/lint/validator_test.go new file mode 100644 index 0000000000..c6fa4a037b --- /dev/null +++ b/src/pkg/packager/lint/validator_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package lint contains functions for verifying zarf yaml files are valid +package lint + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidator(t *testing.T) { + t.Run("Validator Error formatting", func(t *testing.T) { + error1 := errors.New("components.0.import: Additional property not-path is not allowed") + error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") + validator := Validator{errors: []error{error1, error2}} + errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) + require.EqualError(t, validator.getFormatedError(), errorMessage) + }) + + t.Run("Validator Warning formatting", func(t *testing.T) { + warning1 := "components.0.import: Additional property not-path is not allowed" + warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" + validator := Validator{warnings: []string{warning1, warning2}} + message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) + require.Equal(t, validator.getFormatedWarning(), message) + }) +} From bce27962c113f7260c458717ee232ce45990d854 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 17:57:13 +0000 Subject: [PATCH 71/87] lint comment --- src/pkg/packager/lint/lint.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 05cda78353..8792c582b7 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -23,9 +23,8 @@ const ( validatorWarningPrefix = "zarf schema warning:" ) -var ( - ZarfSchema embed.FS -) +// ZarfSchema is exported so main.go can embed the schema file +var ZarfSchema embed.FS func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") From 0bd8d4a8c4a84b214ed79972876e00140b9e5910 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 18:25:46 +0000 Subject: [PATCH 72/87] implementing error in validate --- src/pkg/packager/lint/lint.go | 2 +- src/pkg/packager/lint/validator.go | 7 +++---- src/pkg/packager/lint/validator_test.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 8792c582b7..6771fce20c 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -37,7 +37,7 @@ func (v Validator) DisplayFormattedMessage() { message.Warn(v.getFormatedWarning()) } if v.hasErrors() { - message.Fatal(v.getFormatedError(), v.getFormatedError().Error()) + message.Fatal(v, v.Error()) } if v.isSuccess() { message.Success(v.getFormatedSuccess()) diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 850ca7b0eb..02cf15775e 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -5,7 +5,6 @@ package lint import ( - "errors" "fmt" "strings" @@ -21,15 +20,15 @@ type Validator struct { untypedZarfPackage interface{} } -func (v Validator) getFormatedError() error { +func (v Validator) Error() string { if !v.hasErrors() { - return nil + return "" } errorMessage := validatorInvalidPrefix for _, errorStr := range v.errors { errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) } - return errors.New(errorMessage) + return errorMessage } func (v Validator) getFormatedWarning() string { diff --git a/src/pkg/packager/lint/validator_test.go b/src/pkg/packager/lint/validator_test.go index c6fa4a037b..70adb10931 100644 --- a/src/pkg/packager/lint/validator_test.go +++ b/src/pkg/packager/lint/validator_test.go @@ -18,7 +18,7 @@ func TestValidator(t *testing.T) { error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") validator := Validator{errors: []error{error1, error2}} errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) - require.EqualError(t, validator.getFormatedError(), errorMessage) + require.EqualError(t, validator, errorMessage) }) t.Run("Validator Warning formatting", func(t *testing.T) { From 4eb7fc15c67dfb9db2ab35e30a57fedd4def2787 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 18:46:38 +0000 Subject: [PATCH 73/87] updating zarf docs for lint --- docs/3-create-a-zarf-package/index.md | 3 +++ docs/5-zarf-tutorials/0-creating-a-zarf-package.md | 1 + 2 files changed, 4 insertions(+) diff --git a/docs/3-create-a-zarf-package/index.md b/docs/3-create-a-zarf-package/index.md index 21bc292bef..17746a01b5 100644 --- a/docs/3-create-a-zarf-package/index.md +++ b/docs/3-create-a-zarf-package/index.md @@ -21,6 +21,9 @@ To learn more about creating a Zarf package, you can check out the following res The general flow of a Zarf package deployment on an existing initialized cluster is as follows: ```shell +# Before creating your package you can lint your zarf.yaml +$ zarf package lint + # To create a package run the following: $ zarf package create # - Enter any package templates that have not yet been defined diff --git a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md index d0b53f65dd..f311a4d387 100644 --- a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md +++ b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md @@ -37,6 +37,7 @@ metadata: :::tip If you are using an Integrated Development Environment (such as [VS Code](../3-create-a-zarf-package/8-vscode.md)) to create and edit the `zarf.yaml` file, you can install or reference the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) file to get error checking and autocomplete. +Additionally, you can run `zarf prepare lint ` to validate aginst the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) ::: From 63bcb8c5fc7ca3b5902a153d7c8fd30547fd1d55 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 27 Nov 2023 21:53:22 +0000 Subject: [PATCH 74/87] refactoring message to use table function and printing a table for warning --- src/pkg/message/connect.go | 10 +++--- src/pkg/message/credentials.go | 33 ++++++++--------- src/pkg/message/message.go | 21 +++++++++++ src/pkg/packager/lint/lint.go | 19 ++-------- src/pkg/packager/lint/validator.go | 47 ++++++++++++++++++++----- src/pkg/packager/lint/validator_test.go | 14 ++++---- src/test/e2e/12_lint_test.go | 2 +- 7 files changed, 89 insertions(+), 57 deletions(-) diff --git a/src/pkg/message/connect.go b/src/pkg/message/connect.go index 0df6f08d38..22b125ed51 100644 --- a/src/pkg/message/connect.go +++ b/src/pkg/message/connect.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/defenseunicorns/zarf/src/types" - "github.com/pterm/pterm" ) // PrintConnectStringTable prints a table of connect strings. @@ -16,14 +15,15 @@ func PrintConnectStringTable(connectStrings types.ConnectStrings) { Debugf("message.PrintConnectStringTable(%#v)", connectStrings) if len(connectStrings) > 0 { - list := pterm.TableData{{" Connect Command", "Description"}} + connectData := [][]string{} // Loop over each connectStrings and convert to pterm.TableData for name, connect := range connectStrings { - name = fmt.Sprintf(" zarf connect %s", name) - list = append(list, []string{name, connect.Description}) + name = fmt.Sprintf("zarf connect %s", name) + connectData = append(connectData, []string{name, connect.Description}) } // Create the table output with the data - _ = pterm.DefaultTable.WithHasHeader().WithData(list).Render() + header := []string{"Connect Command", "Description"} + Table(header, connectData) } } diff --git a/src/pkg/message/credentials.go b/src/pkg/message/credentials.go index b89e15da74..15e1e87c71 100644 --- a/src/pkg/message/credentials.go +++ b/src/pkg/message/credentials.go @@ -34,37 +34,32 @@ func PrintCredentialTable(state *types.ZarfState, componentsToDeploy []types.Dep // Set output to os.Stderr to avoid creds being printed in logs pterm.SetDefaultOutput(os.Stderr) - pterm.Println() - loginTableHeader := pterm.TableData{ - {" Application", "Username", "Password", "Connect", "Get-Creds Key"}, - } - - loginTable := pterm.TableData{} + loginData := [][]string{} if state.RegistryInfo.InternalRegistry { - loginTable = append(loginTable, pterm.TableData{ - {" Registry", state.RegistryInfo.PushUsername, state.RegistryInfo.PushPassword, "zarf connect registry", RegistryKey}, - {" Registry (read-only)", state.RegistryInfo.PullUsername, state.RegistryInfo.PullPassword, "zarf connect registry", RegistryReadKey}, - }...) + loginData = append(loginData, + []string{"Registry", state.RegistryInfo.PushUsername, state.RegistryInfo.PushPassword, "zarf connect registry", RegistryKey}, + []string{"Registry (read-only)", state.RegistryInfo.PullUsername, state.RegistryInfo.PullPassword, "zarf connect registry", RegistryReadKey}, + ) } for _, component := range componentsToDeploy { // Show message if including logging stack if component.Name == "logging" { - loginTable = append(loginTable, pterm.TableData{{" Logging", config.ZarfLoggingUser, state.LoggingSecret, "zarf connect logging", LoggingKey}}...) + loginData = append(loginData, []string{"Logging", config.ZarfLoggingUser, state.LoggingSecret, "zarf connect logging", LoggingKey}) } // Show message if including git-server if component.Name == "git-server" { - loginTable = append(loginTable, pterm.TableData{ - {" Git", state.GitServer.PushUsername, state.GitServer.PushPassword, "zarf connect git", GitKey}, - {" Git (read-only)", state.GitServer.PullUsername, state.GitServer.PullPassword, "zarf connect git", GitReadKey}, - {" Artifact Token", state.ArtifactServer.PushUsername, state.ArtifactServer.PushToken, "zarf connect git", ArtifactKey}, - }...) + loginData = append(loginData, + []string{"Git", state.GitServer.PushUsername, state.GitServer.PushPassword, "zarf connect git", GitKey}, + []string{"Git (read-only)", state.GitServer.PullUsername, state.GitServer.PullPassword, "zarf connect git", GitReadKey}, + []string{"Artifact Token", state.ArtifactServer.PushUsername, state.ArtifactServer.PushToken, "zarf connect git", ArtifactKey}, + ) } } - if len(loginTable) > 0 { - loginTable = append(loginTableHeader, loginTable...) - _ = pterm.DefaultTable.WithHasHeader().WithData(loginTable).Render() + if len(loginData) > 0 { + header := []string{"Application", "Username", "Password", "Connect", "Get-Creds Key"} + Table(header, loginData) } // Restore the log file if it was specified diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index 07b3db9983..1a1687a9c0 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -340,3 +340,24 @@ func debugPrinter(offset int, a ...any) { func errorPrinter(offset int) *pterm.PrefixPrinter { return pterm.Error.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) } + +func Table(header []string, data [][]string) { + pterm.Println() + + if len(header) > 0 { + header[0] = fmt.Sprintf(" %s", header[0]) + } + + table := pterm.TableData{ + header, + } + + for _, row := range data { + if len(row) > 0 { + row[0] = fmt.Sprintf(" %s", row[0]) + } + table = append(table, pterm.TableData{row}...) + } + + pterm.DefaultTable.WithHasHeader().WithData(table).Render() +} diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 6771fce20c..b07d1b6b2e 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" @@ -30,20 +29,6 @@ func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } -// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success -// Will exit if there are errors -func (v Validator) DisplayFormattedMessage() { - if v.hasWarnings() { - message.Warn(v.getFormatedWarning()) - } - if v.hasErrors() { - message.Fatal(v, v.Error()) - } - if v.isSuccess() { - message.Success(v.getFormatedSuccess()) - } -} - // ValidateZarfSchema validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist // along with an error if the validation itself failed func ValidateZarfSchema(path string) (*Validator, error) { @@ -73,10 +58,10 @@ func ValidateZarfSchema(path string) (*Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.path will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf("component.[%d].import.path", i), "will not resolve ZARF_PKG_TMPL_* variables") } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.url will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf("component.[%d].import.url", i), "will not resolve ZARF_PKG_TMPL_* variables") } } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 02cf15775e..e340855c90 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -6,14 +6,20 @@ package lint import ( "fmt" - "strings" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) +// Warning holds the location of warning and the message +type Warning struct { + location string + message string +} + // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { - warnings []string + warnings []Warning errors []error jsonSchema []byte typedZarfPackage types.ZarfPackage @@ -31,11 +37,22 @@ func (v Validator) Error() string { return errorMessage } -func (v Validator) getFormatedWarning() string { - if !v.hasWarnings() { - return "" +// func (v Validator) getFormatedWarning() string { +// if !v.hasWarnings() { +// return "" +// } +// return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) +// } + +func (v Validator) printWarningTable() { + if v.hasWarnings() { + connectData := [][]string{} + for _, warning := range v.warnings { + connectData = append(connectData, []string{warning.location, warning.message}) + } + header := []string{"Location", "Warning"} + message.Table(header, connectData) } - return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) } func (v Validator) getFormatedSuccess() string { @@ -54,10 +71,24 @@ func (v Validator) isSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } -func (v *Validator) addWarning(warning string) { - v.warnings = append(v.warnings, warning) +func (v *Validator) addWarning(location string, message string) { + v.warnings = append(v.warnings, Warning{location, message}) } func (v *Validator) addError(err error) { v.errors = append(v.errors, err) } + +// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success +// Will exit if there are errors +func (v Validator) DisplayFormattedMessage() { + if v.hasWarnings() { + v.printWarningTable() + } + if v.hasErrors() { + message.Fatal(v, v.Error()) + } + if v.isSuccess() { + message.Success(v.getFormatedSuccess()) + } +} diff --git a/src/pkg/packager/lint/validator_test.go b/src/pkg/packager/lint/validator_test.go index 70adb10931..deaace5383 100644 --- a/src/pkg/packager/lint/validator_test.go +++ b/src/pkg/packager/lint/validator_test.go @@ -21,11 +21,11 @@ func TestValidator(t *testing.T) { require.EqualError(t, validator, errorMessage) }) - t.Run("Validator Warning formatting", func(t *testing.T) { - warning1 := "components.0.import: Additional property not-path is not allowed" - warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" - validator := Validator{warnings: []string{warning1, warning2}} - message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - require.Equal(t, validator.getFormatedWarning(), message) - }) + // t.Run("Validator Warning formatting", func(t *testing.T) { + // warning1 := "components.0.import: Additional property not-path is not allowed" + // warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" + // validator := Validator{warnings: []string{warning1, warning2}} + // message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) + // require.Equal(t, validator.getFormatedWarning(), message) + // }) } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 6abd4eb706..ec8b3720b6 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -17,7 +17,7 @@ func TestLint(t *testing.T) { _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Expect error here because the yaml file is not following schema") require.Contains(t, stderr, "- components.0.import: Additional property not-path is not allowed") - require.Contains(t, stderr, "component.[2].import.path will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, "component.[2].import.path") }) t.Run("zarf test lint success", func(t *testing.T) { From 3106628e9f803578168bcc2fc744a9574fb8b785 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 28 Nov 2023 16:43:52 +0000 Subject: [PATCH 75/87] formmating lint table --- src/pkg/packager/lint/lint.go | 4 +- src/pkg/packager/lint/validator.go | 49 +++++++------------------ src/pkg/packager/lint/validator_test.go | 31 ---------------- src/pkg/utils/random.go | 8 ++++ src/test/e2e/12_lint_test.go | 5 +-- 5 files changed, 26 insertions(+), 71 deletions(-) delete mode 100644 src/pkg/packager/lint/validator_test.go diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index b07d1b6b2e..9336d156f0 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -58,10 +58,10 @@ func ValidateZarfSchema(path string) (*Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.path", i), "will not resolve ZARF_PKG_TMPL_* variables") + validator.addWarning(fmt.Sprintf("component.[%d].import.path will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.url", i), "will not resolve ZARF_PKG_TMPL_* variables") + validator.addWarning(fmt.Sprintf("component.[%d].import.url will not resolve ZARF_PKG_TMPL_* variables", i)) } } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index e340855c90..dc7b42dd00 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -8,50 +8,33 @@ import ( "fmt" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" + "github.com/fatih/color" ) -// Warning holds the location of warning and the message -type Warning struct { - location string - message string -} - // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { - warnings []Warning + warnings []string errors []error jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} } -func (v Validator) Error() string { - if !v.hasErrors() { - return "" - } - errorMessage := validatorInvalidPrefix - for _, errorStr := range v.errors { - errorMessage = fmt.Sprintf("%s\n - %s", errorMessage, errorStr.Error()) - } - return errorMessage -} - -// func (v Validator) getFormatedWarning() string { -// if !v.hasWarnings() { -// return "" -// } -// return fmt.Sprintf("%s %s", validatorWarningPrefix, strings.Join(v.warnings, ", ")) -// } - func (v Validator) printWarningTable() { if v.hasWarnings() { + header := []string{"Type", "Message"} connectData := [][]string{} for _, warning := range v.warnings { - connectData = append(connectData, []string{warning.location, warning.message}) + connectData = append(connectData, []string{utils.ColorWrap("Warning", color.FgYellow), warning}) + } + for _, err := range v.errors { + connectData = append(connectData, []string{utils.ColorWrap("Error", color.FgRed), err.Error()}) } - header := []string{"Location", "Warning"} message.Table(header, connectData) + message.Info(fmt.Sprintf("%d warnings and %d errors in %q", + len(v.warnings), len(v.errors), v.typedZarfPackage.Metadata.Name)) } } @@ -71,8 +54,8 @@ func (v Validator) isSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } -func (v *Validator) addWarning(location string, message string) { - v.warnings = append(v.warnings, Warning{location, message}) +func (v *Validator) addWarning(message string) { + v.warnings = append(v.warnings, message) } func (v *Validator) addError(err error) { @@ -82,13 +65,9 @@ func (v *Validator) addError(err error) { // DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success // Will exit if there are errors func (v Validator) DisplayFormattedMessage() { - if v.hasWarnings() { - v.printWarningTable() - } - if v.hasErrors() { - message.Fatal(v, v.Error()) - } if v.isSuccess() { message.Success(v.getFormatedSuccess()) + } else { + v.printWarningTable() } } diff --git a/src/pkg/packager/lint/validator_test.go b/src/pkg/packager/lint/validator_test.go deleted file mode 100644 index deaace5383..0000000000 --- a/src/pkg/packager/lint/validator_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package lint contains functions for verifying zarf yaml files are valid -package lint - -import ( - "errors" - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestValidator(t *testing.T) { - t.Run("Validator Error formatting", func(t *testing.T) { - error1 := errors.New("components.0.import: Additional property not-path is not allowed") - error2 := errors.New("components.1.import.path: Invalid type. Expected: string, given: integer") - validator := Validator{errors: []error{error1, error2}} - errorMessage := fmt.Sprintf("%s\n - %s\n - %s", validatorInvalidPrefix, error1.Error(), error2.Error()) - require.EqualError(t, validator, errorMessage) - }) - - // t.Run("Validator Warning formatting", func(t *testing.T) { - // warning1 := "components.0.import: Additional property not-path is not allowed" - // warning2 := "components.1.import.path: Invalid type. Expected: string, given: integer" - // validator := Validator{warnings: []string{warning1, warning2}} - // message := fmt.Sprintf("%s %s, %s", validatorWarningPrefix, warning1, warning2) - // require.Equal(t, validator.getFormatedWarning(), message) - // }) -} diff --git a/src/pkg/utils/random.go b/src/pkg/utils/random.go index 0c074580d2..7966cd765a 100644 --- a/src/pkg/utils/random.go +++ b/src/pkg/utils/random.go @@ -6,8 +6,10 @@ package utils import ( "crypto/rand" + "fmt" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/fatih/color" ) // Very limited special chars for git / basic auth @@ -37,3 +39,9 @@ func First30last30(s string) string { return s } + +// ColorWrap changes a string to an ansi color code and appends the default color to the end +// preventing future characters from taking on the given color +func ColorWrap(str string, attr color.Attribute) string { + return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) +} diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index ec8b3720b6..916bda129a 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -14,9 +14,8 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema fail") path := filepath.Join("src", "test", "packages", "12-lint") - _, stderr, err := e2e.Zarf("prepare", "lint", path) - require.Error(t, err, "Expect error here because the yaml file is not following schema") - require.Contains(t, stderr, "- components.0.import: Additional property not-path is not allowed") + _, stderr, _ := e2e.Zarf("prepare", "lint", path) + require.Contains(t, stderr, "components.0.import: Additional property not-path is not allowed") require.Contains(t, stderr, "component.[2].import.path") }) From 7443793ac7a2a6e18d214e86e8cafc4ceab7396f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 28 Nov 2023 17:13:49 +0000 Subject: [PATCH 76/87] exported function comment --- src/pkg/message/message.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index 1a1687a9c0..c2713250b7 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -341,6 +341,7 @@ func errorPrinter(offset int) *pterm.PrefixPrinter { return pterm.Error.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) } +// Table prints a padded table containing the specified header and data func Table(header []string, data [][]string) { pterm.Println() From 714ba99a1520c57184395261e2b1f1ea914b9e51 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 28 Nov 2023 18:50:43 +0000 Subject: [PATCH 77/87] exiting when there is a warning or error --- src/cmd/prepare.go | 3 +++ src/pkg/packager/lint/validator.go | 13 +++++++------ src/test/e2e/12_lint_test.go | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 6433af1280..2bf8e29d07 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -227,6 +227,9 @@ var lintCmd = &cobra.Command{ message.Fatal(err, err.Error()) } validator.DisplayFormattedMessage() + if !validator.IsSuccess() { + os.Exit(1) + } }, } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index dc7b42dd00..736a54ddcb 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -23,7 +23,7 @@ type Validator struct { } func (v Validator) printWarningTable() { - if v.hasWarnings() { + if !v.IsSuccess() { header := []string{"Type", "Message"} connectData := [][]string{} for _, warning := range v.warnings { @@ -50,10 +50,6 @@ func (v Validator) hasErrors() bool { return len(v.errors) > 0 } -func (v Validator) isSuccess() bool { - return !v.hasWarnings() && !v.hasErrors() -} - func (v *Validator) addWarning(message string) { v.warnings = append(v.warnings, message) } @@ -62,10 +58,15 @@ func (v *Validator) addError(err error) { v.errors = append(v.errors, err) } +// IsSuccess returns true if there are not any warnings or errors +func (v Validator) IsSuccess() bool { + return !v.hasWarnings() && !v.hasErrors() +} + // DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success // Will exit if there are errors func (v Validator) DisplayFormattedMessage() { - if v.isSuccess() { + if v.IsSuccess() { message.Success(v.getFormatedSuccess()) } else { v.printWarningTable() diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 916bda129a..e540c92fda 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -14,7 +14,8 @@ func TestLint(t *testing.T) { t.Log("E2E: Test lint on schema fail") path := filepath.Join("src", "test", "packages", "12-lint") - _, stderr, _ := e2e.Zarf("prepare", "lint", path) + _, stderr, err := e2e.Zarf("prepare", "lint", path) + require.Error(t, err, "Require an exit code since there was warnings / errors") require.Contains(t, stderr, "components.0.import: Additional property not-path is not allowed") require.Contains(t, stderr, "component.[2].import.path") }) From 5b549d2febf394533058647e55d23da555c07594 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:32:24 -0500 Subject: [PATCH 78/87] Apply suggestions from code review Co-authored-by: Lucas Rodriguez --- src/pkg/packager/lint/lint.go | 8 ++++---- src/pkg/packager/lint/lint_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 9336d156f0..5d67088e4c 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -35,21 +35,21 @@ func ValidateZarfSchema(path string) (*Validator, error) { validator := Validator{} var err error if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { - return &validator, err + return nil, err } checkForVarInComponentImport(&validator) if validator.jsonSchema, err = getSchemaFile(); err != nil { - return &validator, err + return nil, err } if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { - return &validator, err + return nil, err } if err = validateSchema(&validator); err != nil { - return &validator, err + return nil, err } return &validator, nil diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index da8f2c5b36..c02df8f699 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -51,7 +51,7 @@ func readAndUnmarshalYaml[T interface{}](t *testing.T, yamlString string) T { var unmarshalledYaml T err := goyaml.Unmarshal([]byte(yamlString), &unmarshalledYaml) if err != nil { - t.Errorf("error unmarshalling yaml %v", err) + t.Errorf("error unmarshalling yaml: %v", err) } return unmarshalledYaml } @@ -61,7 +61,7 @@ func TestValidateSchema(t *testing.T) { t.Helper() file, err := os.ReadFile("../../../../zarf.schema.json") if err != nil { - t.Errorf("error reading file: %s", err) + t.Errorf("error reading file: %v", err) } return file } From da445e53a3763f148a27b88a973ab890ca4e49b8 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 13:55:16 +0000 Subject: [PATCH 79/87] fixing language to be lint --- src/cmd/prepare.go | 3 ++- src/config/lang/english.go | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 2bf8e29d07..fcd5e113c8 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -211,6 +211,7 @@ var lintCmd = &cobra.Command{ Args: cobra.MaximumNArgs(1), Aliases: []string{"l"}, Short: lang.CmdPrepareLintShort, + Long: lang.CmdPrepareLintLong, Run: func(cmd *cobra.Command, args []string) { baseDir := "" if len(args) > 0 { @@ -219,7 +220,7 @@ var lintCmd = &cobra.Command{ var err error baseDir, err = os.Getwd() if err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) + message.Fatalf(err, lang.CmdPrepareLintErr, err.Error()) } } validator, err := lint.ValidateZarfSchema(baseDir) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 0afd52beb1..13f337682a 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -360,6 +360,8 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" CmdPrepareLintShort = "Verifies the package schema" + CmdPrepareLintLong = "Verifies the package schema and warns the user if they have variables that won't be evaluated" + CmdPrepareLintErr = "Unable to lint package: %s" // zarf tools CmdToolsShort = "Collection of additional tools to make airgap easier" From ca7cc1be3372f81bc9f6ecdc7f2925f4cf87e20a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 13:58:32 +0000 Subject: [PATCH 80/87] changing order of functions --- src/pkg/message/message.go | 44 +++++++++++++++--------------- src/pkg/packager/lint/validator.go | 24 +++++++--------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index c2713250b7..ef1d2e235f 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -318,6 +318,28 @@ func Truncate(text string, length int, invert bool) string { return textEscaped } +// Table prints a padded table containing the specified header and data +func Table(header []string, data [][]string) { + pterm.Println() + + if len(header) > 0 { + header[0] = fmt.Sprintf(" %s", header[0]) + } + + table := pterm.TableData{ + header, + } + + for _, row := range data { + if len(row) > 0 { + row[0] = fmt.Sprintf(" %s", row[0]) + } + table = append(table, pterm.TableData{row}...) + } + + pterm.DefaultTable.WithHasHeader().WithData(table).Render() +} + func debugPrinter(offset int, a ...any) { printer := pterm.Debug.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) now := time.Now().Format(time.RFC3339) @@ -340,25 +362,3 @@ func debugPrinter(offset int, a ...any) { func errorPrinter(offset int) *pterm.PrefixPrinter { return pterm.Error.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) } - -// Table prints a padded table containing the specified header and data -func Table(header []string, data [][]string) { - pterm.Println() - - if len(header) > 0 { - header[0] = fmt.Sprintf(" %s", header[0]) - } - - table := pterm.TableData{ - header, - } - - for _, row := range data { - if len(row) > 0 { - row[0] = fmt.Sprintf(" %s", row[0]) - } - table = append(table, pterm.TableData{row}...) - } - - pterm.DefaultTable.WithHasHeader().WithData(table).Render() -} diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 736a54ddcb..e299e860d3 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -22,6 +22,16 @@ type Validator struct { untypedZarfPackage interface{} } +// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success +// Will exit if there are errors +func (v Validator) DisplayFormattedMessage() { + if v.IsSuccess() { + message.Success(fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name)) + } else { + v.printWarningTable() + } +} + func (v Validator) printWarningTable() { if !v.IsSuccess() { header := []string{"Type", "Message"} @@ -38,10 +48,6 @@ func (v Validator) printWarningTable() { } } -func (v Validator) getFormatedSuccess() string { - return fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) -} - func (v Validator) hasWarnings() bool { return len(v.warnings) > 0 } @@ -62,13 +68,3 @@ func (v *Validator) addError(err error) { func (v Validator) IsSuccess() bool { return !v.hasWarnings() && !v.hasErrors() } - -// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success -// Will exit if there are errors -func (v Validator) DisplayFormattedMessage() { - if v.IsSuccess() { - message.Success(v.getFormatedSuccess()) - } else { - v.printWarningTable() - } -} From 3a95e031d38df37c79361c98049f994eadf715f6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 13:59:58 +0000 Subject: [PATCH 81/87] refactor validate schema --- src/pkg/packager/lint/lint.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 5d67088e4c..26901d3dc3 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -78,7 +78,8 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - validator.addError(errors.New(desc.String())) + err := errors.New(desc.String()) + validator.addError(err) } } From 37df5f7bc80881315e98ab59c43126f9f0c630d7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 14:51:25 +0000 Subject: [PATCH 82/87] making error strings yqable --- src/pkg/packager/lint/lint.go | 25 ++++++++++++++++--------- src/pkg/packager/lint/lint_test.go | 15 +++++++++++---- src/test/e2e/12_lint_test.go | 5 +++-- src/test/packages/12-lint/zarf.yaml | 5 ++++- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 26901d3dc3..e62a25ab99 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -6,9 +6,9 @@ package lint import ( "embed" - "errors" "fmt" "path/filepath" + "regexp" "strings" "github.com/defenseunicorns/zarf/src/pkg/layout" @@ -17,11 +17,6 @@ import ( "github.com/xeipuuv/gojsonschema" ) -const ( - validatorInvalidPrefix = "schema is invalid:" - validatorWarningPrefix = "zarf schema warning:" -) - // ZarfSchema is exported so main.go can embed the schema file var ZarfSchema embed.FS @@ -58,15 +53,26 @@ func ValidateZarfSchema(path string) (*Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.path will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf(".component.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf("component.[%d].import.url will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf(".component.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) } } } +func wrapNumbersInBrackets(input string) string { + // . is a non-word chacter (\b) so this gets digits between two . + re := regexp.MustCompile(`\b\d+\b`) + + wrapped := re.ReplaceAllStringFunc(input, func(match string) string { + return "[" + match + "]" + }) + + return wrapped +} + func validateSchema(validator *Validator) error { schemaLoader := gojsonschema.NewBytesLoader(validator.jsonSchema) documentLoader := gojsonschema.NewGoLoader(validator.untypedZarfPackage) @@ -78,7 +84,8 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - err := errors.New(desc.String()) + err := fmt.Errorf( + ".%s: %s", wrapNumbersInBrackets(desc.Field()), desc.Description()) validator.addError(err) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index c02df8f699..c932e7452d 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -79,8 +79,8 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.EqualError(t, validator.errors[0], "components.0.import: Additional property not-path is not allowed") - require.EqualError(t, validator.errors[1], "components.1.import.path: Invalid type. Expected: string, given: integer") + require.EqualError(t, validator.errors[0], ".components.[0].import: Additional property not-path is not allowed") + require.EqualError(t, validator.errors[1], ".components.[1].import.path: Invalid type. Expected: string, given: integer") }) t.Run("Template in component import success", func(t *testing.T) { @@ -94,8 +94,15 @@ func TestValidateSchema(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} checkForVarInComponentImport(&validator) - require.Equal(t, validator.warnings[0], "component.[2].import.path will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1], "component.[3].import.url will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[0], ".component.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[1], ".component.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") + }) + + t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { + input := "components12.12.import.path" + expected := "components12.[12].import.path" + acutal := wrapNumbersInBrackets(input) + require.Equal(t, expected, acutal) }) } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index e540c92fda..aa59a0dc3c 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -16,8 +16,9 @@ func TestLint(t *testing.T) { path := filepath.Join("src", "test", "packages", "12-lint") _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Require an exit code since there was warnings / errors") - require.Contains(t, stderr, "components.0.import: Additional property not-path is not allowed") - require.Contains(t, stderr, "component.[2].import.path") + require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") + require.Contains(t, stderr, "component.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, ".variables: Invalid type. Expected: array, given: null") }) t.Run("zarf test lint success", func(t *testing.T) { diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 3c2d75c816..7c026a15b7 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -1,7 +1,10 @@ kind: ZarfInitConfig metadata: name: init - description: Testing bad yaml + description1: Testing bad yaml + + +variables: components: - name: first-test-component From 47e724d3c9a2545de87153ad172fcd6862005835 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 15:12:38 +0000 Subject: [PATCH 83/87] fix .(root) exit only on error now --- src/pkg/packager/lint/lint.go | 11 +++++++---- src/pkg/packager/lint/lint_test.go | 9 +++++++-- src/pkg/packager/lint/validator.go | 21 ++++++++++----------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index e62a25ab99..7a9001a816 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -62,15 +62,18 @@ func checkForVarInComponentImport(validator *Validator) { } -func wrapNumbersInBrackets(input string) string { +func makeFieldYqEval(field string) string { + if field == "(root)" { + return field + } // . is a non-word chacter (\b) so this gets digits between two . re := regexp.MustCompile(`\b\d+\b`) - wrapped := re.ReplaceAllStringFunc(input, func(match string) string { + wrappedField := re.ReplaceAllStringFunc(field, func(match string) string { return "[" + match + "]" }) - return wrapped + return fmt.Sprintf(".%s", wrappedField) } func validateSchema(validator *Validator) error { @@ -85,7 +88,7 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { err := fmt.Errorf( - ".%s: %s", wrapNumbersInBrackets(desc.Field()), desc.Description()) + "%s: %s", makeFieldYqEval(desc.Field()), desc.Description()) validator.addError(err) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index c932e7452d..3e7166db2a 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -100,9 +100,14 @@ func TestValidateSchema(t *testing.T) { t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { input := "components12.12.import.path" - expected := "components12.[12].import.path" - acutal := wrapNumbersInBrackets(input) + expected := ".components12.[12].import.path" + acutal := makeFieldYqEval(input) require.Equal(t, expected, acutal) }) + t.Run("root doesn't change", func(t *testing.T) { + input := "(root)" + acutal := makeFieldYqEval(input) + require.Equal(t, input, acutal) + }) } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index e299e860d3..46771eb5b1 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -22,18 +22,22 @@ type Validator struct { untypedZarfPackage interface{} } -// DisplayFormattedMessage Displays the message to the user with proper warnings, failures, or success -// Will exit if there are errors +// DisplayFormattedMessage message sent to user based on validator results func (v Validator) DisplayFormattedMessage() { - if v.IsSuccess() { + if !v.hasWarnings() && !v.hasErrors() { message.Success(fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name)) } else { - v.printWarningTable() + v.printValidationTable() } } -func (v Validator) printWarningTable() { - if !v.IsSuccess() { +// IsSuccess returns true if there are not any errors +func (v Validator) IsSuccess() bool { + return !v.hasErrors() +} + +func (v Validator) printValidationTable() { + if v.hasWarnings() || v.hasErrors() { header := []string{"Type", "Message"} connectData := [][]string{} for _, warning := range v.warnings { @@ -63,8 +67,3 @@ func (v *Validator) addWarning(message string) { func (v *Validator) addError(err error) { v.errors = append(v.errors, err) } - -// IsSuccess returns true if there are not any warnings or errors -func (v Validator) IsSuccess() bool { - return !v.hasWarnings() && !v.hasErrors() -} From 170938f76684988dd247e0feaff1559724a66dbc Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 15:37:22 +0000 Subject: [PATCH 84/87] make docs and schema --- docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md index 4de015b12d..bb36206a1d 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md @@ -3,6 +3,10 @@ Verifies the package schema +## Synopsis + +Verifies the package schema and warns the user if they have variables that won't be evaluated + ``` zarf prepare lint [ DIRECTORY ] [flags] ``` From 140a5b8b4e88940a947b33b860c986ac9d97c987 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 16:20:57 +0000 Subject: [PATCH 85/87] refactor validator --- src/pkg/packager/lint/lint.go | 13 ++++++------- src/pkg/packager/lint/lint_test.go | 4 ++-- src/pkg/packager/lint/validator.go | 3 +-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 7a9001a816..82bd52820d 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -62,16 +62,15 @@ func checkForVarInComponentImport(validator *Validator) { } -func makeFieldYqEval(field string) string { +func makeFieldPathYqCompat(field string) string { if field == "(root)" { return field } - // . is a non-word chacter (\b) so this gets digits between two . - re := regexp.MustCompile(`\b\d+\b`) + // \b is a metacharacter that will stop at the next non-word character (including .) + // https://regex101.com/r/pIRPk0/1 + re := regexp.MustCompile(`(\b\d+\b)`) - wrappedField := re.ReplaceAllStringFunc(field, func(match string) string { - return "[" + match + "]" - }) + wrappedField := re.ReplaceAllString(field, "[$1]") return fmt.Sprintf(".%s", wrappedField) } @@ -88,7 +87,7 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { err := fmt.Errorf( - "%s: %s", makeFieldYqEval(desc.Field()), desc.Description()) + "%s: %s", makeFieldPathYqCompat(desc.Field()), desc.Description()) validator.addError(err) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 3e7166db2a..fedbaf49a1 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -101,13 +101,13 @@ func TestValidateSchema(t *testing.T) { t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { input := "components12.12.import.path" expected := ".components12.[12].import.path" - acutal := makeFieldYqEval(input) + acutal := makeFieldPathYqCompat(input) require.Equal(t, expected, acutal) }) t.Run("root doesn't change", func(t *testing.T) { input := "(root)" - acutal := makeFieldYqEval(input) + acutal := makeFieldPathYqCompat(input) require.Equal(t, input, acutal) }) } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index 46771eb5b1..d11f0930fb 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -26,9 +26,8 @@ type Validator struct { func (v Validator) DisplayFormattedMessage() { if !v.hasWarnings() && !v.hasErrors() { message.Success(fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name)) - } else { - v.printValidationTable() } + v.printValidationTable() } // IsSuccess returns true if there are not any errors From b7a7b4c9c92afa3dcc1b11b9fecbe893a8f128c6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 29 Nov 2023 19:35:31 +0000 Subject: [PATCH 86/87] make warning actually yq compatiable, refactor message --- src/pkg/packager/lint/lint.go | 4 ++-- src/pkg/packager/lint/lint_test.go | 4 ++-- src/pkg/packager/lint/validator.go | 2 +- src/test/e2e/12_lint_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 82bd52820d..b9142cc3fb 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -53,10 +53,10 @@ func ValidateZarfSchema(path string) (*Validator, error) { func checkForVarInComponentImport(validator *Validator) { for i, component := range validator.typedZarfPackage.Components { if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".component.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf(".components.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", i)) } if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".component.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) + validator.addWarning(fmt.Sprintf(".components.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index fedbaf49a1..ddc9907e0f 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -94,8 +94,8 @@ func TestValidateSchema(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} checkForVarInComponentImport(&validator) - require.Equal(t, validator.warnings[0], ".component.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1], ".component.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[0], ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") + require.Equal(t, validator.warnings[1], ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") }) t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index d11f0930fb..f88c22de5f 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -25,7 +25,7 @@ type Validator struct { // DisplayFormattedMessage message sent to user based on validator results func (v Validator) DisplayFormattedMessage() { if !v.hasWarnings() && !v.hasErrors() { - message.Success(fmt.Sprintf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name)) + message.Successf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) } v.printValidationTable() } diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index aa59a0dc3c..98257fdc89 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -17,7 +17,7 @@ func TestLint(t *testing.T) { _, stderr, err := e2e.Zarf("prepare", "lint", path) require.Error(t, err, "Require an exit code since there was warnings / errors") require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, "component.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") + require.Contains(t, stderr, ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") require.Contains(t, stderr, ".variables: Invalid type. Expected: array, given: null") }) From a19e30df8a784f88b00fc82651ea9b394aa5db4e Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Wed, 29 Nov 2023 14:44:00 -0500 Subject: [PATCH 87/87] Update docs/3-create-a-zarf-package/index.md Co-authored-by: razzle --- docs/3-create-a-zarf-package/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/3-create-a-zarf-package/index.md b/docs/3-create-a-zarf-package/index.md index 17746a01b5..692f37a9db 100644 --- a/docs/3-create-a-zarf-package/index.md +++ b/docs/3-create-a-zarf-package/index.md @@ -22,7 +22,7 @@ The general flow of a Zarf package deployment on an existing initialized cluster ```shell # Before creating your package you can lint your zarf.yaml -$ zarf package lint +$ zarf prepare lint # To create a package run the following: $ zarf package create