diff --git a/go.mod b/go.mod index 25832aa4..0a4746ea 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( cloud.google.com/go/resourcemanager v1.9.4 cloud.google.com/go/serviceusage v1.6.0 cloud.google.com/go/storage v1.35.1 + dario.cat/mergo v1.0.1 filippo.io/age v1.1.1 github.com/AlecAivazis/survey/v2 v2.3.5 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 @@ -40,6 +41,7 @@ require ( github.com/ktrysmt/go-bitbucket v0.9.55 github.com/likexian/doh v0.7.1 github.com/linode/linodego v1.26.0 + github.com/lithammer/dedent v1.1.0 github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.5.0 @@ -47,8 +49,8 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/packethost/packngo v0.29.0 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 - github.com/pluralsh/console/go/client v1.22.3 - github.com/pluralsh/console/go/controller v0.0.0-20240918005717-8285a4b181b1 + github.com/pluralsh/console/go/client v1.23.0 + github.com/pluralsh/console/go/controller v0.0.0-20241106170618-0c255ee72c4c github.com/pluralsh/gqlclient v1.12.2 github.com/pluralsh/plural-operator v0.5.5 github.com/pluralsh/polly v0.1.10 @@ -67,6 +69,7 @@ require ( golang.org/x/oauth2 v0.19.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 + gotest.tools/v3 v3.4.0 helm.sh/helm/v3 v3.14.3 k8s.io/api v0.30.1 k8s.io/apimachinery v0.31.0-beta.0 @@ -80,7 +83,6 @@ require ( require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/longrunning v0.5.4 // indirect - dario.cat/mergo v1.0.0 // indirect github.com/99designs/gqlgen v0.17.49 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect diff --git a/go.sum b/go.sum index e9fb9d36..93341925 100644 --- a/go.sum +++ b/go.sum @@ -610,8 +610,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/age v1.1.1 h1:pIpO7l151hCnQ4BdyBujnGP2YlUo0uj6sAVNHGBvXHg= filippo.io/age v1.1.1/go.mod h1:l03SrzDUrBkdBx8+IILdnn2KZysqQdbEBUQ4p3sqEQE= @@ -1502,6 +1502,8 @@ github.com/likexian/gokit v0.25.15 h1:QjospM1eXhdMMHwZRpMKKAHY/Wig9wgcREmLtf9Nsl github.com/likexian/gokit v0.25.15/go.mod h1:S2QisdsxLEHWeD/XI0QMVeggp+jbxYqUxMvSBil7MRg= github.com/linode/linodego v1.26.0 h1:2tOZ3Wxn4YvGBRgZi3Vz6dab+L16XUntJ9sJxh3ZBio= github.com/linode/linodego v1.26.0/go.mod h1:kD7Bf1piWg/AXb9TA0ThAVwzR+GPf6r2PvbTbVk7PMA= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= @@ -1699,10 +1701,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pluralsh/console/go/client v1.22.3 h1:5CUV4E/EH5G84ZVIIdr4NuUzM92AKUrZBLEh2SjLeEc= -github.com/pluralsh/console/go/client v1.22.3/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k= -github.com/pluralsh/console/go/controller v0.0.0-20240918005717-8285a4b181b1 h1:AXudlzS4Q8Y8J+0Q+kb8b4D/2tok4mryTJOKRvRlzJA= -github.com/pluralsh/console/go/controller v0.0.0-20240918005717-8285a4b181b1/go.mod h1:B0WeS6z0Ila4kTBosiMOjNsTKNHrvXRJuLoPT37MEzU= +github.com/pluralsh/console/go/client v1.23.0 h1:aS6CuC0o5ONSoRKFbm/rb6JVDD858KIs0t+HLLMMX/w= +github.com/pluralsh/console/go/client v1.23.0/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k= +github.com/pluralsh/console/go/controller v0.0.0-20241106170618-0c255ee72c4c h1:1bSNjddgWG/C0Zgm0BoOOngTlv+sFPvBjxo18I7jEdw= +github.com/pluralsh/console/go/controller v0.0.0-20241106170618-0c255ee72c4c/go.mod h1:MN71cuC4jWgH2Lk3SYNNno2FZP22u0uGsL/MqyreBs0= github.com/pluralsh/controller-reconcile-helper v0.0.4 h1:1o+7qYSyoeqKFjx+WgQTxDz4Q2VMpzprJIIKShxqG0E= github.com/pluralsh/controller-reconcile-helper v0.0.4/go.mod h1:AfY0gtteD6veBjmB6jiRx/aR4yevEf6K0M13/pGan/s= github.com/pluralsh/gqlclient v1.12.2 h1:BrEFAASktf4quFw57CIaLAd+NZUTLhG08fe6tnhBQN4= @@ -2958,7 +2960,6 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= diff --git a/pkg/pr/apply_test.go b/pkg/pr/apply_test.go new file mode 100644 index 00000000..ec2c1d4c --- /dev/null +++ b/pkg/pr/apply_test.go @@ -0,0 +1,315 @@ +package pr_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/samber/lo" + "gotest.tools/v3/assert" + + "github.com/pluralsh/plural-cli/pkg/pr" +) + +func createFile(path, content string) (*os.File, error) { + f, err := os.Create(path) + if err != nil { + return nil, err + } + + _, err = f.WriteString(content) + return f, err +} + +func createFiles(fileMap map[string]string) (func(), error) { + files := make([]*os.File, len(fileMap)) + for path, content := range fileMap { + f, err := createFile(path, content) + if err != nil { + return nil, err + } + + files = append(files, f) + } + + return func() { + for _, file := range files { + file.Close() + } + }, nil +} + +func readFiles(paths []string) (map[string]string, error) { + files := make(map[string]string, len(paths)) + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + files[path] = string(content) + } + + return files, nil +} + +// Notes: +// - YAML encoder adds a new line at the end! +// - YAML encoder can reorder fields compared to the overlay YAML. +// Output YAML field order is stable though. +const ( + baseYAMLIn = `include: + - directory: foo/foo1 + - directory: foo/foo2 +stringtest: old` + + overlayYAML = `include: + - directory: foo/foo1 + extra: true + version: '{{ context.version }}' + stuff: + stuff1: true + stuff2: true + - directory: something/else +stringtest: new +nulltest: ~` + + baseYAMLTemplated = `include: + - directory: foo/foo1 + extra: true + stuff: + stuff1: true + stuff2: true + version: "1.28" + - directory: something/else +nulltest: null +stringtest: new +` + + baseYAMLNonTemplated = `include: + - directory: foo/foo1 + extra: true + stuff: + stuff1: true + stuff2: true + version: '{{ context.version }}' + - directory: something/else +nulltest: null +stringtest: new +` + + baseYAMLAppend = `include: + - directory: foo/foo1 + - directory: foo/foo2 + - directory: foo/foo1 + extra: true + stuff: + stuff1: true + stuff2: true + version: "1.28" + - directory: something/else +nulltest: null +stringtest: new +` + + baseYAMLAppendNonTemplated = `include: + - directory: foo/foo1 + - directory: foo/foo2 + - directory: foo/foo1 + extra: true + stuff: + stuff1: true + stuff2: true + version: '{{ context.version }}' + - directory: something/else +nulltest: null +stringtest: new +` +) + +func TestApply(t *testing.T) { + dir := t.TempDir() + cases := []struct { + name string + files map[string]string + template *pr.PrTemplate + expectedFiles map[string]string + expectedErr error + }{ + { + name: "should work with single line regex replacements", + files: map[string]string{ + filepath.Join(dir, "workload.tf"): ` + module "staging" { + source = "./eks" + cluster_name = "boot-staging" + vpc_name = "plural-stage" + kubernetes_version = "1.22" + create_db = false + providers = { + helm = helm.staging + } + }`, + }, + template: &pr.PrTemplate{ + Context: map[string]interface{}{ + "version": "1.28", + }, + Spec: pr.PrTemplateSpec{ + Updates: &pr.UpdateSpec{ + RegexReplacements: []pr.RegexReplacement{ + { + Regex: "kubernetes_version = \"1.[0-9]+\"", + Replacement: "kubernetes_version = \"{{ context.version }}\"", + File: filepath.Join(dir, "workload.tf"), + Templated: false, + }, + }, + }, + }, + }, + expectedFiles: map[string]string{ + filepath.Join(dir, "workload.tf"): ` + module "staging" { + source = "./eks" + cluster_name = "boot-staging" + vpc_name = "plural-stage" + kubernetes_version = "1.28" + create_db = false + providers = { + helm = helm.staging + } + }`, + }, + expectedErr: nil, + }, + { + name: "should template and overlay with overwrite yaml file", + files: map[string]string{ + filepath.Join(dir, "base.yaml"): baseYAMLIn, + }, + template: &pr.PrTemplate{ + Context: map[string]interface{}{ + "version": "1.28", + }, + Spec: pr.PrTemplateSpec{ + Updates: &pr.UpdateSpec{ + YamlOverlays: []pr.YamlOverlay{ + { + File: filepath.Join(dir, "base.yaml"), + Yaml: overlayYAML, + ListMerge: pr.ListMergeOverwrite, + Templated: true, + }, + }, + }, + }, + }, + expectedFiles: map[string]string{ + filepath.Join(dir, "base.yaml"): baseYAMLTemplated, + }, + expectedErr: nil, + }, + { + name: "should not template and overlay with overwrite yaml file", + files: map[string]string{ + filepath.Join(dir, "base.yaml"): baseYAMLIn, + }, + template: &pr.PrTemplate{ + Context: map[string]interface{}{ + "version": "1.28", + }, + Spec: pr.PrTemplateSpec{ + Updates: &pr.UpdateSpec{ + YamlOverlays: []pr.YamlOverlay{ + { + File: filepath.Join(dir, "base.yaml"), + Yaml: overlayYAML, + ListMerge: pr.ListMergeOverwrite, + Templated: false, + }, + }, + }, + }, + }, + expectedFiles: map[string]string{ + filepath.Join(dir, "base.yaml"): baseYAMLNonTemplated, + }, + expectedErr: nil, + }, + { + name: "should template and overlay with append yaml file", + files: map[string]string{ + filepath.Join(dir, "base.yaml"): baseYAMLIn, + }, + template: &pr.PrTemplate{ + Context: map[string]interface{}{ + "version": "1.28", + }, + Spec: pr.PrTemplateSpec{ + Updates: &pr.UpdateSpec{ + YamlOverlays: []pr.YamlOverlay{ + { + File: filepath.Join(dir, "base.yaml"), + Yaml: overlayYAML, + ListMerge: pr.ListMergeAppend, + Templated: true, + }, + }, + }, + }, + }, + expectedFiles: map[string]string{ + filepath.Join(dir, "base.yaml"): baseYAMLAppend, + }, + expectedErr: nil, + }, + { + name: "should not template and overlay with append yaml file", + files: map[string]string{ + filepath.Join(dir, "base.yaml"): baseYAMLIn, + }, + template: &pr.PrTemplate{ + Context: map[string]interface{}{ + "version": "1.28", + }, + Spec: pr.PrTemplateSpec{ + Updates: &pr.UpdateSpec{ + YamlOverlays: []pr.YamlOverlay{ + { + File: filepath.Join(dir, "base.yaml"), + Yaml: overlayYAML, + ListMerge: pr.ListMergeAppend, + Templated: false, + }, + }, + }, + }, + }, + expectedFiles: map[string]string{ + filepath.Join(dir, "base.yaml"): baseYAMLAppendNonTemplated, + }, + expectedErr: nil, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cleanupFunc, err := createFiles(c.files) + assert.NilError(t, err) + defer cleanupFunc() + + err = pr.Apply(c.template) + assert.ErrorIs(t, err, c.expectedErr) + + files, err := readFiles(lo.Keys(c.expectedFiles)) + assert.NilError(t, err) + + for file, content := range files { + expectedContent, exists := c.expectedFiles[file] + assert.Check(t, exists) + assert.Equal(t, content, expectedContent) + } + }) + } +} diff --git a/pkg/pr/crd.go b/pkg/pr/crd.go index f3e65186..c2a936e6 100644 --- a/pkg/pr/crd.go +++ b/pkg/pr/crd.go @@ -5,11 +5,13 @@ import ( "os" "strings" + "github.com/samber/lo" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/pluralsh/plural-cli/pkg/api" "github.com/pluralsh/plural-cli/pkg/bundle" "github.com/pluralsh/plural-cli/pkg/manifest" "github.com/pluralsh/plural-cli/pkg/utils" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/pluralsh/console/go/controller/api/v1alpha1" "github.com/pluralsh/polly/algorithms" @@ -152,6 +154,7 @@ func updates(pr *v1alpha1.PrAutomation) *UpdateSpec { Regexes: make([]string, 0), Files: make([]string, 0), RegexReplacements: make([]RegexReplacement, 0), + YamlOverlays: make([]YamlOverlay, 0), } if u.ReplaceTemplate != nil { prUpdates.ReplaceTemplate = *u.ReplaceTemplate @@ -175,16 +178,22 @@ func updates(pr *v1alpha1.PrAutomation) *UpdateSpec { } if len(u.RegexReplacements) > 0 { prUpdates.RegexReplacements = algorithms.Map(u.RegexReplacements, func(t v1alpha1.RegexReplacement) RegexReplacement { - r := RegexReplacement{ + return RegexReplacement{ Regex: t.Regex, Replacement: t.Replacement, File: t.File, - Templated: false, + Templated: lo.FromPtr(t.Templated), } - if t.Templated != nil { - r.Templated = *t.Templated + }) + } + if len(u.YamlOverlays) > 0 { + prUpdates.YamlOverlays = algorithms.Map(u.YamlOverlays, func(y v1alpha1.YamlOverlay) YamlOverlay { + return YamlOverlay{ + File: y.File, + Yaml: y.Yaml, + Templated: lo.FromPtr(y.Templated), + ListMerge: toListMerge(y.ListMerge), } - return r }) } return prUpdates diff --git a/pkg/pr/types.go b/pkg/pr/types.go index dfcb8362..9b3f3f49 100644 --- a/pkg/pr/types.go +++ b/pkg/pr/types.go @@ -2,7 +2,9 @@ package pr import ( "os" + "strings" + console "github.com/pluralsh/console/go/client" "sigs.k8s.io/yaml" ) @@ -27,6 +29,37 @@ type UpdateSpec struct { Yq string `json:"yq"` MatchStrategy string `json:"match_strategy"` RegexReplacements []RegexReplacement `json:"regex_replacements"` + YamlOverlays []YamlOverlay `json:"yaml_overlays"` +} + +type ListMerge string + +func toListMerge(listMerge *console.ListMerge) ListMerge { + // default to overwrite + if listMerge == nil { + return ListMergeOverwrite + } + + switch strings.ToUpper(string(*listMerge)) { + case string(console.ListMergeOverwrite): + return ListMergeOverwrite + case string(console.ListMergeAppend): + return ListMergeAppend + } + + return ListMergeOverwrite +} + +const ( + ListMergeAppend = "APPEND" + ListMergeOverwrite = "OVERWRITE" +) + +type YamlOverlay struct { + File string `json:"file"` + Yaml string `json:"yaml"` + ListMerge ListMerge `json:"list_merge"` + Templated bool `json:"templated"` } type CreateSpec struct { diff --git a/pkg/pr/updates.go b/pkg/pr/updates.go index 150f44bc..88129fc0 100644 --- a/pkg/pr/updates.go +++ b/pkg/pr/updates.go @@ -1,11 +1,14 @@ package pr import ( + "bytes" "io/fs" "path/filepath" "regexp" + "dario.cat/mergo" "github.com/samber/lo" + "gopkg.in/yaml.v3" ) func applyUpdates(updates *UpdateSpec, ctx map[string]interface{}) error { @@ -17,6 +20,10 @@ func applyUpdates(updates *UpdateSpec, ctx map[string]interface{}) error { return err } + if err := processYamlOverlays(updates.YamlOverlays, ctx); err != nil { + return err + } + replacement, err := templateReplacement([]byte(updates.ReplaceTemplate), ctx) if err != nil { return err @@ -84,6 +91,70 @@ func processRegexReplacements(replacements []RegexReplacement, ctx map[string]in return nil } +func processYamlOverlays(overlays []YamlOverlay, ctx map[string]interface{}) error { + if len(overlays) == 0 { + return nil + } + + for _, overlay := range overlays { + var err error + var overlayYaml = []byte(overlay.Yaml) + + if overlay.Templated { + overlayYaml, err = templateReplacement([]byte(overlay.Yaml), ctx) + if err != nil { + return err + } + } + + mergeFunc := func(data []byte) ([]byte, error) { + return mergeYaml(data, overlayYaml, overlay.ListMerge) + } + + if err = replaceInPlace(overlay.File, mergeFunc); err != nil { + return err + } + } + + return nil +} + +func mergeYaml(base, overlay []byte, merge ListMerge) ([]byte, error) { + baseMap := make(map[string]interface{}) + overlayMap := make(map[string]interface{}) + + if err := yaml.Unmarshal(base, &baseMap); err != nil { + return nil, err + } + + if err := yaml.Unmarshal(overlay, &overlayMap); err != nil { + return nil, err + } + + options := []func(*mergo.Config){mergo.WithOverride, mergo.WithSliceDeepCopy} + if merge == ListMergeAppend { + options = append(options, mergo.WithAppendSlice) + } + + if err := mergo.Merge( + &baseMap, + overlayMap, + options..., + ); err != nil { + return nil, err + } + + var b bytes.Buffer + encoder := yaml.NewEncoder(&b) + encoder.SetIndent(2) + if err := encoder.Encode(baseMap); err != nil { + return nil, err + } + + defer encoder.Close() + return b.Bytes(), nil +} + func updateFile(path string, updates *UpdateSpec, replacement []byte) error { switch updates.MatchStrategy { case "any":