From 69e192d150f44d44c21ae2857c99cc19b85db7f9 Mon Sep 17 00:00:00 2001 From: Edward Natanzon Date: Thu, 29 Feb 2024 16:32:59 +0200 Subject: [PATCH 1/4] Pattern detector works as a regex --- README.md | 2 +- pkg/atlantis/multiple-workspace-discovery.go | 4 ++-- pkg/atlantis/single-workspace-discovery.go | 3 ++- pkg/config/config.go | 2 +- pkg/helpers/helpers.go | 11 ++++++++++ pkg/helpers/helpers_test.go | 22 ++++++++++++++++++++ 6 files changed, 39 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 91e7bc0..5997df9 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Run the tool using the following command: | `-e, --output-type` | Atlantis YAML output type [file stdout] | `OUTPUT_TYPE` | `file` | | `--parallel-apply` | Atlantis parallel apply config value. | `PARALLEL_APPLY` | `true` | | `--parallel-plan` | Atlantis parallel plan config value. | `PARALLEL_PLAN` | `true` | -| `-q, --pattern-detector`| Discover projects based on files or directories names. | `PATTERN_DETECTOR` | `main.tf` | +| `-q, --pattern-detector`| Discover projects based on files, directories names or regex. | `PATTERN_DETECTOR` | `main.tf` | | `-u, --pr-filter` | Filter projects based on the PR changes (Only for github SCM).| `PR_FILTER` | `false` | | `-p, --pull-num` | Github Pull Request Number to check diffs. | `PULL_NUM` | | | `--terraform-base-dir` | Basedir for terraform resources. | `TERRAFORM_BASE_DIR`| `./` | diff --git a/pkg/atlantis/multiple-workspace-discovery.go b/pkg/atlantis/multiple-workspace-discovery.go index 692c77b..d8a202b 100644 --- a/pkg/atlantis/multiple-workspace-discovery.go +++ b/pkg/atlantis/multiple-workspace-discovery.go @@ -12,7 +12,7 @@ import ( func multiWorkspaceGetProjectScope(relPath, patternDetector string, changedFiles []string) string { for _, file := range changedFiles { if strings.HasPrefix(file, fmt.Sprintf("%s/", relPath)) && - !strings.Contains(file, patternDetector) { + !helpers.MatchesPattern(patternDetector, file) { return "crossWorkspace" } } @@ -58,6 +58,6 @@ func multiWorkspaceDetectProjectWorkspaces(changedFiles []string, enablePRFilter func multiWorkspaceDiscoveryFilter(info os.FileInfo, path, patternDetector string) bool { return info.IsDir() && - info.Name() == patternDetector && + helpers.MatchesPattern(patternDetector, info.Name()) && !strings.Contains(path, ".terraform") } diff --git a/pkg/atlantis/single-workspace-discovery.go b/pkg/atlantis/single-workspace-discovery.go index 2b2e402..44c6414 100644 --- a/pkg/atlantis/single-workspace-discovery.go +++ b/pkg/atlantis/single-workspace-discovery.go @@ -1,13 +1,14 @@ package atlantis import ( + "github.com/totmicro/atlantis-yaml-generator/pkg/helpers" "os" "strings" ) func singleWorkspaceDiscoveryFilter(info os.FileInfo, path, patternDetector string) bool { return !info.IsDir() && - info.Name() == patternDetector && + helpers.MatchesPattern(patternDetector, info.Name()) && !strings.Contains(path, ".terraform") } diff --git a/pkg/config/config.go b/pkg/config/config.go index 3d4ce48..6da4eb3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -103,7 +103,7 @@ var ParameterList = []Parameter{ }, { Name: "pattern-detector", - Description: "discover projects based on files or directories names.", + Description: "discover projects based on files, directories names or regex.", Required: false, DefaultValue: "main.tf", Shorthand: "q", diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index a6a5ce2..6f50ea7 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -3,6 +3,7 @@ package helpers import ( "os" "path/filepath" + "regexp" "strings" ) @@ -51,3 +52,13 @@ func ReadFile(filename string) (string, error) { _, err = file.Read(content) return string(content), err } + +// MatchesPattern checks if the given string matches the specified regex pattern. +// It returns true if the pattern matches the string, and false otherwise. +func MatchesPattern(pattern string, str string) bool { + matched, err := regexp.MatchString(pattern, str) + if err != nil { + return false + } + return matched +} diff --git a/pkg/helpers/helpers_test.go b/pkg/helpers/helpers_test.go index f59a21d..f8020ec 100644 --- a/pkg/helpers/helpers_test.go +++ b/pkg/helpers/helpers_test.go @@ -175,3 +175,25 @@ func TestReadFile_ReadError(t *testing.T) { _, err = ReadFile(tempFile.Name()) assert.Error(t, err) // Check that an error is returned } + +func TestMatchesPattern(t *testing.T) { + tests := []struct { + pattern string + str string + want bool + }{ + {"workspace_vars", "workspace_vars", true}, + {"main.tf", "main.tf", true}, + {".*\\.tf", "main.tf", true}, + {".*\\.tf", "vpc.tf", true}, + {"^a.*z$", "alphabet", false}, // Negative test case + } + + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + if got := MatchesPattern(tt.pattern, tt.str); got != tt.want { + t.Errorf("MatchesPattern(%q, %q) = %v, want %v", tt.pattern, tt.str, got, tt.want) + } + }) + } +} From d5c2bbdcce8b4ec705939760acdaaf4b9debd526 Mon Sep 17 00:00:00 2001 From: Edward Natanzon Date: Thu, 29 Feb 2024 17:28:34 +0200 Subject: [PATCH 2/4] Fix non-unique list of projects --- pkg/atlantis/atlantis.go | 11 ++++++++- pkg/helpers/helpers.go | 49 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/pkg/atlantis/atlantis.go b/pkg/atlantis/atlantis.go index 1fa906f..1e7c8eb 100644 --- a/pkg/atlantis/atlantis.go +++ b/pkg/atlantis/atlantis.go @@ -41,6 +41,11 @@ type ProjectFolder struct { WorkspaceList []string } +func (pf ProjectFolder) Hash() string { + // Implement a unique string generation based on the content of atlantis.ProjectFolder + return fmt.Sprintf("%s", pf.Path) +} + // GenerateAtlantisYAML generates the atlantis.yaml file func GenerateAtlantisYAML() error { @@ -121,17 +126,21 @@ func GenerateAtlantisYAML() error { } func scanProjectFolders(basePath, discoveryMode, patternDetector string) (projectFolders []ProjectFolder, err error) { + uniques := helpers.NewSet() err = filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error { if err != nil || info == nil { return err } if discoveryFilter(info, path, discoveryMode, patternDetector) { relPath, _ := filepath.Rel(basePath, filepath.Dir(path)) - projectFolders = append(projectFolders, ProjectFolder{Path: relPath}) + uniques.Add(ProjectFolder{Path: relPath}) } return nil }) + for _, projectFolder := range uniques.Elements { + projectFolders = append(projectFolders, projectFolder.(ProjectFolder)) + } return projectFolders, err } diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index 6f50ea7..966347e 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -62,3 +62,52 @@ func MatchesPattern(pattern string, str string) bool { } return matched } + +// Set data structure, using map as the underlying storage +type Set struct { + Elements map[string]Hashable +} + +// NewSet creates a new Set +func NewSet() *Set { + return &Set{ + Elements: make(map[string]Hashable), + } +} + +// Add adds an element to the set +func (s *Set) Add(element Hashable) { + key := element.Hash() + s.Elements[key] = element +} + +// Remove removes an element from the set +func (s *Set) Remove(element Hashable) { + key := element.Hash() + delete(s.Elements, key) +} + +// Contains checks if an element is in the set +func (s *Set) Contains(element Hashable) bool { + key := element.Hash() + _, exists := s.Elements[key] + return exists +} + +// Size returns the number of Elements in the set +func (s *Set) Size() int { + return len(s.Elements) +} + +// List returns all the Elements in the set +func (s *Set) List() []Hashable { + list := make([]Hashable, 0, len(s.Elements)) + for _, element := range s.Elements { + list = append(list, element) + } + return list +} + +type Hashable interface { + Hash() string +} From de10d00a56213aa168c93fe4da968001d00dde41 Mon Sep 17 00:00:00 2001 From: Edward Natanzon Date: Mon, 4 Mar 2024 08:55:05 +0200 Subject: [PATCH 3/4] Added test that ensures no duplicate projects are returned --- go.mod | 6 +++++- go.sum | 4 ++++ pkg/atlantis/atlantis.go | 12 +++++++++-- pkg/atlantis/atlantis_test.go | 40 +++++++++++++++++++++++++++++------ pkg/helpers/helpers.go | 8 ++++++- 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 4a2f644..1aca823 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module github.com/totmicro/atlantis-yaml-generator go 1.21 -require github.com/google/go-github v17.0.0+incompatible +require ( + github.com/google/go-github v17.0.0+incompatible + github.com/spf13/afero v1.11.0 +) require ( github.com/davecgh/go-spew v1.1.1 // indirect @@ -13,6 +16,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index 0aa3169..87dc5c1 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -58,6 +60,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/pkg/atlantis/atlantis.go b/pkg/atlantis/atlantis.go index 1e7c8eb..8f708e8 100644 --- a/pkg/atlantis/atlantis.go +++ b/pkg/atlantis/atlantis.go @@ -46,6 +46,12 @@ func (pf ProjectFolder) Hash() string { return fmt.Sprintf("%s", pf.Path) } +type OSFileSystem struct{} + +func (OSFileSystem) Walk(root string, walkFn filepath.WalkFunc) error { + return filepath.Walk(root, walkFn) +} + // GenerateAtlantisYAML generates the atlantis.yaml file func GenerateAtlantisYAML() error { @@ -64,6 +70,7 @@ func GenerateAtlantisYAML() error { // Scan folders to detect projects projectFoldersList, err := scanProjectFolders( + OSFileSystem{}, config.GlobalConfig.Parameters["terraform-base-dir"], config.GlobalConfig.Parameters["discovery-mode"], config.GlobalConfig.Parameters["pattern-detector"], @@ -125,9 +132,10 @@ func GenerateAtlantisYAML() error { return nil } -func scanProjectFolders(basePath, discoveryMode, patternDetector string) (projectFolders []ProjectFolder, err error) { +// Scans a folder for projects and returns a list of unique projects. +func scanProjectFolders(filesystem helpers.Walkable, basePath, discoveryMode, patternDetector string) (projectFolders []ProjectFolder, err error) { uniques := helpers.NewSet() - err = filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error { + err = filesystem.Walk(basePath, func(path string, info os.FileInfo, err error) error { if err != nil || info == nil { return err } diff --git a/pkg/atlantis/atlantis_test.go b/pkg/atlantis/atlantis_test.go index 75f1a71..ff98131 100644 --- a/pkg/atlantis/atlantis_test.go +++ b/pkg/atlantis/atlantis_test.go @@ -1,15 +1,15 @@ package atlantis import ( - "os" - "path/filepath" - "reflect" - "testing" - + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/totmicro/atlantis-yaml-generator/pkg/config" "github.com/totmicro/atlantis-yaml-generator/pkg/helpers" "gopkg.in/yaml.v3" + "os" + "path/filepath" + "reflect" + "testing" ) func TestApplyProjectFilter(t *testing.T) { @@ -387,7 +387,11 @@ func TestScanProjectFolders(t *testing.T) { for _, test := range tests { t.Run(test.discoveryMode, func(t *testing.T) { - projectFolders, err := scanProjectFolders(test.basePath, test.discoveryMode, test.patternDetector) + projectFolders, err := scanProjectFolders( + OSFileSystem{}, + test.basePath, + test.discoveryMode, + test.patternDetector) if test.expectedError { assert.Error(t, err) } else { @@ -720,3 +724,27 @@ func TestGenerateAtlantisYAML(t *testing.T) { assert.NoError(t, err) } + +// Tests multiple project hits returns unique projects only. +// e.g. if we scan for *.tf the same project isn't hit twice. +func TestScanProjectFoldersUniques(t *testing.T) { + memFS := afero.NewMemMapFs() + fs := afero.Afero{Fs: memFS} + // Create directories and files + // project3 has multiple hits + afero.WriteFile(fs, "projects_root/project1/main.tf", []byte("content"), 0644) + afero.WriteFile(fs, "projects_root/project2/main.tf", []byte("content"), 0644) + afero.WriteFile(fs, "projects_root/project3/main.tf", []byte("content"), 0644) + afero.WriteFile(fs, "projects_root/project3/outputs.tf", []byte("content"), 0644) + + // Use the fs (implementing Walkable) to call scanProjectFolders + projectFolders, err := scanProjectFolders(fs, "projects_root", "single-workspace", `.*\.tf`) + if err != nil { + t.Errorf("scanProjectFolders returned an error: %v", err) + } + + // Verify that 3 project folders were returned + if len(projectFolders) != 3 { + t.Errorf("Expected 3 project folders, got %d. Projects %v", len(projectFolders), projectFolders) + } +} diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index 966347e..2f9ebba 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -63,7 +63,8 @@ func MatchesPattern(pattern string, str string) bool { return matched } -// Set data structure, using map as the underlying storage +// Set data structure +// Keys are strings and elements must implement Hashable to calculate keys. type Set struct { Elements map[string]Hashable } @@ -108,6 +109,11 @@ func (s *Set) List() []Hashable { return list } +// Enables use of Set by requiring its elements to be hashable. type Hashable interface { Hash() string } + +type Walkable interface { + Walk(root string, walkFn filepath.WalkFunc) error +} From 583dbb84784fd31c7dabbd3b0dc8e6396868b401 Mon Sep 17 00:00:00 2001 From: Edward Natanzon Date: Mon, 4 Mar 2024 09:28:49 +0200 Subject: [PATCH 4/4] Added tests for Set --- pkg/helpers/helpers_test.go | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pkg/helpers/helpers_test.go b/pkg/helpers/helpers_test.go index f8020ec..735e4db 100644 --- a/pkg/helpers/helpers_test.go +++ b/pkg/helpers/helpers_test.go @@ -3,6 +3,7 @@ package helpers import ( "fmt" "os" + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -197,3 +198,57 @@ func TestMatchesPattern(t *testing.T) { }) } } + +// HashableString is a simple Hashable type for testing. +type HashableString string + +func (h HashableString) Hash() string { + return string(h) +} + +func TestNewSet(t *testing.T) { + s := NewSet() + if s == nil || len(s.Elements) != 0 { + t.Errorf("NewSet() = %v, want a new Set instance with empty Elements", s) + } +} + +func TestAddAndContains(t *testing.T) { + s := NewSet() + element := HashableString("test") + s.Add(element) + if !s.Contains(element) { + t.Errorf("Set does not contain element %v after Add", element) + } +} + +func TestRemove(t *testing.T) { + s := NewSet() + element := HashableString("test") + s.Add(element) + s.Remove(element) + if s.Contains(element) { + t.Errorf("Set contains element %v after Remove", element) + } +} + +func TestSize(t *testing.T) { + s := NewSet() + s.Add(HashableString("one")) + s.Add(HashableString("two")) + if s.Size() != 2 { + t.Errorf("Size() = %d, want 2", s.Size()) + } +} + +func TestList(t *testing.T) { + s := NewSet() + elements := []Hashable{HashableString("one"), HashableString("two")} + for _, e := range elements { + s.Add(e) + } + list := s.List() + if !reflect.DeepEqual(list, elements) && len(list) == len(elements) { + t.Errorf("List() = %v, want %v", list, elements) + } +}