Merge pull request totmicro#5 from w-edd/pattern-detector-regex

Pattern detector works as a regex

README.md

@@ -39,7 +39,7 @@ Run the tool using the following command:
 | `-e, --output-type`    | Atlantis YAML output type [file stdout]                      | `OUTPUT_TYPE`       | `file`          |
 | `--parallel-apply`     | Atlantis parallel apply config value.                         | `PARALLEL_APPLY`    | `true`          |
 | `--parallel-plan`      | Atlantis parallel plan config value.                          | `PARALLEL_PLAN`    | `true`          |
-| `-q, --pattern-detector`| Discover projects based on files or directories names.      | `PATTERN_DETECTOR`  | `main.tf`      |
+| `-q, --pattern-detector`| Discover projects based on files, directories names or regex.      | `PATTERN_DETECTOR`  | `main.tf`      |
 | `-u, --pr-filter`      | Filter projects based on the PR changes (Only for github SCM).| `PR_FILTER`       | `false`          |
 | `-p, --pull-num`       | Github Pull Request Number to check diffs.                    | `PULL_NUM`          |               |
 | `--terraform-base-dir` | Basedir for terraform resources.                               | `TERRAFORM_BASE_DIR`| `./`            |

go.mod

@@ -2,7 +2,10 @@ module github.com/totmicro/atlantis-yaml-generator
 
 go 1.21
 
-require github.com/google/go-github v17.0.0+incompatible
+require (
+	github.com/google/go-github v17.0.0+incompatible
+	github.com/spf13/afero v1.11.0
+)
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -13,6 +16,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

go.sum

@@ -30,6 +30,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -58,6 +60,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

pkg/atlantis/atlantis.go

@@ -41,6 +41,17 @@ type ProjectFolder struct {
 	WorkspaceList []string
 }
 
+func (pf ProjectFolder) Hash() string {
+	// Implement a unique string generation based on the content of atlantis.ProjectFolder
+	return fmt.Sprintf("%s", pf.Path)
+}
+
+type OSFileSystem struct{}
+
+func (OSFileSystem) Walk(root string, walkFn filepath.WalkFunc) error {
+	return filepath.Walk(root, walkFn)
+}
+
 // GenerateAtlantisYAML generates the atlantis.yaml file
 func GenerateAtlantisYAML() error {
 
@@ -59,6 +70,7 @@ func GenerateAtlantisYAML() error {
 
 	// Scan folders to detect projects
 	projectFoldersList, err := scanProjectFolders(
+		OSFileSystem{},
 		config.GlobalConfig.Parameters["terraform-base-dir"],
 		config.GlobalConfig.Parameters["discovery-mode"],
 		config.GlobalConfig.Parameters["pattern-detector"],
@@ -120,18 +132,23 @@ func GenerateAtlantisYAML() error {
 	return nil
 }
 
-func scanProjectFolders(basePath, discoveryMode, patternDetector string) (projectFolders []ProjectFolder, err error) {
-	err = filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error {
+// Scans a folder for projects and returns a list of unique projects.
+func scanProjectFolders(filesystem helpers.Walkable, basePath, discoveryMode, patternDetector string) (projectFolders []ProjectFolder, err error) {
+	uniques := helpers.NewSet()
+	err = filesystem.Walk(basePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil || info == nil {
 			return err
 		}
 		if discoveryFilter(info, path, discoveryMode, patternDetector) {
 			relPath, _ := filepath.Rel(basePath, filepath.Dir(path))
-			projectFolders = append(projectFolders, ProjectFolder{Path: relPath})
+			uniques.Add(ProjectFolder{Path: relPath})
 		}
 		return nil
 	})
 
+	for _, projectFolder := range uniques.Elements {
+		projectFolders = append(projectFolders, projectFolder.(ProjectFolder))
+	}
 	return projectFolders, err
 }
 

pkg/atlantis/atlantis_test.go

@@ -1,15 +1,15 @@
 package atlantis
 
 import (
-	"os"
-	"path/filepath"
-	"reflect"
-	"testing"
-
+	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/totmicro/atlantis-yaml-generator/pkg/config"
 	"github.com/totmicro/atlantis-yaml-generator/pkg/helpers"
 	"gopkg.in/yaml.v3"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
 )
 
 func TestApplyProjectFilter(t *testing.T) {
@@ -387,7 +387,11 @@ func TestScanProjectFolders(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.discoveryMode, func(t *testing.T) {
-			projectFolders, err := scanProjectFolders(test.basePath, test.discoveryMode, test.patternDetector)
+			projectFolders, err := scanProjectFolders(
+				OSFileSystem{},
+				test.basePath,
+				test.discoveryMode,
+				test.patternDetector)
 			if test.expectedError {
 				assert.Error(t, err)
 			} else {
@@ -720,3 +724,27 @@ func TestGenerateAtlantisYAML(t *testing.T) {
 	assert.NoError(t, err)
 
 }
+
+// Tests multiple project hits returns unique projects only.
+// e.g. if we scan for *.tf the same project isn't hit twice.
+func TestScanProjectFoldersUniques(t *testing.T) {
+	memFS := afero.NewMemMapFs()
+	fs := afero.Afero{Fs: memFS}
+	// Create directories and files
+	// project3 has multiple hits
+	afero.WriteFile(fs, "projects_root/project1/main.tf", []byte("content"), 0644)
+	afero.WriteFile(fs, "projects_root/project2/main.tf", []byte("content"), 0644)
+	afero.WriteFile(fs, "projects_root/project3/main.tf", []byte("content"), 0644)
+	afero.WriteFile(fs, "projects_root/project3/outputs.tf", []byte("content"), 0644)
+
+	// Use the fs (implementing Walkable) to call scanProjectFolders
+	projectFolders, err := scanProjectFolders(fs, "projects_root", "single-workspace", `.*\.tf`)
+	if err != nil {
+		t.Errorf("scanProjectFolders returned an error: %v", err)
+	}
+
+	// Verify that 3 project folders were returned
+	if len(projectFolders) != 3 {
+		t.Errorf("Expected 3 project folders, got %d. Projects %v", len(projectFolders), projectFolders)
+	}
+}

pkg/atlantis/multiple-workspace-discovery.go

@@ -12,7 +12,7 @@ import (
 func multiWorkspaceGetProjectScope(relPath, patternDetector string, changedFiles []string) string {
 	for _, file := range changedFiles {
 		if strings.HasPrefix(file, fmt.Sprintf("%s/", relPath)) &&
-			!strings.Contains(file, patternDetector) {
+			!helpers.MatchesPattern(patternDetector, file) {
 			return "crossWorkspace"
 		}
 	}
@@ -58,6 +58,6 @@ func multiWorkspaceDetectProjectWorkspaces(changedFiles []string, enablePRFilter
 
 func multiWorkspaceDiscoveryFilter(info os.FileInfo, path, patternDetector string) bool {
 	return info.IsDir() &&
-		info.Name() == patternDetector &&
+		helpers.MatchesPattern(patternDetector, info.Name()) &&
 		!strings.Contains(path, ".terraform")
 }

pkg/atlantis/single-workspace-discovery.go

@@ -1,13 +1,14 @@
 package atlantis
 
 import (
+	"github.com/totmicro/atlantis-yaml-generator/pkg/helpers"
 	"os"
 	"strings"
 )
 
 func singleWorkspaceDiscoveryFilter(info os.FileInfo, path, patternDetector string) bool {
 	return !info.IsDir() &&
-		info.Name() == patternDetector &&
+		helpers.MatchesPattern(patternDetector, info.Name()) &&
 		!strings.Contains(path, ".terraform")
 }
 

pkg/config/config.go

@@ -103,7 +103,7 @@ var ParameterList = []Parameter{
 	},
 	{
 		Name:         "pattern-detector",
-		Description:  "discover projects based on files or directories names.",
+		Description:  "discover projects based on files, directories names or regex.",
 		Required:     false,
 		DefaultValue: "main.tf",
 		Shorthand:    "q",

pkg/helpers/helpers.go

@@ -3,6 +3,7 @@ package helpers
 import (
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 )
 
@@ -51,3 +52,68 @@ func ReadFile(filename string) (string, error) {
 	_, err = file.Read(content)
 	return string(content), err
 }
+
+// MatchesPattern checks if the given string matches the specified regex pattern.
+// It returns true if the pattern matches the string, and false otherwise.
+func MatchesPattern(pattern string, str string) bool {
+	matched, err := regexp.MatchString(pattern, str)
+	if err != nil {
+		return false
+	}
+	return matched
+}
+
+// Set data structure
+// Keys are strings and elements must implement Hashable to calculate keys.
+type Set struct {
+	Elements map[string]Hashable
+}
+
+// NewSet creates a new Set
+func NewSet() *Set {
+	return &Set{
+		Elements: make(map[string]Hashable),
+	}
+}
+
+// Add adds an element to the set
+func (s *Set) Add(element Hashable) {
+	key := element.Hash()
+	s.Elements[key] = element
+}
+
+// Remove removes an element from the set
+func (s *Set) Remove(element Hashable) {
+	key := element.Hash()
+	delete(s.Elements, key)
+}
+
+// Contains checks if an element is in the set
+func (s *Set) Contains(element Hashable) bool {
+	key := element.Hash()
+	_, exists := s.Elements[key]
+	return exists
+}
+
+// Size returns the number of Elements in the set
+func (s *Set) Size() int {
+	return len(s.Elements)
+}
+
+// List returns all the Elements in the set
+func (s *Set) List() []Hashable {
+	list := make([]Hashable, 0, len(s.Elements))
+	for _, element := range s.Elements {
+		list = append(list, element)
+	}
+	return list
+}
+
+// Enables use of Set by requiring its elements to be hashable.
+type Hashable interface {
+	Hash() string
+}
+
+type Walkable interface {
+	Walk(root string, walkFn filepath.WalkFunc) error
+}

pkg/helpers/helpers_test.go

@@ -3,6 +3,7 @@ package helpers
 import (
 	"fmt"
 	"os"
+	"reflect"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -175,3 +176,79 @@ func TestReadFile_ReadError(t *testing.T) {
 	_, err = ReadFile(tempFile.Name())
 	assert.Error(t, err) // Check that an error is returned
 }
+
+func TestMatchesPattern(t *testing.T) {
+	tests := []struct {
+		pattern string
+		str     string
+		want    bool
+	}{
+		{"workspace_vars", "workspace_vars", true},
+		{"main.tf", "main.tf", true},
+		{".*\\.tf", "main.tf", true},
+		{".*\\.tf", "vpc.tf", true},
+		{"^a.*z$", "alphabet", false}, // Negative test case
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.pattern, func(t *testing.T) {
+			if got := MatchesPattern(tt.pattern, tt.str); got != tt.want {
+				t.Errorf("MatchesPattern(%q, %q) = %v, want %v", tt.pattern, tt.str, got, tt.want)
+			}
+		})
+	}
+}
+
+// HashableString is a simple Hashable type for testing.
+type HashableString string
+
+func (h HashableString) Hash() string {
+	return string(h)
+}
+
+func TestNewSet(t *testing.T) {
+	s := NewSet()
+	if s == nil || len(s.Elements) != 0 {
+		t.Errorf("NewSet() = %v, want a new Set instance with empty Elements", s)
+	}
+}
+
+func TestAddAndContains(t *testing.T) {
+	s := NewSet()
+	element := HashableString("test")
+	s.Add(element)
+	if !s.Contains(element) {
+		t.Errorf("Set does not contain element %v after Add", element)
+	}
+}
+
+func TestRemove(t *testing.T) {
+	s := NewSet()
+	element := HashableString("test")
+	s.Add(element)
+	s.Remove(element)
+	if s.Contains(element) {
+		t.Errorf("Set contains element %v after Remove", element)
+	}
+}
+
+func TestSize(t *testing.T) {
+	s := NewSet()
+	s.Add(HashableString("one"))
+	s.Add(HashableString("two"))
+	if s.Size() != 2 {
+		t.Errorf("Size() = %d, want 2", s.Size())
+	}
+}
+
+func TestList(t *testing.T) {
+	s := NewSet()
+	elements := []Hashable{HashableString("one"), HashableString("two")}
+	for _, e := range elements {
+		s.Add(e)
+	}
+	list := s.List()
+	if !reflect.DeepEqual(list, elements) && len(list) == len(elements) {
+		t.Errorf("List() = %v, want %v", list, elements)
+	}
+}