SSH Url support (#9)

* adds ssh support, refactors regex matchers, tests
Darth-Tech · Aug 15, 2024 · c82780d · c82780d
1 parent c9cb496
commit c82780d
Show file tree

Hide file tree

Showing 14 changed files with 322 additions and 125 deletions.
diff --git a/.github/workflows/go-test-workflow.yml b/.github/workflows/go-test-workflow.yml
@@ -1,10 +1,10 @@
 name: Go Test
 on: [push]
-
+env:
+  SAMWISE_CLI_GIT_SSH_KEY_PATH: ${{secrets.SAMWISE_CLI_GIT_SSH_KEY_PATH}}
 jobs:
     test:
         runs-on: ubuntu-latest
-
         steps:
             - uses: actions/checkout@v4
             - name: Setup Go
@@ -15,5 +15,11 @@ jobs:
               run: go get .
             - name: Build
               run: go build -v ./...
+            - name: setup ssh key for testing
+              run: |
+                      install --directory ~/.ssh --mode 700
+                      echo '${{ secrets.SSH_PRIVATE }}' > ~/.ssh/id_rsa
+                      chmod 600 ~/.ssh/*
             - name: Test with the Go CLI
               run: go test -json ./cmd/...
+
diff --git a/.gitignore b/.gitignore
@@ -118,3 +118,8 @@ fabric.properties
 coverage
 
 dist/
+cmd/failure_report.json
+cmd/module_dependency_report.csv
+cmd/module_dependency_report.json
+cmd/module_dependency.json
+cmd/module_report.csv
diff --git a/.samwise.yaml.example b/.samwise.yaml.example
@@ -1,2 +1,3 @@
-github_key:
-github_username:
+git_user:
+git_key:
+git_ssh_key_path:
diff --git a/cmd/checkForUpdates.go b/cmd/checkForUpdates.go
@@ -4,16 +4,17 @@ Copyright © 2024 Agastya Dev Addepally ([email protected])
 package cmd
 
 import (
-	"github.com/schollz/progressbar/v3"
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 	"io/fs"
 	"log/slog"
 	"os"
 	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
+
+	"github.com/schollz/progressbar/v3"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
 // checkForUpdatesCmd represents the checkForUpdates command
@@ -123,8 +124,8 @@ func init() {
 	checkForUpdatesCmd.Flags().StringArrayP("ignore", "i", []string{".git", ".idea"}, "Directories to ignore when searching for the One Ring(modules and their sources.")
 	checkForUpdatesCmd.Flags().StringP("output", "o", "csv", "Output format. Supports \"csv\" and \"json\". Default value is csv.")
 	checkForUpdatesCmd.Flags().StringP("output-filename", "f", "module_report", "Output file name.")
-	checkForUpdatesCmd.Flags().Bool("ci", false, "Set this flag for usage in CI systems. Does not generate a report. Prints JSON to Stdout and returns exit code 1 if modules are outdated.")
-	checkForUpdatesCmd.Flags().Bool("allow-failure", true, "Set this flag for usage in CI systems. If true, does NOT exit code 1 when modules are outdated.")
+	//checkForUpdatesCmd.Flags().Bool("ci", false, "Set this flag for usage in CI systems. Does not generate a report. Prints JSON to Stdout and returns exit code 1 if modules are outdated.")
+	//checkForUpdatesCmd.Flags().Bool("allow-failure", true, "Set this flag for usage in CI systems. If true, does NOT exit code 1 when modules are outdated.")
 
 	err := checkForUpdatesCmd.MarkFlagRequired("path")
 	if err != nil {

diff --git a/cmd/errorHandlers/errorConstants.go b/cmd/errorHandlers/errorConstants.go
@@ -1,4 +1,4 @@
 package errorHandlers
 
 const CheckOutputFormatError = "output format not supported. Please use csv or json"
-const CloningErrorPrefix = "unable to clone repo"
+const CloningErrorPrefix = "unable to clone repo "
diff --git a/cmd/readFiles.go b/cmd/readFiles.go
@@ -2,25 +2,18 @@ package cmd
 
 import (
 	"bufio"
-	"golang.org/x/exp/maps"
 	"log/slog"
-	"net/url"
 	"os"
 	"regexp"
 	"strings"
 )
 
 var (
 	// Regex to check if the line has "source=" in it
-	sourceLineRegex      = regexp.MustCompile("source=\"(.+)\"")
-	moduleSourceRegexMap = map[string]*regexp.Regexp{
-		"generic_git": regexp.MustCompile("source=\"git::(.+)\""),
-		"github":      regexp.MustCompile("source=\"(github.com.+)\""),
-		"https":       regexp.MustCompile("source=\"(https://.+)\""),
-		"bitbucket":   regexp.MustCompile("source=\".*(bitbucket.org.+)\""),
-	}
-	submoduleRegex  = regexp.MustCompile("(.*/.*)//(.*)")
-	removeUrlParams = regexp.MustCompile("(\\?.*)")
+	sourceLineRegex = regexp.MustCompile(`source="(.+\..+)"`)
+	submoduleRegex  = regexp.MustCompile(`(?P<base_url>.*/.*)//(?P<submodule>.*)`)
+	removeUrlParams = regexp.MustCompile(`(\?.*)`)
+	refRegex        = regexp.MustCompile(".*?ref=(.*)&.*|.*?ref=(.*)")
 	moduleRepoList  []map[string]string
 )
 
@@ -31,76 +24,68 @@ func fixTrailingSlashForPath(path string) string {
 	return path
 }
 
+func getNamedMatchesForRegex(reg *regexp.Regexp, sourceString string) map[string]string {
+	match := reg.FindStringSubmatch(sourceString)
+	result := make(map[string]string)
+
+	if len(match) > 0 {
+		for i, name := range reg.SubexpNames() {
+			if i != 0 && name != "" {
+				result[name] = match[i]
+			}
+		}
+	}
+
+	return result
+}
+
 // Returns base url and cleaned paths
 func extractSubmoduleFromSource(source string) (string, string) {
-	subModuleMatch := submoduleRegex.FindAllStringSubmatch(source, -1)
+	subModuleMatch := getNamedMatchesForRegex(submoduleRegex, source)
 	var baseUrl = ""
 	var path = ""
-	if len(subModuleMatch) > 0 && len(subModuleMatch[0][1]) > 0 {
-		baseUrl = subModuleMatch[0][1]
+	// If there is a match and one group is matched(1st), then it
+	if len(subModuleMatch) > 0 && len(subModuleMatch["base_url"]) > 0 {
+		baseUrl = subModuleMatch["base_url"]
 	} else {
 		baseUrl = source
 	}
-	if len(subModuleMatch) > 0 && len(subModuleMatch[0][2]) > 0 {
-		path = subModuleMatch[0][2]
+	if len(subModuleMatch) > 0 && len(subModuleMatch["submodule"]) > 0 {
+		path = subModuleMatch["submodule"]
 	}
 	return baseUrl, path
 }
-func checkRegexMatchNotEmpty(match [][]string) string {
-	if len(match) > 0 && len(match[0][1]) > 0 {
-		return match[0][1]
-	}
-	return ""
-}
 
-// TODO Add tests
 func getTagFromUrl(source string) string {
 	var refTag string
-	refParams, err := url.Parse(source)
-	// TODO: Refactor
-	if CheckNonPanic(err, "readFiles :: extractRefAndPath :: unable to parse url for params") {
-		refTag = ""
-	} else {
-		params, err := url.ParseQuery(refParams.RawQuery)
-		if CheckNonPanic(err, "readFiles :: extractRefAndPath :: unable to parse url for params") {
-			refTag = ""
-		}
-		refTag = params.Get("ref")
+	refTagMatches := refRegex.FindStringSubmatch(source)
+	if len(refTagMatches) > 0 {
+		refTag = refTagMatches[2]
+		return refTag
 	}
+
 	return refTag
 }
 
 // Returns url, tag submodules(if any) in that order
 func extractRefAndPath(sourceUrl string) (string, string, string) {
-	var refTag, finalUrl, tempUrl, submodulePaths, submodulePathsParams string
-	if strings.Count(sourceUrl, "//") == 2 {
-		sourceUrl, submodulePathsParams = extractSubmoduleFromSource(sourceUrl)
-		refTag = getTagFromUrl(sourceUrl + "/" + submodulePathsParams)
-		submodulePaths = removeUrlParams.ReplaceAllString(submodulePathsParams, "")
-	} else {
-		refTag = getTagFromUrl(sourceUrl)
-	}
-	tempUrl = sourceUrl
-	urlCleaner, _ := url.Parse(tempUrl)
+	var refTag, submodulePaths string
 
-	if urlCleaner.Scheme != "" {
-		finalUrl = urlCleaner.Scheme + "://"
-	}
-	finalUrl = finalUrl + urlCleaner.Host + urlCleaner.Path
+	baseUrl, submodulePathsParams := extractSubmoduleFromSource(sourceUrl)
+	submodulePaths = removeUrlParams.ReplaceAllString(submodulePathsParams, "")
+	baseUrl = removeUrlParams.ReplaceAllString(baseUrl, "")
+	refTag = getTagFromUrl(sourceUrl)
 
-	return finalUrl, refTag, submodulePaths
+	return baseUrl, refTag, submodulePaths
 }
 
 func extractModuleSource(line string) string {
-	keys := maps.Keys(moduleSourceRegexMap)
-	var match [][]string
 	var matchedString = ""
-	for _, sourceRegex := range keys {
-		match = moduleSourceRegexMap[sourceRegex].FindAllStringSubmatch(line, 1)
-		matchedString = checkRegexMatchNotEmpty(match)
-		if matchedString != "" {
-			break
-		}
+	match := sourceLineRegex.FindStringSubmatch(line)
+	if len(match) > 0 {
+		matchedString = match[1]
+		matchedString = strings.ReplaceAll(matchedString, "git::", "")
+
 	}
 	return matchedString
 }
@@ -109,14 +94,16 @@ func extractModuleSource(line string) string {
 func preProcessingSourceString(line string) (string, string, string) {
 	// Will help avoid running moduleSourceRegexMap on every string
 	line = strings.ReplaceAll(line, " ", "")
-	sourceLineCheck := sourceLineRegex.FindAllStringSubmatch(line, -1)
-	if checkRegexMatchNotEmpty(sourceLineCheck) == "" {
+	sourceLineCheck := sourceLineRegex.FindStringSubmatch(line)
+	if len(sourceLineCheck) == 0 {
 		return "", "", ""
 	} else {
 		repoLink := extractModuleSource(line)
+		//repoLink := sourceLineCheck[1]
 		slog.Debug("Git repo link before: " + repoLink)
 		var sourceUrl, refTag, submodule string
 		if repoLink != "" {
+
 			sourceUrl, refTag, submodule = extractRefAndPath(repoLink)
 		}
 		slog.Debug("Git repo link after: " + sourceUrl)

diff --git a/cmd/readFiles_test.go b/cmd/readFiles_test.go
@@ -1,9 +1,11 @@
 package cmd
 
 import (
-	"github.com/stretchr/testify/assert"
 	"log/slog"
+	"regexp"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestReadDirectoryWithSlash(t *testing.T) {
@@ -14,19 +16,47 @@ func TestReadDirectoryWithoutSlash(t *testing.T) {
 	assert.Equal(t, "./test_dir", fixTrailingSlashForPath("./test_dir"), "no slash at the end")
 }
 
+func TestGetNamedMatchesForRegex(t *testing.T) {
+	submoduleRegex = regexp.MustCompile("(?P<base_url>.*/.*)//(?P<submodule>.*)")
+
+	//nonSubmoduleMatch := getNamedMatchesForRegex(submoduleRegex, "github.com/hashicorp/example?ref=1.0.0")
+	submoduleMatch := getNamedMatchesForRegex(submoduleRegex, "github.com/hashicorp/example//test_1/test_2?ref=1.0.0")
+	doubleSlashSubmoduleMatch := getNamedMatchesForRegex(submoduleRegex, "https://github.com/hashicorp/example//test_1/test_2?ref=1.0.0")
+
+	//	assert.NotEmpty(t, nonSubmoduleMatch, "readFiles_test :: getNamedMatchesForRegex :: github.com/hashicorp/example?ref=1.0.0 was not matched for baseurl")
+	//	assert.Equal(t, "github.com/hashicorp/example?ref=1.0.0", nonSubmoduleMatch["base_url"], "readFiles_test :: getNamedMatchesForRegex :: unable to extract base url")
+	//	assert.Empty(t, nonSubmoduleMatch["submodule"], "readFiles_test :: getNamedMatchesForRegex :: incorrectly extracts submodule")
+
+	assert.NotEmpty(t, submoduleMatch, "readFiles_test :: getNamedMatchesForRegex :: github.com/hashicorp/example//test_1/test_2?ref=1.0.0 was not matched for baseurl")
+	assert.Equal(t, "github.com/hashicorp/example", submoduleMatch["base_url"], "readFiles_test :: getNamedMatchesForRegex :: unable to extract base url")
+	assert.Equal(t, "test_1/test_2?ref=1.0.0", submoduleMatch["submodule"], "readFiles_test :: getNamedMatchesForRegex :: unable to extract submodule")
+
+	assert.NotEmpty(t, doubleSlashSubmoduleMatch, "readFiles_test :: getNamedMatchesForRegex :: https://github.com/hashicorp/example//test_1/test_2?ref=1.0.0 was not matched for baseurl")
+	assert.Equal(t, "https://github.com/hashicorp/example", doubleSlashSubmoduleMatch["base_url"], "readFiles_test :: getNamedMatchesForRegex :: unable to extract base url")
+	assert.Equal(t, "test_1/test_2?ref=1.0.0", doubleSlashSubmoduleMatch["submodule"], "readFiles_test :: getNamedMatchesForRegex :: unable to extract submodule")
+
+}
+
 func TestGetTagFromUrl(t *testing.T) {
-	assert.Equal(t, "1.0.0", getTagFromUrl("github.com/hashicorp/example?ref=1.0.0"), "ref param not extracted")
-	assert.Empty(t, getTagFromUrl("github.com/hashicorp/example.git"), "ref param found")
+	assert.Equal(t, "1.0.0", getTagFromUrl("github.com/hashicorp/example?ref=1.0.0"), "ref param not extracted for github.com/hashicorp/example?ref=1.0.0")
+	assert.Equal(t, "1.0.0", getTagFromUrl("[email protected]:hashicorp/example?ref=1.0.0"), "ref param not extracted for [email protected]:hashicorp/example?ref=1.0.0")
+	assert.Empty(t, getTagFromUrl("github.com/hashicorp/example.git"), "ref param found for github.com/hashicorp/example.git")
 	assert.Empty(t, getTagFromUrl("github.com/hashicorp/example?depth=1"), "ref param found")
 	assert.Empty(t, getTagFromUrl("https://github.com/hashicorp/example?depth=1"), "ref param found")
-	assert.Equal(t, "2.0.0", getTagFromUrl("https://github.com/hashicorp/example?ref=2.0.0"), "ref param not found")
+	assert.Equal(t, "1.0.1", getTagFromUrl("https://github.com/hashicorp/example?depth=&ref=1.0.1"), "ref param found")
+
+	assert.Equal(t, "2.0.0", getTagFromUrl("https://github.com/hashicorp/example?ref=2.0.0"), "ref param not found for https://github.com/hashicorp/example?ref=2.0.0")
 }
 func TestExtractRefAndPathRepo(t *testing.T) {
 	gitHubExampleRepo, _, _ := extractRefAndPath("github.com/hashicorp/example?ref=1.0.0")
 	assert.Equal(t, "github.com/hashicorp/example", gitHubExampleRepo, "readFiles :: extractRefAndPath :: repo :: "+gitHubExampleRepo)
 
-	// gitHubSSHExampleRepo, gitHubSSHExampleTag := extractRefAndPath("[email protected]:hashicorp/example.git")
-	//assert.Equal(t, "[email protected]:hashicorp/example.git", gitHubSSHExampleRepo, "readFiles :: extractRefAndPath :: repo :: "+gitHubSSHExampleRepo+" :: "+gitHubSSHExampleTag)
+	gitHubSSHExampleRepo, _, _ := extractRefAndPath("[email protected]:hashicorp/example.git")
+	assert.Equal(t, "[email protected]:hashicorp/example.git", gitHubSSHExampleRepo, "readFiles :: extractRefAndPath :: repo :: "+gitHubSSHExampleRepo)
+
+	gitHubSSHExampleRepoWithRef, _, _ := extractRefAndPath("[email protected]:hashicorp/example.git?ref=test")
+	assert.Equal(t, "[email protected]:hashicorp/example.git", gitHubSSHExampleRepoWithRef, "readFiles :: extractRefAndPath :: repo :: "+gitHubSSHExampleRepo)
+
 	bitbucketExampleRepo, _, _ := extractRefAndPath("bitbucket.org/hashicorp/terraform-consul-aws?ref=1.0.0&test=woho")
 	assert.Equal(t, "bitbucket.org/hashicorp/terraform-consul-aws", bitbucketExampleRepo, "readFiles :: extractRefAndPath :: repo :: "+bitbucketExampleRepo)
 
@@ -42,6 +72,8 @@ func TestExtractRefAndPathTag(t *testing.T) {
 	_, gitHubExampleTag, _ := extractRefAndPath("github.com/hashicorp/example?ref=1.0.0")
 	assert.Equal(t, "1.0.0", gitHubExampleTag, "readFiles :: extractRefAndPath :: tag :: "+gitHubExampleTag)
 
+	//_, gitHubSSHExampleTag, _ := extractRefAndPath("[email protected]:hashicorp/example.git")
+
 	_, gitHubExampleNoTag, _ := extractRefAndPath("github.com/hashicorp/example")
 	assert.Equal(t, "", gitHubExampleNoTag, "readFiles :: extractRefAndPath :: no tag :: ")
 
@@ -58,6 +90,10 @@ func TestExtractRefAndPathSubmodule(t *testing.T) {
 	assert.Equal(t, "", gitHubExampleSubmodule, "readFiles :: extractRefAndPath :: submodule :: "+gitHubExampleSubmodule)
 	assert.Equal(t, "github.com/hashicorp/example", gitHubExample, "readFiles :: extractRefAndPath :: repo :: "+gitHubExample)
 
+	gitHubSSHExample, _, gitHubSSHExampleSubmodule := extractRefAndPath("[email protected]:hashicorp/example//module/test")
+	assert.Equal(t, "module/test", gitHubSSHExampleSubmodule, "readFiles :: extractRefAndPath :: submodule :: "+gitHubSSHExampleSubmodule)
+	assert.Equal(t, "[email protected]:hashicorp/example", gitHubSSHExample, "readFiles :: extractRefAndPath :: repo :: "+gitHubSSHExample)
+
 	gitHubExampleRepo, _, gitHubExampleRepoSubmodule := extractRefAndPath("https://github.com/org/repo//submodules/folder?ref=1.1.1\n")
 	assert.Equal(t, "submodules/folder", gitHubExampleRepoSubmodule, "readFiles :: extractRefAndPath :: submodule :: "+gitHubExampleRepoSubmodule)
 	assert.Equal(t, "https://github.com/org/repo", gitHubExampleRepo, "readFiles :: extractRefAndPath :: repo :: "+gitHubExampleRepo)
@@ -68,20 +104,27 @@ func TestExtractRefAndPathSubmodule(t *testing.T) {
 
 }
 func TestExtractModuleSource(t *testing.T) {
+	nonGitSource := extractModuleSource("source=\"Terraform-VMWare-Modules/vm/vsphere\"")
+	assert.Empty(t, nonGitSource, "readFiles :: extractRefAndPath :: repo :: non git terraform source extracted incorrectly")
+
 	gitHubExampleSource := extractModuleSource("source=\"github.com/hashicorp/example?ref=1.0.0\"")
 	assert.Equal(t, "github.com/hashicorp/example?ref=1.0.0", gitHubExampleSource, "readFiles :: extractRefAndPath :: repo :: "+gitHubExampleSource)
-	// gitHubSSHExampleSource := extractModuleSource("source=\"[email protected]:hashicorp/example.git")
-	//assert.Equal(t, "[email protected]:hashicorp/example.git", gitHubSSHExampleSource, "readFiles :: extractRefAndPath :: repo :: "+gitHubSSHExampleSource+" :: "+gitHubSSHExampleTag)
+
+	gitHubSSHExampleSource := extractModuleSource("source=\"[email protected]:hashicorp/example.git\"")
+	assert.Equal(t, "[email protected]:hashicorp/example.git", gitHubSSHExampleSource, "readFiles :: extractRefAndPath :: repo :: "+gitHubSSHExampleSource)
 
 	gitGitHubExampleSource := extractModuleSource("source=\"git::https://github.com/test_repo_labala?ref=1.3.1\"")
 	assert.Equal(t, "https://github.com/test_repo_labala?ref=1.3.1", gitGitHubExampleSource, "readFiles :: extractRefAndPath :: repo :: "+gitGitHubExampleSource)
 
 	bitbucketExampleSource := extractModuleSource("source=\"bitbucket.org/hashicorp/terraform-consul-aws?ref=1.0.0&test=woho\"")
 	assert.Equal(t, "bitbucket.org/hashicorp/terraform-consul-aws?ref=1.0.0&test=woho", bitbucketExampleSource, "readFiles :: extractRefAndPath :: repo :: "+bitbucketExampleSource)
+
 	genericGitExampleSource := extractModuleSource("source=\"https://example.com/vpc.git?ref=1.1.0&test=woho\"")
 	assert.Equal(t, "https://example.com/vpc.git?ref=1.1.0&test=woho", genericGitExampleSource, "readFiles :: extractRefAndPath :: repo :: "+genericGitExampleSource)
+
 	gitParamExampleSource := extractModuleSource("source=\"https://example.com/vpc.git?depth=1&ref=v1.2.0\"")
 	assert.Equal(t, "https://example.com/vpc.git?depth=1&ref=v1.2.0", gitParamExampleSource, "readFiles :: extractRefAndPath :: repo :: "+gitParamExampleSource)
+
 	slog.Debug("repos and refs", gitHubExampleSource, bitbucketExampleSource, genericGitExampleSource, gitParamExampleSource)
 }
 
@@ -103,5 +146,3 @@ func TestExtractSubmoduleFromSource(t *testing.T) {
 	assert.Equal(t, "submodule/folder1/folder2", submoduleWithDepths)
 
 }
-
-//gitHubSSHExampleTag, gitHubSSHExampleRepo,