Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify Repo File Parsing #167

Merged
merged 6 commits into from
Jul 19, 2024
Merged

Simplify Repo File Parsing #167

Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c395178
WIP use a single walk to parse
SUSTAPLE117 Jul 15, 2024
73714dc
removed the dir read for github action workflows and just using the w…
SUSTAPLE117 Jul 18, 2024
cccb297
updated regexes and simplified checks
SUSTAPLE117 Jul 19, 2024
bbf548d
always pattern match to the relative path to where we are scanning to…
SUSTAPLE117 Jul 19, 2024
52b59c8
skip dirs
SUSTAPLE117 Jul 19, 2024
38cd0eb
making sure we skip .git folder
SUSTAPLE117 Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 13 additions & 29 deletions scanner/scanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,6 @@ type parseFunc func(*Scanner, string, fs.FileInfo) error
func parseGithubActionsMetadata(scanner *Scanner, filePath string, fileInfo fs.FileInfo) error {
metadata := make([]models.GithubActionsMetadata, 0)

if fileInfo.IsDir() && fileInfo.Name() == ".git" {
return filepath.SkipDir
}

if fileInfo.IsDir() || (fileInfo.Name() != "action.yml" && fileInfo.Name() != "action.yaml") {
return nil
}

relPath, err := filepath.Rel(scanner.Path, filePath)
if err != nil {
return err
Expand Down Expand Up @@ -60,18 +52,6 @@ func parseGithubActionsMetadata(scanner *Scanner, filePath string, fileInfo fs.F
}

func parseGithubWorkflows(scanner *Scanner, filePath string, fileInfo fs.FileInfo) error {
if !strings.HasPrefix(filePath, filepath.Join(scanner.Path, ".github/workflows")) {
return nil
}

if fileInfo.IsDir() {
return nil
}

if !strings.HasSuffix(fileInfo.Name(), ".yml") && !strings.HasSuffix(fileInfo.Name(), ".yaml") {
return nil
}

relPath, err := filepath.Rel(scanner.Path, filePath)
if err != nil {
return err
Expand Down Expand Up @@ -99,10 +79,6 @@ func parseGithubWorkflows(scanner *Scanner, filePath string, fileInfo fs.FileInf
}

func parseAzurePipelines(scanner *Scanner, filePath string, fileInfo fs.FileInfo) error {
if !strings.HasSuffix(fileInfo.Name(), ".yaml") && !strings.HasSuffix(fileInfo.Name(), ".yml") {
return nil
}

relPath, err := filepath.Rel(scanner.Path, filePath)
if err != nil {
return err
Expand Down Expand Up @@ -191,10 +167,10 @@ func NewScanner(path string) Scanner {
Package: &models.PackageInsights{},
ResolvedPurls: map[string]bool{},
ParseFuncs: map[*regexp.Regexp]parseFunc{
regexp.MustCompile(`action\.ya?ml$`): parseGithubActionsMetadata,
regexp.MustCompile(`.github/workflows`): parseGithubWorkflows,
regexp.MustCompile(`\.?azure-pipelines(-.+)?\.ya?ml$`): parseAzurePipelines,
regexp.MustCompile(`\.?gitlab-ci(-.+)?\.y?ml$`): parseGitlabCi,
regexp.MustCompile(`(\b|/)action\.ya?ml$`): parseGithubActionsMetadata,
regexp.MustCompile(`^\.github/workflows/[^/]+\.ya?ml$`): parseGithubWorkflows,
regexp.MustCompile(`\.?azure-pipelines(-.+)?\.ya?ml$`): parseAzurePipelines,
regexp.MustCompile(`\.?gitlab-ci(-.+)?\.ya?ml$`): parseGitlabCi,
},
}
}
Expand All @@ -213,8 +189,16 @@ func (s *Scanner) walkAndParse() error {
if err != nil {
return err
}
if info.IsDir() && info.Name() == ".git" {
return filepath.SkipDir
}
SUSTAPLE117 marked this conversation as resolved.
Show resolved Hide resolved
relativePath, err := filepath.Rel(s.Path, filePath)
if err != nil {
log.Error().Err(err).Msg("error getting relative path")
return err
}
for pattern, parseFunc := range s.ParseFuncs {
if pattern.MatchString(filePath) {
if pattern.MatchString(relativePath) {
if err := parseFunc(s, filePath, info); err != nil {
log.Error().Err(err).Msg("error parsing file")
// Decide whether to return error or continue processing other files
Expand Down