Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify Repo File Parsing #167

Merged
merged 6 commits into from
Jul 19, 2024
Merged

Simplify Repo File Parsing #167

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c395178
WIP use a single walk to parse
SUSTAPLE117 Jul 15, 2024
73714dc
removed the dir read for github action workflows and just using the w…
SUSTAPLE117 Jul 18, 2024
cccb297
updated regexes and simplified checks
SUSTAPLE117 Jul 19, 2024
bbf548d
always pattern match to the relative path to where we are scanning to…
SUSTAPLE117 Jul 19, 2024
52b59c8
skip dirs
SUSTAPLE117 Jul 19, 2024
38cd0eb
making sure we skip .git folder
SUSTAPLE117 Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
291 changes: 120 additions & 171 deletions scanner/scanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,10 @@ package scanner

import (
"context"
"errors"
"github.com/boostsecurityio/poutine/models"
"github.com/rs/zerolog/log"
"io/fs"
"os"
"path"
"path/filepath"
"strings"

Expand All @@ -18,177 +16,104 @@ import (

const MAX_DEPTH = 150

type Scanner struct {
Path string
Package *models.PackageInsights
ResolvedPurls map[string]bool
}
type parseFunc func(*Scanner, string, fs.FileInfo) error

func NewScanner(path string) Scanner {
return Scanner{
Path: path,
Package: &models.PackageInsights{},
ResolvedPurls: map[string]bool{},
}
}
func parseGithubActionsMetadata(scanner *Scanner, filePath string, fileInfo fs.FileInfo) error {
metadata := make([]models.GithubActionsMetadata, 0)

func (s *Scanner) Run(ctx context.Context, o *opa.Opa) error {
err := s.parse()
relPath, err := filepath.Rel(scanner.Path, filePath)
if err != nil {
return err
}

return s.inventory(ctx, o)
}

func (s *Scanner) inventory(ctx context.Context, o *opa.Opa) error {
result := opa.InventoryResult{}
err := o.Eval(ctx,
"data.poutine.queries.inventory.result",
map[string]interface{}{
"packages": []interface{}{s.Package},
},
&result,
)
data, err := os.ReadFile(filePath)
if err != nil {
return err
}

s.Package.BuildDependencies = result.BuildDependencies
s.Package.PackageDependencies = result.PackageDependencies
meta := models.GithubActionsMetadata{
Path: relPath,
}
err = yaml.Unmarshal(data, &meta)
if err != nil {
log.Debug().Err(err).Str("file", relPath).Msg("failed to unmarshal yaml file")
return nil
}

if meta.IsValid() {
metadata = append(metadata, meta)
} else {
log.Debug().Str("file", relPath).Msg("failed to parse github actions metadata")
}

scanner.Package.GithubActionsMetadata = append(scanner.Package.GithubActionsMetadata, metadata...)

return nil
}

func (s *Scanner) parse() error {
var err error
s.Package.GithubActionsMetadata, err = s.GithubActionsMetadata()
func parseGithubWorkflows(scanner *Scanner, filePath string, fileInfo fs.FileInfo) error {
relPath, err := filepath.Rel(scanner.Path, filePath)
if err != nil {
return err
}

s.Package.GithubActionsWorkflows, err = s.GithubWorkflows()
data, err := os.ReadFile(filePath)
if err != nil {
return err
}

s.Package.GitlabciConfigs, err = s.GitlabciConfigs()
workflow := models.GithubActionsWorkflow{Path: relPath}
err = yaml.Unmarshal(data, &workflow)
if err != nil {
return err
log.Debug().Err(err).Str("file", relPath).Msg("failed to unmarshal yaml file")
return nil
}

s.Package.AzurePipelines, err = s.AzurePipelines()
if err != nil {
return err
if workflow.IsValid() {
scanner.Package.GithubActionsWorkflows = append(scanner.Package.GithubActionsWorkflows, workflow)
} else {
log.Debug().Str("file", relPath).Msg("failed to parse github actions workflow")
}

return nil
}

func (s *Scanner) GithubActionsMetadata() ([]models.GithubActionsMetadata, error) {
metadata := make([]models.GithubActionsMetadata, 0)

err := filepath.Walk(s.Path,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

if info.IsDir() && info.Name() == ".git" {
return filepath.SkipDir
}

if info.IsDir() || (info.Name() != "action.yml" && info.Name() != "action.yaml") {
return nil
}

rel_path, err := filepath.Rel(s.Path, path)
if err != nil {
return err
}

data, err := os.ReadFile(path)
if err != nil {
return err
}

meta := models.GithubActionsMetadata{
Path: rel_path,
}
err = yaml.Unmarshal(data, &meta)
if err != nil {
log.Debug().Err(err).Str("file", rel_path).Msg("failed to unmarshal yaml file")
return nil
}

if meta.IsValid() {
metadata = append(metadata, meta)
} else {
log.Debug().Str("file", rel_path).Msg("failed to parse github actions metadata")
}

return nil
},
)

return metadata, err
}

func (s *Scanner) GithubWorkflows() ([]models.GithubActionsWorkflow, error) {
folder := filepath.Join(s.Path, ".github/workflows")
files, err := os.ReadDir(folder)
func parseAzurePipelines(scanner *Scanner, filePath string, fileInfo fs.FileInfo) error {
relPath, err := filepath.Rel(scanner.Path, filePath)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return []models.GithubActionsWorkflow{}, nil
}
return nil, err
return err
}

workflows := make([]models.GithubActionsWorkflow, 0, len(files))
for _, file := range files {
if file.IsDir() {
continue
}

path := path.Join(folder, file.Name())
if !strings.HasSuffix(path, ".yml") && !strings.HasSuffix(path, ".yaml") {
continue
}
rel_path, err := filepath.Rel(s.Path, path)
if err != nil {
return nil, err
}

data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
data, err := os.ReadFile(filePath)
if err != nil {
return err
}

workflow := models.GithubActionsWorkflow{Path: rel_path}
err = yaml.Unmarshal(data, &workflow)
if err != nil {
log.Debug().Err(err).Str("file", rel_path).Msg("failed to unmarshal yaml file")
continue
}
pipeline := models.AzurePipeline{}
err = yaml.Unmarshal(data, &pipeline)
if err != nil {
log.Debug().Err(err).Str("file", relPath).Msg("failed to unmarshal yaml file")
return nil
}

if workflow.IsValid() {
workflows = append(workflows, workflow)
} else {
log.Debug().Str("file", rel_path).Msg("failed to parse github actions workflow")
}
if pipeline.IsValid() {
pipeline.Path = relPath
scanner.Package.AzurePipelines = append(scanner.Package.AzurePipelines, pipeline)
} else {
log.Debug().Str("file", relPath).Msg("failed to parse azure pipeline")
}

return workflows, err
return nil
}

func (s *Scanner) GitlabciConfigs() ([]models.GitlabciConfig, error) {
func parseGitlabCi(scanner *Scanner, filePath string, fileInfo fs.FileInfo) error {
files := map[string]bool{}
queue := []string{"/.gitlab-ci.yml"}
configs := []models.GitlabciConfig{}

for len(queue) > 0 && len(configs) < MAX_DEPTH {
repoPath := filepath.Join("/", queue[0])
configPath := filepath.Join(s.Path, repoPath)
configPath := filepath.Join(scanner.Path, repoPath)
queue = queue[1:]

if files[repoPath] {
Expand Down Expand Up @@ -224,60 +149,84 @@ func (s *Scanner) GitlabciConfigs() ([]models.GitlabciConfig, error) {
configs = append(configs, *config)
}

return configs, nil
}

var azurePipelineFileRegex = regexp.MustCompile(`\.?azure-pipelines(-.+)?\.ya?ml$`)

func (s *Scanner) AzurePipelines() ([]models.AzurePipeline, error) {
pipelines := []models.AzurePipeline{}
err := filepath.Walk(s.Path,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
scanner.Package.GitlabciConfigs = append(scanner.Package.GitlabciConfigs, configs...)

if info.IsDir() && info.Name() == ".git" {
return filepath.SkipDir
}
return nil
}

if info.IsDir() {
return nil
}
type Scanner struct {
Path string
Package *models.PackageInsights
ResolvedPurls map[string]bool
ParseFuncs map[*regexp.Regexp]parseFunc
}

if !azurePipelineFileRegex.MatchString(info.Name()) {
return nil
}
func NewScanner(path string) Scanner {
return Scanner{
Path: path,
Package: &models.PackageInsights{},
ResolvedPurls: map[string]bool{},
ParseFuncs: map[*regexp.Regexp]parseFunc{
regexp.MustCompile(`(\b|/)action\.ya?ml$`): parseGithubActionsMetadata,
regexp.MustCompile(`^\.github/workflows/[^/]+\.ya?ml$`): parseGithubWorkflows,
regexp.MustCompile(`\.?azure-pipelines(-.+)?\.ya?ml$`): parseAzurePipelines,
regexp.MustCompile(`\.?gitlab-ci(-.+)?\.ya?ml$`): parseGitlabCi,
},
}
}

rel_path, err := filepath.Rel(s.Path, path)
if err != nil {
return err
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
func (s *Scanner) Run(ctx context.Context, o *opa.Opa) error {
err := s.walkAndParse()
if err != nil {
return err
}

pipeline := models.AzurePipeline{}
err = yaml.Unmarshal(data, &pipeline)
if err != nil {
return err
}
return s.inventory(ctx, o)
}

if pipeline.IsValid() {
pipeline.Path = rel_path
pipelines = append(pipelines, pipeline)
} else {
log.Debug().Str("file", rel_path).Msg("failed to parse azure pipeline")
func (s *Scanner) walkAndParse() error {
return filepath.Walk(s.Path, func(filePath string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
SUSTAPLE117 marked this conversation as resolved.
Show resolved Hide resolved
if info.IsDir() && info.Name() == ".git" {
return filepath.SkipDir
}
SUSTAPLE117 marked this conversation as resolved.
Show resolved Hide resolved
if info.IsDir() {
return nil
}
relativePath, err := filepath.Rel(s.Path, filePath)
if err != nil {
log.Error().Err(err).Msg("error getting relative path")
return err
}
for pattern, parseFunc := range s.ParseFuncs {
if pattern.MatchString(relativePath) {
if err := parseFunc(s, filePath, info); err != nil {
log.Error().Err(err).Msg("error parsing file")
// Decide whether to return error or continue processing other files
}
}
}
return nil
})
}

return nil
func (s *Scanner) inventory(ctx context.Context, o *opa.Opa) error {
result := opa.InventoryResult{}
err := o.Eval(ctx,
"data.poutine.queries.inventory.result",
map[string]interface{}{
"packages": []interface{}{s.Package},
},
&result,
)

if err != nil {
return nil, err
return err
}

return pipelines, nil
s.Package.BuildDependencies = result.BuildDependencies
s.Package.PackageDependencies = result.PackageDependencies

return nil
}
Loading