Filtering supported file types for scanning by default (#16)

Co-authored-by: Britton Hayes <[email protected]>

benchmarks_test.go

@@ -23,9 +23,9 @@ func BenchmarkHunterLoadRules(b *testing.B) {
 func BenchmarkHunterHoundHowl(b *testing.B) {
 	b.StopTimer()
 	h := hunter.NewHound(&hunter.Config{
-		System: afero.NewMemMapFs(),
-		Rules:  rules.Load(""),
-		Format: hunter.JSONFormat,
+		System:   afero.NewMemMapFs(),
+		Gitleaks: rules.Load(""),
+		Format:   hunter.JSONFormat,
 	})
 	findings := scan.Report{
 		Leaks: []scan.Leak{

cmd/hunt.go

@@ -4,6 +4,8 @@
 package cmd
 
 import (
+	"runtime"
+
 	"github.com/brittonhayes/pillager/hunter"
 	"github.com/brittonhayes/pillager/rules"
 	"github.com/spf13/afero"
@@ -42,10 +44,10 @@ pillager hunt ./example -r rules.toml -f custom --template "$(cat templates/simp
 
 func init() {
 	rootCmd.AddCommand(huntCmd)
-	huntCmd.Flags().IntVarP(&workers, "workers", "w", 5, "number of concurrent workers to create")
+	huntCmd.Flags().IntVarP(&workers, "workers", "w", runtime.NumCPU(), "number of concurrent workers to create")
 	huntCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "toggle verbose output")
 	huntCmd.Flags().StringVarP(&rulesConfig, "rules", "r", "", "path to gitleaks rules.toml config")
-	huntCmd.Flags().StringVarP(&output, "format", "f", "yaml", "set output format (json, yaml, custom)")
+	huntCmd.Flags().StringVarP(&output, "format", "f", "json", "set output format (json, yaml, custom)")
 	huntCmd.Flags().StringVarP(
 		&templ,
 		"template",

hunter/README.md

@@ -13,15 +13,7 @@ Package hunter contains the types\, methods\, and interfaces for the file huntin
 - [Constants](<#constants>)
 - [func RenderTemplate(w io.Writer, tpl string, f scan.Report)](<#func-rendertemplate>)
 - [type Config](<#type-config>)
-  - [func NewConfig(
-    fs afero.Fs,
-    path string,
-    verbose bool,
-    rules []gitleaks.Rule,
-    format Format,
-    template string,
-    workers int,
-) *Config](<#func-newconfig>)
+  - [func NewConfig(fs afero.Fs, path string, verbose bool, gitleaks gitleaks.Config, format Format, template string, workers int) *Config](<#func-newconfig>)
   - [func (c *Config) Default() *Config](<#func-config-default>)
   - [func (c *Config) Validate() (err error)](<#func-config-validate>)
 - [type Configer](<#type-configer>)
@@ -44,14 +36,15 @@ DefaultTemplate is the base template used to format a Finding into the custom ou
 
 ```go
 const DefaultTemplate = `{{ with . -}}
-{{ range .Leaks -}}Loot: {{.LineNumber}}
-{{ .File }}
+{{ range .Leaks -}}Line: {{.LineNumber}}
+File: {{ .File }}
+Offender: {{ .Offender }}
 
 {{end}}
 {{- end}}`
 ```
 
-## func [RenderTemplate](<https://github.com/brittonhayes/pillager/blob/main/hunter/template.go#L22>)
+## func [RenderTemplate](<https://github.com/brittonhayes/pillager/blob/main/hunter/template.go#L23>)
 
 ```go
 func RenderTemplate(w io.Writer, tpl string, f scan.Report)
@@ -69,37 +62,29 @@ type Config struct {
     BasePath string
     Verbose  bool
     Workers  int
-    Rules    []gitleaks.Rule
+    Gitleaks gitleaks.Config
     Format   Format
     Template string
 }
 ```
 
-### func [NewConfig](<https://github.com/brittonhayes/pillager/blob/main/hunter/config.go#L32-L40>)
+### func [NewConfig](<https://github.com/brittonhayes/pillager/blob/main/hunter/config.go#L32>)
 
 ```go
-func NewConfig(
-    fs afero.Fs,
-    path string,
-    verbose bool,
-    rules []gitleaks.Rule,
-    format Format,
-    template string,
-    workers int,
-) *Config
+func NewConfig(fs afero.Fs, path string, verbose bool, gitleaks gitleaks.Config, format Format, template string, workers int) *Config
 ```
 
 NewConfig generates a new config for the Hunter
 
-### func \(\*Config\) [Default](<https://github.com/brittonhayes/pillager/blob/main/hunter/config.go#L55>)
+### func \(\*Config\) [Default](<https://github.com/brittonhayes/pillager/blob/main/hunter/config.go#L47>)
 
 ```go
 func (c *Config) Default() *Config
 ```
 
 Default loads the default configuration for the Hunter
 
-### func \(\*Config\) [Validate](<https://github.com/brittonhayes/pillager/blob/main/hunter/config.go#L68>)
+### func \(\*Config\) [Validate](<https://github.com/brittonhayes/pillager/blob/main/hunter/config.go#L60>)
 
 ```go
 func (c *Config) Validate() (err error)
@@ -177,9 +162,9 @@ Here is an example of utilizing the Howl function on a slice of findings\. The H
 ```go
 {
 	h := NewHound(&Config{
-		System: afero.NewMemMapFs(),
-		Rules:  rules.Load(""),
-		Format: JSONFormat,
+		System:   afero.NewMemMapFs(),
+		Gitleaks: rules.Load(""),
+		Format:   JSONFormat,
 	})
 	findings := scan.Report{
 		Leaks: []scan.Leak{

hunter/config.go

@@ -18,7 +18,7 @@ type Config struct {
 	BasePath string
 	Verbose  bool
 	Workers  int
-	Rules    []gitleaks.Rule
+	Gitleaks gitleaks.Config
 	Format   Format
 	Template string
 }
@@ -29,21 +29,13 @@ type Configer interface {
 }
 
 // NewConfig generates a new config for the Hunter
-func NewConfig(
-	fs afero.Fs,
-	path string,
-	verbose bool,
-	rules []gitleaks.Rule,
-	format Format,
-	template string,
-	workers int,
-) *Config {
+func NewConfig(fs afero.Fs, path string, verbose bool, gitleaks gitleaks.Config, format Format, template string, workers int) *Config {
 	p := validate.New().Path(fs, path)
 	return &Config{
 		System:   fs,
 		BasePath: p,
 		Verbose:  verbose,
-		Rules:    rules,
+		Gitleaks: gitleaks,
 		Format:   format,
 		Template: template,
 		Workers:  workers,
@@ -60,7 +52,7 @@ func (c *Config) Default() *Config {
 		BasePath: v.Path(fs, "."),
 		Verbose:  false,
 		Template: DefaultTemplate,
-		Rules:    rules.Load(""),
+		Gitleaks: rules.Load(""),
 		Format:   JSONFormat,
 	}
 }
@@ -70,8 +62,8 @@ func (c *Config) Validate() (err error) {
 		err = fmt.Errorf("missing filesystem in Hunter Config")
 	}
 
-	if c.Rules == nil {
-		err = fmt.Errorf("no rules provided")
+	if c.Gitleaks.Rules == nil {
+		err = fmt.Errorf("no gitleaks config provided")
 	}
 	return
 }

hunter/hound_test.go

@@ -12,9 +12,9 @@ import (
 // has been found and outputs it for the user.
 func ExampleHound_Howl_json() {
 	h := NewHound(&Config{
-		System: afero.NewMemMapFs(),
-		Rules:  rules.Load(""),
-		Format: JSONFormat,
+		System:   afero.NewMemMapFs(),
+		Gitleaks: rules.Load(""),
+		Format:   JSONFormat,
 	})
 	findings := scan.Report{
 		Leaks: []scan.Leak{

hunter/hunter.go

@@ -46,15 +46,18 @@ func (h Hunter) Hunt() error {
 		return fmt.Errorf("config file does not exist")
 	}
 
-	opt := options.Options{Path: h.Config.BasePath, Verbose: h.Config.Verbose}
-	conf := config.Config{Rules: h.Config.Rules}
+	opt := options.Options{Path: h.Config.BasePath, Verbose: h.Config.Verbose, Threads: h.Config.Workers}
+	conf := config.Config{Allowlist: h.Config.Gitleaks.Allowlist, Rules: h.Config.Gitleaks.Rules}
 
 	scanner := scan.NewNoGitScanner(opt, conf)
 	report, err := scanner.Scan()
 	if err != nil {
 		return err
 	}
 
-	h.Hound.Howl(report)
+	if !opt.Verbose {
+		h.Hound.Howl(report)
+	}
+
 	return nil
 }

hunter/template.go

@@ -11,8 +11,9 @@ import (
 // DefaultTemplate is the base template used to
 // format a Finding into the custom output format
 const DefaultTemplate = `{{ with . -}}
-{{ range .Leaks -}}Loot: {{.LineNumber}}
-{{ .File }}
+{{ range .Leaks -}}Line: {{.LineNumber}}
+File: {{ .File }}
+Offender: {{ .Offender }}
 
 {{end}}
 {{- end}}`

main.go

@@ -1,7 +1,7 @@
 package main
 
 //go:generate golangci-lint run ./...
-//go:generate golines ./ -w -m 120
+//go:generate golines ./ -w -m 150
 //go:generate gomarkdoc ./hunter/...
 //go:generate gomarkdoc ./rules/...
 

rules/README.md

@@ -9,7 +9,7 @@ import "github.com/brittonhayes/pillager/rules"
 ## Index
 
 - [Constants](<#constants>)
-- [func Load(filepath string) []gitleaks.Rule](<#func-load>)
+- [func Load(filepath string) gitleaks.Config](<#func-load>)
 
 
 ## Constants
@@ -27,14 +27,6 @@ title = "pillager config"
 	description = "AWS Secret Key"
 	regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]'''
 	tags = ["key", "AWS"]
-[[rules]]
-description = "Email Address"
-regex = '''(?i)([A-Za-z0-9!#$%&'*+\/=?^_{|.}~-]+@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)'''
-tags = ["email", "User Info"]
-[[rules]]
-description = "Github Repository"
-regex = '''((git|ssh|http(s)?)|(git@[\w\.]+))(:(\/\/)?)([\w\.@\:/\-~]+)(\.git)(\/)?'''
-tags = ["repo", "Github"]
 [[rules]]
 	description = "Github"
 	regex = '''(?i)github(.{0,20})?(?-i)[0-9a-zA-Z]{35,40}'''
@@ -47,10 +39,6 @@ tags = ["repo", "Github"]
 	description = "Asymmetric Private Key"
 	regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----'''
 	tags = ["key", "AsymmetricPrivateKey"]
-[[rules]]
-	description = "Google (GCP) Service Account"
-	regex = '''"type": "service_account"'''
-	tags = ["key", "Google"]
 [[rules]]
 	description = "Slack Webhook"
 	regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}'''
@@ -61,7 +49,7 @@ tags = ["repo", "Github"]
 ## func [Load](<https://github.com/brittonhayes/pillager/blob/main/rules/rules.go#L11>)
 
 ```go
-func Load(filepath string) []gitleaks.Rule
+func Load(filepath string) gitleaks.Config
 ```
 
 Load loads the config file into an array of gitleaks rules

rules/default.go

@@ -12,14 +12,6 @@ title = "pillager config"
 	description = "AWS Secret Key"
 	regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]'''
 	tags = ["key", "AWS"]
-[[rules]]
-description = "Email Address"
-regex = '''(?i)([A-Za-z0-9!#$%&'*+\/=?^_{|.}~-]+@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)'''
-tags = ["email", "User Info"]
-[[rules]]
-description = "Github Repository"
-regex = '''((git|ssh|http(s)?)|(git@[\w\.]+))(:(\/\/)?)([\w\.@\:/\-~]+)(\.git)(\/)?'''
-tags = ["repo", "Github"]
 [[rules]]
 	description = "Github"
 	regex = '''(?i)github(.{0,20})?(?-i)[0-9a-zA-Z]{35,40}'''
@@ -32,12 +24,14 @@ tags = ["repo", "Github"]
 	description = "Asymmetric Private Key"
 	regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----'''
 	tags = ["key", "AsymmetricPrivateKey"]
-[[rules]]
-	description = "Google (GCP) Service Account"
-	regex = '''"type": "service_account"'''
-	tags = ["key", "Google"]
 [[rules]]
 	description = "Slack Webhook"
 	regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}'''
 	tags = ["key", "slack"]
+
+[allowlist]
+	description = "Allowlisted files"
+	files = ['''^\.?gitleaks.toml$''',
+	'''(.*?)(png|jpg|gif|doc|docx|pdf|bin|xls|pyc|zip)$''',
+	'''(go.mod|go.sum)$''']
 `

rules/rules.go

@@ -8,7 +8,7 @@ import (
 )
 
 // Load loads the config file into an array of gitleaks rules
-func Load(filepath string) []gitleaks.Rule {
+func Load(filepath string) gitleaks.Config {
 	var (
 		config gitleaks.TomlLoader
 		err    error
@@ -28,5 +28,5 @@ func Load(filepath string) []gitleaks.Rule {
 		log.Fatal("Failed to parse in toml config")
 	}
 
-	return c.Rules
+	return c
 }