Cleanup, simplification, and preparation for new modules (#14)

* Large code cleanup in preparation for expanding with new modules

* Fixed typo in github action

Co-authored-by: Britton Hayes <[email protected]>

.goreleaser.yml

@@ -28,7 +28,7 @@ changelog:
       - '^docs:'
       - '^test:'
       - '^images:'
-      - '^examples:'
+      - '^_examples:'
 scoop:
   # Template for the url which is determined by the given Token (github or gitlab)
   url_template: "https://github.com/brittonhayes/pillager/releases/download/{{ .Tag }}/{{ .ArtifactName }}"

README.md

@@ -63,15 +63,19 @@ pillager [cmd] --help
 
 Pillager provides full support for [Gitleaks](https://github.com/zricethezav/gitleaks) rules. This can either be passed in with a [rules.toml](./rules.toml) file, or you can use the default ruleset by leaving the rules flag blank. 
 
-> Currently entropy is not implemented, but it is absolutely planned to add support for Gitleaks' entropy configs in future updates.
-
 ```toml
 # rules.toml
 title = "pillager rules"
+
 [[rules]]
 description = "AWS Access Key"
 regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
 tags = ["key", "AWS"]
+[[rules.entropies]]
+    Min = "3.5"
+    Max = "4.5"
+    Group = "1"
+
 [[rules]]
 description = "Email Address"
 regex = '''(?i)([A-Za-z0-9!#$%&'*+\/=?^_{|.}~-]+@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)'''
@@ -80,23 +84,15 @@ tags = ["email", "User Info"]
 
 ## Documentation
 
-:books: [View the docs](./pkg/hunter)
+:books: [View the docs](hunter)
 
 GoDoc documentation is available on [pkg.go.dev for pillager](https://pkg.go.dev/github.com/brittonhayes/pillager) but it also available for all packages in the [./pkg](./pkg) directory. 
-Just open the folder of any package and you'll see the GoDocs rendered in beautiful Github-flavored markdown thanks to the awesome [gomarkdoc](https://github.com/princjef/gomarkdoc) tool.
+Just open the folder of any package, and you'll see the GoDocs rendered in beautiful Github-flavored markdown thanks to the awesome [gomarkdoc](https://github.com/princjef/gomarkdoc) tool.
 
 ---
 
 ### Shoulders of Giants :star:
 
-#### [afero's regexpFs](https://github.com/spf13/afero#regexpfs). 
-
-**What is RegexpFs?**
-
-> A filtered view on file names, any file NOT matching the provided patterns will be treated as non-existing.
-
-This is important because it limits the number of files being scanned in the first place, thus cutting down on the time to finish the scan. In other words, we probably don't want to attempt to scan a `*.mp4` from top to bottom, but we definitely _do_ want to scan a `*.env` from top to bottom.
-
 #### [afero's Cobra](https://github.com/spf13/cobra)
 
 **What is Cobra?**

_examples/hunter/main.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"github.com/brittonhayes/pillager/hunter"
+	"github.com/brittonhayes/pillager/rules"
+	"github.com/spf13/afero"
+)
+
+func main() {
+	// Create a new hunter config
+	c := hunter.NewConfig(afero.NewOsFs(), ".", true, rules.Load(""), hunter.StringToFormat("JSON"))
+
+	// Create a new hunter from the config
+	h := hunter.NewHunter(c)
+
+	// Start hunting
+	_ = h.Hunt()
+}

_examples/testdata/github.txt

@@ -0,0 +1 @@
+[email protected]

examples/loot/password.txt → _examples/testdata/password.txt

@@ -13,3 +13,4 @@ [email protected]
 [email protected]
 [email protected]
 [email protected]
+[email protected]

_examples/testdata/ssh.txt

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAsZsEGWDwTgdodV5CXE6it5k7ttD2SN+VTOLY+KLD0pISpZwq
+GQW5u8cfz4yLDm2CWb5wPXhC49Xh5WXB0xjnIbrjU+vIJB46SFyZf73yF8cMtx62
+LnCE28YrqOgQB0QLrPL3fP9JfaZRKzTbE/Ydu11eJSvTpWeyRoLiQpudiQtetSJk
+52q6wpCIYSw+eLV8PiycqBMXreL4+qghbwSf06Ulq/k3I780Fksa4YoPAdSJBsyd
+T1dBPpVueWXiZDDNUDlIEtyMXVZwxM7M0NhyiQ8eq7rkd7Gy2ZqjdhMQyEwCSyRU
+I2kJL4HsVECbNtriUH6wGOTVHlTc5m+0LXI1YwIDAQABAoIBAEpSZV+9ajIs9XCQ
+FlMMcRlFnXB+d2G9dnvrPgKmEi4vBHCbLh60f6ntxQkUdih+eRrXD58ZkVxIZ7SY
+ZYjSFf7Od0zqESHUKG2Zmn6TXRjIKjEDQBek7GQPxLt1CSgROJWSQzDin71d+P+M
+qdTrnDuI8dm6QmIoR7ksEkTvW/g5wFA+qJFHPYtYxN/zasL3JRuMi+SwNgiQBW9v
+lhFdrgJICsBiCKAKhYmo2dNQKVDiiHW9kqyBmyGd5HKPFa8eN7vJfaOkB4tWy66G
+0hyFG1YLDKO6MCZa3VxedjUyD1YE0KMy4zpXR1OdgxdOgFlWIpvvgD03TmU3wTdE
+UHiJI/ECgYEA2lMaSMSsq8Ye1DWpr4w/UEaRBAqoBRUKrm6Bn8jScuC/E1b2bZ/g
+TXCscqKScDOHfNhTTrkenb274A5hiXCtz/8OkABC1b03+WnJNRyoIDj6kWQwOU2j
+ls8lYQKTsbKLpkOxJuANKQNpkIO3dcnHTwcNxvywBGqxAxL5DsykvJsCgYEA0EES
+V6J8zWfJ3dUppj9E7msF/LnLV6pdhfLBbWAApv00HhIrDmwMLW3cYc5RbBJ4+2oE
+xUZcj7ipLyBymHYrE//91aSuDTf4Y/IuS2szK3+UiBBDmxgt59lltlCVdhptNeW4
+Iu/dr1zlzjOsJQa1k7JqludInhD0xJ4jussBYtkCgYAaOxXTXF3qnXkiwffLMayB
+tpjBxY3+XlLUNk64peYzCzGp3y7oaOcIJttxQnsZo/dJVqxbrv4wsZUbDF9HphQc
+A2b6gi2FhKBvBPi5uuGwTWg9g64PlHZtLiItj6OzG+KPbV6hbxXlsZVDWa/3JPEf
+W5eaKs3zGeZ+mg72ajv4tQKBgDMB/TAhyFe3OCZxc4YYsRWEeS7xgvCx9b49O0tC
+3GYKVp2W7ZzWshTed8As/Jxv2doamZiIJ1u8axPz6p2VM+/uH4oYj9kFAH3ab6Rh
+WWMqGINLGIQAZB4JjXQeQL1ws5cj8Pn9rlqpclATL/LRtd+7L4hOlebBWHJgGXhD
+CM/hAoGBAJC3PdClk2nUT3gVLSaIaS+MuwV/pOIydB8mP7arTW/I1hR/m32pNdhL
+o7FreZIOKDPWPcjd4hKlyMtteQoZtBT8mzvOKMnKGT4D3m+tHqQddxabBQ3yeDmh
+wXfDn7PCx2tH98tQoo9wO+HuRdgXTLjlrJ7tSV8ovLQmXUynOw9D
+-----END RSA PRIVATE KEY-----

benchmarks_test.go

@@ -1,71 +1,42 @@
 package main_test
 
 import (
-	"github.com/brittonhayes/pillager/pkg/hunter"
-	"github.com/spf13/afero"
 	"log"
 	"os"
 	"testing"
+
+	"github.com/brittonhayes/pillager/hunter"
+	"github.com/brittonhayes/pillager/rules"
+	"github.com/spf13/afero"
+	"github.com/zricethezav/gitleaks/v7/scan"
 )
 
 // A benchmark of the Hunter Load Rules method
 func BenchmarkHunterLoadRules(b *testing.B) {
 	for n := 0; n < b.N; n++ {
-		hunter.LoadRules("")
+		rules.Load("")
 	}
 }
 
-// A benchmark of the Hound Fetch method which
+// A benchmark of the Hound Howl method which
 // prints results out in desired format
-func BenchmarkHunterHoundFetch(b *testing.B) {
+func BenchmarkHunterHoundHowl(b *testing.B) {
 	b.StopTimer()
 	h := hunter.NewHound(&hunter.Config{
 		System: afero.NewMemMapFs(),
-		Rules:  hunter.LoadRules(""),
+		Rules:  rules.Load(""),
 		Format: hunter.JSONFormat,
 	})
-	h.Findings = []hunter.Finding{
-		{
-			Count:   1,
-			Message: "Found something juicy",
-			Path:    "example.toml",
-			Loot:    []string{"Token 1234560"},
+	findings := scan.Report{
+		Leaks: []scan.Leak{
+			{Line: "[email protected]", LineNumber: 16, Offender: "[email protected]", Rule: "Email Addresses"},
 		},
 	}
 
 	b.StartTimer()
 	for n := 0; n < b.N; n++ {
 		defer quiet()()
-		h.Fetch()
-	}
-}
-
-// A benchmark of the Hunter Inspect method
-func BenchmarkHunterInspect(b *testing.B) {
-	b.StopTimer()
-	fs := afero.NewMemMapFs()
-	f, err := fs.Create("fake.toml")
-	if err != nil {
-		panic(err)
-	}
-	defer f.Close()
-	_, err = f.Write([]byte(`[email protected]`))
-	if err != nil {
-		panic(err)
-	}
-
-	h := hunter.NewHunter(&hunter.Config{
-		System:   fs,
-		Rules:    hunter.LoadRules(""),
-		BasePath: ".",
-		Verbose:  true,
-		Format:   hunter.JSONFormat,
-	})
-
-	b.StartTimer()
-	for n := 0; n < b.N; n++ {
-		defer quiet()()
-		h.Inspect(f.Name(), h.Config.System)
+		h.Howl(findings)
 	}
 }
 

cmd/hunt.go

@@ -4,7 +4,8 @@
 package cmd
 
 import (
-	"github.com/brittonhayes/pillager/pkg/hunter"
+	"github.com/brittonhayes/pillager/hunter"
+	"github.com/brittonhayes/pillager/rules"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 )
@@ -13,39 +14,56 @@ var (
 	verbose     bool
 	rulesConfig string
 	output      string
+	templ       string
 )
 
 // huntCmd represents the hunt command
 var huntCmd = &cobra.Command{
 	Use:   "hunt [directory]",
 	Short: "Hunt for loot",
 	Long:  "Hunt inside the file system for valuable information",
-	Args:  cobra.MinimumNArgs(1),
-	RunE:  StartHunt(),
+	Example: `
+# Run a basic hunt
+pillager hunt ./
+
+# Print out results in JSON format
+pillager hunt ./example -r rules.toml -f json
+
+# Print out results with a custom inline template
+pillager hunt ./example -r rules.toml -f custom --template "{{ range .Leaks}}Leak: {{.Line}}{{end}}"
+
+# Print out results with a custom template file
+pillager hunt ./example -r rules.toml -f custom --template "$(cat templates/simple.tmpl)"
+`,
+	Args: cobra.MinimumNArgs(1),
+	RunE: StartHunt(),
 }
 
 func init() {
 	rootCmd.AddCommand(huntCmd)
 	huntCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "toggle verbose output")
-	huntCmd.Flags().StringVarP(&rulesConfig, "rules-config", "r", "", "path to gitleaks rules config")
-	huntCmd.Flags().StringVarP(&output, "output", "o", "yaml", "set output format (json, yaml)")
+	huntCmd.Flags().StringVarP(&rulesConfig, "rules", "r", "", "path to gitleaks rules.toml config")
+	huntCmd.Flags().StringVarP(&output, "format", "f", "yaml", "set output format (json, yaml, custom)")
+	huntCmd.Flags().StringVarP(
+		&templ,
+		"template",
+		"t",
+		hunter.DefaultTemplate,
+		"set go text/template string for output format",
+	)
 }
 
 func StartHunt() func(cmd *cobra.Command, args []string) error {
 	return func(cmd *cobra.Command, args []string) error {
-		fs := afero.NewOsFs()
-		c := hunter.Config{
-			System:   fs,
-			BasePath: hunter.CheckPath(fs, args[0]),
-			Verbose:  verbose,
-			Format:   hunter.StringToFormat(output),
-			Rules:    hunter.LoadRules(rulesConfig),
-		}
-		h := hunter.NewHunter(&c)
-		err := h.Hunt()
-		if err != nil {
-			return err
-		}
-		return nil
+		c := hunter.NewConfig(
+			afero.NewOsFs(),
+			args[0],
+			verbose,
+			rules.Load(rulesConfig),
+			hunter.StringToFormat(output),
+			templ,
+		)
+		h := hunter.NewHunter(c)
+		return h.Hunt()
 	}
 }

cmd/root.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+
 	"github.com/gookit/color"
 	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"os"
 )
 
 var cfgFile string

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.7.1
 	github.com/zricethezav/gitleaks/v7 v7.2.2
+	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
 	golang.org/x/sys v0.0.0-20201204225414-ed752295db88 // indirect
 	golang.org/x/text v0.3.4 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect