diff --git a/.github/workflows/gobuild.yml b/.github/workflows/gobuild.yml index 2779c9f..6c99de5 100644 --- a/.github/workflows/gobuild.yml +++ b/.github/workflows/gobuild.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version: 1.21 - name: Build executables run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ecc54e..e9bb3af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version: 1.21 - name: Run GoReleaser and release executables uses: goreleaser/goreleaser-action@v2 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4007b40..6fc8537 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version: 1.21 - name: Test run: | diff --git a/.golangci.yaml b/.golangci.yaml index 7cdf1ea..1b2753b 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,14 +1,11 @@ linters: enable: - - gofumpt - gofmt - gosimple - - godot - godox - dupl - funlen - gocritic - goprintffuncname - - ifshort presets: - unused diff --git a/Dockerfile b/Dockerfile index e55295e..e5cde6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17-alpine +FROM golang:1.21-alpine WORKDIR /src/ COPY . /src/ diff --git a/README.md b/README.md index 83d4357..82440f8 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Pillager is designed to provide a simple means of leveraging Go's strong concurr directories for sensitive information in files. Pillager does this by standing on the shoulders of [a few giants](#shoulders-of-giants). Once pillager finds files that match the specified pattern, the file is scanned using a series of concurrent workers that each take a line of the file from the job queue and hunt for sensitive pattern -matches. The available pattern filters can be defined in a rules.toml file or you can use the default ruleset. +matches. The available pattern filters can be defined in a pillager.toml file or you can use the default ruleset. ## Installation @@ -31,7 +31,7 @@ matches. The available pattern filters can be defined in a rules.toml file or yo If you have Go setup on your system, you can install Pillager with `go get` ```shell script -go get github.com/brittonhayes/pillager +go install github.com/brittonhayes/pillager@latest ``` ### Scoop (Windows) @@ -79,29 +79,56 @@ Pillager provides a terminal user interface built with [bubbletea](https://githu ### Gitleaks Rules Pillager provides full support for Gitleaks[^2] rules. This can either be passed -in with a rules.toml[^1] file, or you can use the default ruleset by leaving the rules flag blank. +in with a rules[^1] section in your pillager.toml file, or you can use the default ruleset by leaving the config flag blank. [^1]: [Gitleaks Rules Reference](https://github.com/zricethezav/gitleaks/blob/57f9bc83d169bea363f2990a4de334b54efc3d7d/config/gitleaks.toml) ```toml -# rules.toml -title = "pillager rules" +# pillager.toml +# Basic configuration +verbose = false +path = "." +workers = 4 +redact = true +reporter = "json-pretty" + +# Rules for secret detection +[[rules]] +description = "AWS Access Key" +id = "aws-access-key" +regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' +tags = ["aws", "credentials"] [[rules]] -id = "gitlab-pat" -description = "GitLab Personal Access Token" -regex = '''glpat-[0-9a-zA-Z\-\_]{20}''' +description = "AWS Secret Key" +id = "aws-secret-key" +regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]''' +tags = ["aws", "credentials"] [[rules]] -id = "aws-access-token" -description = "AWS" -regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' +description = "GitHub Token" +id = "github-token" +regex = '''ghp_[0-9a-zA-Z]{36}''' +tags = ["github", "token"] -# Cryptographic keys [[rules]] -id = "PKCS8-PK" -description = "PKCS8 private key" -regex = '''-----BEGIN PRIVATE KEY-----''' +description = "Private Key" +id = "private-key" +regex = '''-----BEGIN (?:RSA|OPENSSH|DSA|EC|PGP) PRIVATE KEY( BLOCK)?-----''' +tags = ["key", "private"] + +# Allowlist configuration +[allowlist] +paths = [ + ".*/_test\\.go$", + ".*/testdata/.*", + ".*\\.md$", + ".*/vendor/.*" +] +regexes = [ + "EXAMPLE_KEY", + "DUMMY_SECRET" +] ``` ### Built-in Output Formats @@ -132,10 +159,10 @@ pillager hunt ./example -f json | jq pillager hunt . -f yaml ``` -#### TOML +#### JSON Pretty ```shell -pillager hunt . -f toml +pillager hunt . -f json-pretty ``` #### HTML @@ -171,14 +198,14 @@ pillager hunt . --template "{{ range .}}Secret: {{.Secret}}{{end}}" #### Custom Go Template from File ```shell -pillager hunt . -t "$(cat pkg/templates/simple.tmpl)" +pillager hunt . -t "$(cat internal/templates/simple.tmpl)" ``` ### Custom Templates -Pillager allows you to use powerful `go text/template` to customize the output format. Here are a few template examples. +Pillager allows you to use powerful `go text/template` and [sprig](https://masterminds.github.io/sprig/) functions to customize the output format. Here are a few template examples. #### Basic @@ -203,16 +230,11 @@ Pillager allows you to use powerful `go text/template` to customize the output f ``` -> More template examples can be found in the [templates](./pkg/templates) directory. +> More template examples can be found in the [templates](./internal/templates) directory. ## Documentation -:books: [View the docs](pkg/hunter) - -GoDoc documentation is available on [pkg.go.dev for pillager](https://pkg.go.dev/github.com/brittonhayes/pillager) but -it is also available for all packages in the repository in markdown format. Just open the folder of any package, and -you'll see the GoDocs rendered in beautiful Github-flavored markdown thanks to the -awesome [gomarkdoc](https://github.com/princjef/gomarkdoc) tool. +GoDoc documentation is available on [pkg.go.dev for pillager](https://pkg.go.dev/github.com/brittonhayes/pillager). ## Development @@ -237,7 +259,7 @@ If you've seen a CLI written in Go before, there's a pretty high chance it was b library enough. It empowers developers to make consistent, dynamic, and self-documenting command line tools with ease. Some examples include `kubectl`, `hugo`, and Github's `gh` CLI. -#### [Gitleaks](https://github.com/zricethezav/gitleaks) +#### [Gitleaks](https://github.com/gitleaks/gitleaks) **What is Gitleaks?** @@ -248,9 +270,9 @@ it's worth your time to check it out. **Why is Gitleaks relevant to Pillager?** -[^2]: [Gitleaks](https://github.com/zricethezav/gitleaks) +[^2]: [Gitleaks](https://github.com/gitleaks/gitleaks) -Pillager implements the powerful [rules](https://github.com/zricethezav/gitleaks#rules-summary) functionality of +Pillager implements the powerful [rules](https://github.com/gitleaks/gitleaks#rules-summary) functionality of Gitleaks while taking a different approach to presenting and handling the secrets found. While I have provided a baseline set of default rules, Pillager becomes much more powerful if you allow users to create rules for their own use-cases. diff --git a/Taskfile.yml b/Taskfile.yml index d60a1d9..24f010f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -17,8 +17,8 @@ tasks: cmds: - rm bin/pillager - go clean -cache - - rm -rf pkg/hunter/testdata - - mkdir pkg/hunter/testdata + - rm -rf pkg/pillager/testdata + - mkdir pkg/pillager/testdata lint: desc: runs golint diff --git a/_examples/hunter/main.go b/_examples/hunter/main.go deleted file mode 100644 index f65c999..0000000 --- a/_examples/hunter/main.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "os" - - "github.com/brittonhayes/pillager/pkg/format" - "github.com/brittonhayes/pillager/pkg/hunter" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" -) - -func main() { - // Create a new hunter config - h, err := hunter.New( - hunter.WithScanPath("."), - hunter.WithWorkers(2), - hunter.WithFormat(format.Simple{}), - hunter.WithLogLevel(zerolog.DebugLevel.String()), - ) - if err != nil { - log.Fatal().Err(err).Send() - } - - // Start hunting - results, err := h.Hunt() - if err != nil { - log.Fatal().Err(err).Send() - } - - // Report results - if err = h.Report(os.Stdout, results); err != nil { - log.Fatal().Err(err).Send() - } -} diff --git a/_examples/scanner/main.go b/_examples/scanner/main.go new file mode 100644 index 0000000..d7f489a --- /dev/null +++ b/_examples/scanner/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "fmt" + "runtime" + + "github.com/brittonhayes/pillager" + "github.com/brittonhayes/pillager/pkg/scanner" +) + +func main() { + // Set scanner options + opts := pillager.Options{ + Path: ".", + Workers: runtime.NumCPU(), + Reporter: "json", + Redact: true, + } + + // Create a new gitleaks scanner + s, _ := scanner.NewGitleaksScanner(opts) + + // Scan the current directory + results, _ := s.Scan(opts.Path) + + // Report results + fmt.Println(results) +} diff --git a/cmd/pillager/main.go b/cmd/pillager/main.go index e54566f..86b6841 100644 --- a/cmd/pillager/main.go +++ b/cmd/pillager/main.go @@ -1,10 +1,19 @@ -// Package pillager is the entrypoint to the Pillager CLI package main import ( - pillager "github.com/brittonhayes/pillager/internal/commands" + "os" + + "github.com/brittonhayes/pillager/internal/commands" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) +func init() { + // Setup default logging + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + zerolog.SetGlobalLevel(zerolog.InfoLevel) +} + func main() { - pillager.Execute() + commands.Execute() } diff --git a/doc.go b/doc.go deleted file mode 100644 index 23d5449..0000000 --- a/doc.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright © 2020 Britton Hayes - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -// Package pillager is a tool for hunting through filesystems for sensitive information. -// -// Installation -// -// Go -// -// go get github.com/brittonhayes/pillager -// -// Windows -// -// scoop bucket add pillager https://github.com/brittonhayes/pillager-scoop.git -// scoop install pillager -// -// OSX/Linux -// -// brew tap brittonhayes/homebrew-pillager -// brew install pillager -// -//go:generate golangci-lint run ./... -//go:generate gomarkdoc ./pkg/hunter/... -//go:generate gomarkdoc ./pkg/rules/... -//go:generate gomarkdoc ./pkg/format/... -package pillager diff --git a/go.mod b/go.mod index 94f7744..54a0227 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/brittonhayes/pillager -go 1.18 +go 1.21 require ( github.com/BurntSushi/toml v1.1.0 diff --git a/internal/commands/hunt.go b/internal/commands/hunt.go index 7f29ac9..7d93576 100644 --- a/internal/commands/hunt.go +++ b/internal/commands/hunt.go @@ -4,14 +4,16 @@ package commands import ( + "fmt" "os" "runtime" - "github.com/brittonhayes/pillager/pkg/format" - "github.com/brittonhayes/pillager/pkg/hunter" - "github.com/brittonhayes/pillager/pkg/rules" + "github.com/brittonhayes/pillager" + "github.com/brittonhayes/pillager/pkg/scanner" "github.com/brittonhayes/pillager/pkg/tui/model" tea "github.com/charmbracelet/bubbletea" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -19,11 +21,11 @@ var ( verbose bool redact bool level string - rulesConfig string reporter string templ string workers int interactive bool + config string ) // huntCmd represents the hunt command. @@ -56,46 +58,77 @@ var huntCmd = &cobra.Command{ Custom Go Template Format from Template File: pillager hunt ./example --template "$(cat pkg/templates/simple.tmpl)" `, - Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - // Read gitleaks config from file - // or fallback to default - gitleaksConfig := rules.NewLoader( - rules.WithFile(rulesConfig), - ).Load() + if level != "" { + lvl, err := zerolog.ParseLevel(level) + if err != nil { + return fmt.Errorf("invalid log level: %w", err) + } + zerolog.SetGlobalLevel(lvl) + } - h, err := hunter.New( - hunter.WithGitleaksConfig(gitleaksConfig), - hunter.WithScanPath(args[0]), - hunter.WithWorkers(workers), - hunter.WithVerbose(verbose), - hunter.WithTemplate(templ), - hunter.WithRedact(redact), - hunter.WithFormat(format.StringToReporter(reporter)), - hunter.WithLogLevel(level), - ) + configLoader := scanner.NewConfigLoader() + opts, err := configLoader.LoadConfig(config) if err != nil { - return err + if config != "" { + return fmt.Errorf("failed to load config: %w", err) + } + opts = &pillager.Options{ + Workers: runtime.NumCPU(), + Verbose: false, + Template: "", + Redact: false, + Reporter: "json", + } + } + + // Get path from args if provided, otherwise use config path + scanPath := "" + if len(args) > 0 { + scanPath = args[0] + } + + // Merge command line flags with config file + flagOpts := &pillager.Options{ + Path: scanPath, + Redact: redact, + Verbose: verbose, + Workers: workers, + Reporter: reporter, + Template: templ, + } + configLoader.MergeWithFlags(opts, flagOpts) + + // Check if path is provided either via args or config + if opts.Path == "" { + return fmt.Errorf("scan path must be provided either as an argument or in the config file") + } + + s, err := scanner.NewGitleaksScanner(*opts) + if err != nil { + return fmt.Errorf("failed to create scanner: %w", err) } if interactive { - return runInteractive(h) + return runInteractive(s) } - results, err := h.Hunt() + results, err := s.Scan() if err != nil { return err } - if err = h.Report(os.Stdout, results); err != nil { - return err + if len(results) == 0 { + fmt.Println("[]") + log.Debug().Msg("no secrets or sensitive information were found at the target directory") + return nil } - return nil + return s.Reporter().Report(os.Stdout, results) }, } -func runInteractive(h *hunter.Hunter) error { +func runInteractive(h scanner.Scanner) error { m := model.NewModel(h) p := tea.NewProgram(m, tea.WithAltScreen()) return p.Start() @@ -106,9 +139,9 @@ func init() { huntCmd.Flags().BoolVarP(&interactive, "interactive", "i", false, "run in interactive mode") huntCmd.Flags().IntVarP(&workers, "workers", "w", runtime.NumCPU(), "number of concurrent workers") huntCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "enable scanner verbose output") - huntCmd.Flags().StringVarP(&level, "log-level", "l", "error", "set logging level") - huntCmd.Flags().StringVarP(&rulesConfig, "rules", "r", "", "path to gitleaks rules.toml config") - huntCmd.Flags().StringVarP(&reporter, "format", "f", "json", "set secret reporter (json, yaml)") + huntCmd.Flags().StringVarP(&level, "log-level", "l", "info", "set logging level") + huntCmd.Flags().StringVarP(&config, "config", "c", "", "path to pillager config file") + huntCmd.Flags().StringVarP(&reporter, "format", "f", "json-pretty", "set secret reporter format (json, yaml, html, html-table, table, markdown)") huntCmd.Flags().BoolVar(&redact, "redact", false, "redact secret from results") huntCmd.Flags().StringVarP(&templ, "template", "t", "", "set go text/template string for output format") } diff --git a/internal/templates/html-table.tmpl b/internal/templates/html-table.tmpl new file mode 100644 index 0000000..e6b37cd --- /dev/null +++ b/internal/templates/html-table.tmpl @@ -0,0 +1,175 @@ + + + + + + Pillager - Scan Results + + + + + + +
+
+

+ Pillager +

+

+ Results of your latest hunt from {{ now.Format "2006-01-02 15:04 MST" }} +

+ + {{/* Group results by file using dict/values */}} + {{ $grouped := dict }} + {{ range . }} + {{ $findings := index $grouped .File | default list }} + {{ $grouped = set $grouped .File (append $findings .) }} + {{ end }} + + {{ range $file, $findings := $grouped }} +
+ + {{ $file }} + {{ len $findings }} findings + +
+ + + + + + + + + {{ range $findings }} + + + + + {{ end }} + +
LineSecret
{{.StartLine}}{{.Secret}}
+
+
+ {{ end }} +
+
+ + + diff --git a/internal/templates/html.tmpl b/internal/templates/html.tmpl new file mode 100644 index 0000000..103fcd1 --- /dev/null +++ b/internal/templates/html.tmpl @@ -0,0 +1,192 @@ + + + + + + Pillager - Scan Results + + + + + + +
+
+

+ Pillager +

+

+ Results of your latest hunt from {{ now.Format "2006-01-02 15:04 MST" }} +

+ +
+ {{ range . }} +
+
{{.File}}
+
{{.Secret}}
+
+ View JSON + Line: {{.StartLine}} +
+
+
+ {{end}} +
+
+
+ + + + diff --git a/internal/templates/markdown.tmpl b/internal/templates/markdown.tmpl new file mode 100644 index 0000000..21000a9 --- /dev/null +++ b/internal/templates/markdown.tmpl @@ -0,0 +1,25 @@ +# Pillager +Results of your latest hunt from {{ now.Format "2006-01-02 15:04 MST" }} + +{{ len . }} Findings + +{{ range . -}} +
+Finding in `{{ .File }}` (Line {{ .StartLine }}) + +- **Location**: Line {{ .StartLine }} to {{ .EndLine }} +{{- if .RuleID }} +- **Rule Matched**: {{ .RuleID }} +{{- end }} +{{- if .Description }} +- **Description**: {{ .Description }} +{{- end }} +- **Secret**: {{ .Secret }} +- **Match Context**: `{{ .Match }}` +{{- if .StartColumn }} +- **Position**: Column {{ .StartColumn }} +{{- end }} + +
+ +{{end}} diff --git a/pkg/templates/simple.tmpl b/internal/templates/simple.tmpl similarity index 100% rename from pkg/templates/simple.tmpl rename to internal/templates/simple.tmpl diff --git a/pkg/templates/table.tmpl b/internal/templates/table.tmpl similarity index 64% rename from pkg/templates/table.tmpl rename to internal/templates/table.tmpl index 1a65f34..1f9d6ac 100644 --- a/pkg/templates/table.tmpl +++ b/internal/templates/table.tmpl @@ -1,3 +1,6 @@ +# Pillager +Results of your latest hunt from {{ now.Format "2006-01-02 15:04 MST" }} + | File | Line | Secret | | --------| ---------| -------- | {{ range . -}} diff --git a/internal/templates/templates.go b/internal/templates/templates.go new file mode 100644 index 0000000..5d7d3c6 --- /dev/null +++ b/internal/templates/templates.go @@ -0,0 +1,33 @@ +// Package templates contains a compilation of go templates for rendering secret findings. +package templates + +import ( + _ "embed" +) + +var ( + //go:embed simple.tmpl + Simple string + + //go:embed html.tmpl + HTML string + + //go:embed markdown.tmpl + Markdown string + + //go:embed table.tmpl + Table string + + //go:embed html-table.tmpl + HTMLTable string +) + +// DefaultTemplate is the base template used to format a Finding into the +// custom output format. +const DefaultTemplate = `{{ with . -}} +{{ range . -}} +Line: {{ quote .StartLine}} +File: {{ quote .File }} +Secret: {{ quote .Secret }} +--- +{{ end -}}{{- end}}` diff --git a/pillager.go b/pillager.go new file mode 100644 index 0000000..01129cc --- /dev/null +++ b/pillager.go @@ -0,0 +1,96 @@ +/* +Copyright © 2020 Britton Hayes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +// Package pillager is a tool for hunting through filesystems for sensitive information. +// +// # Installation +// +// Go +// +// go install github.com/brittonhayes/pillager@latest +// +// Windows +// +// scoop bucket add pillager https://github.com/brittonhayes/pillager-scoop.git +// scoop install pillager +// +// OSX/Linux +// +// brew tap brittonhayes/homebrew-pillager +// brew install pillager +// +//go:generate golangci-lint run ./... +package pillager + +// Finding contains information about strings that +// have been captured by a tree-sitter query. +type Finding struct { + Description string + StartLine int + EndLine int + StartColumn int + EndColumn int + + Match string + + // Secret contains the full content of what is matched in + // the tree-sitter query. + Secret string + + // File is the name of the file containing the finding + File string + + // Entropy is the shannon entropy of Value + Entropy float32 + + // Rule is the name of the rule that was matched + RuleID string +} + +// Options holds configuration for scanners +type Options struct { + Path string `toml:"path"` + Template string `toml:"template"` + Workers int `toml:"workers"` + Verbose bool `toml:"verbose"` + Redact bool `toml:"redact"` + Reporter string `toml:"reporter"` + Rules []Rule `toml:"rules"` + Allowlist Allowlist `toml:"allowlist"` +} + +// Rule represents a scanning rule +type Rule struct { + ID string `toml:"id"` + Description string `toml:"description"` + Path string `toml:"path"` + Regex string `toml:"regex"` + Keywords []string `toml:"keywords"` + Tags []string `toml:"tags"` + Allowlist Allowlist +} + +// Allowlist represents paths and patterns to ignore +type Allowlist struct { + Paths []string `toml:"paths"` + Regexes []string `toml:"regexes"` +} diff --git a/pillager.toml b/pillager.toml new file mode 100644 index 0000000..ff1853c --- /dev/null +++ b/pillager.toml @@ -0,0 +1,44 @@ +# Basic configuration +verbose = false +path = "." +workers = 4 +redact = true +reporter = "json-pretty" + +# Rules for secret detection +[[rules]] +description = "AWS Access Key" +id = "aws-access-key" +regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' +tags = ["aws", "credentials"] + +[[rules]] +description = "AWS Secret Key" +id = "aws-secret-key" +regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]''' +tags = ["aws", "credentials"] + +[[rules]] +description = "GitHub Token" +id = "github-token" +regex = '''ghp_[0-9a-zA-Z]{36}''' +tags = ["github", "token"] + +[[rules]] +description = "Private Key" +id = "private-key" +regex = '''-----BEGIN (?:RSA|OPENSSH|DSA|EC|PGP) PRIVATE KEY( BLOCK)?-----''' +tags = ["key", "private"] + +# Allowlist configuration +[allowlist] +paths = [ + ".*/_test\\.go$", + ".*/testdata/.*", + ".*\\.md$", + ".*/vendor/.*" +] +regexes = [ + "EXAMPLE_KEY", + "DUMMY_SECRET" +] \ No newline at end of file diff --git a/pkg/format/README.md b/pkg/format/README.md deleted file mode 100755 index 15c33cc..0000000 --- a/pkg/format/README.md +++ /dev/null @@ -1,172 +0,0 @@ - - -# format - -```go -import "github.com/brittonhayes/pillager/pkg/format" -``` - -Package format contains the renderer and available output formats - -## Index - -- [type Custom](<#type-custom>) - - [func (c Custom) Report(w io.Writer, findings []report.Finding) error](<#func-custom-report>) - - [func (c *Custom) WithTemplate(t string)](<#func-custom-withtemplate>) -- [type HTML](<#type-html>) - - [func (h HTML) Report(w io.Writer, findings []report.Finding) error](<#func-html-report>) -- [type HTMLTable](<#type-htmltable>) - - [func (h HTMLTable) Report(w io.Writer, findings []report.Finding) error](<#func-htmltable-report>) -- [type JSON](<#type-json>) - - [func (j JSON) Report(w io.Writer, findings []report.Finding) error](<#func-json-report>) -- [type Markdown](<#type-markdown>) - - [func (m Markdown) Report(w io.Writer, findings []report.Finding) error](<#func-markdown-report>) -- [type Reporter](<#type-reporter>) - - [func StringToReporter(s string) Reporter](<#func-stringtoreporter>) -- [type Simple](<#type-simple>) - - [func (s Simple) Report(w io.Writer, findings []report.Finding) error](<#func-simple-report>) -- [type TOML](<#type-toml>) - - [func (t TOML) Report(w io.Writer, findings []report.Finding) error](<#func-toml-report>) -- [type Table](<#type-table>) - - [func (t Table) Report(w io.Writer, findings []report.Finding) error](<#func-table-report>) -- [type YAML](<#type-yaml>) - - [func (y YAML) Report(w io.Writer, findings []report.Finding) error](<#func-yaml-report>) - - -## type [Custom]() - -```go -type Custom struct { - // contains filtered or unexported fields -} -``` - -### func \(Custom\) [Report]() - -```go -func (c Custom) Report(w io.Writer, findings []report.Finding) error -``` - -### func \(\*Custom\) [WithTemplate]() - -```go -func (c *Custom) WithTemplate(t string) -``` - -## type [HTML]() - -```go -type HTML struct{} -``` - -### func \(HTML\) [Report]() - -```go -func (h HTML) Report(w io.Writer, findings []report.Finding) error -``` - -## type [HTMLTable]() - -```go -type HTMLTable struct{} -``` - -### func \(HTMLTable\) [Report]() - -```go -func (h HTMLTable) Report(w io.Writer, findings []report.Finding) error -``` - -## type [JSON]() - -```go -type JSON struct{} -``` - -### func \(JSON\) [Report]() - -```go -func (j JSON) Report(w io.Writer, findings []report.Finding) error -``` - -## type [Markdown]() - -```go -type Markdown struct{} -``` - -### func \(Markdown\) [Report]() - -```go -func (m Markdown) Report(w io.Writer, findings []report.Finding) error -``` - -## type [Reporter]() - -Reporter is the interface that each of the canonical output formats implement\. - -```go -type Reporter interface { - Report(io.Writer, []report.Finding) error -} -``` - -### func [StringToReporter]() - -```go -func StringToReporter(s string) Reporter -``` - -StringToReporter takes in a string representation of the preferred reporter\. - -## type [Simple]() - -```go -type Simple struct{} -``` - -### func \(Simple\) [Report]() - -```go -func (s Simple) Report(w io.Writer, findings []report.Finding) error -``` - -## type [TOML]() - -```go -type TOML struct{} -``` - -### func \(TOML\) [Report]() - -```go -func (t TOML) Report(w io.Writer, findings []report.Finding) error -``` - -## type [Table]() - -```go -type Table struct{} -``` - -### func \(Table\) [Report]() - -```go -func (t Table) Report(w io.Writer, findings []report.Finding) error -``` - -## type [YAML]() - -```go -type YAML struct{} -``` - -### func \(YAML\) [Report]() - -```go -func (y YAML) Report(w io.Writer, findings []report.Finding) error -``` - - - -Generated by [gomarkdoc]() diff --git a/pkg/format/format.go b/pkg/format/format.go deleted file mode 100644 index e765868..0000000 --- a/pkg/format/format.go +++ /dev/null @@ -1,35 +0,0 @@ -// Package format contains the renderer and available output formats -package format - -import ( - "strings" -) - -// StringToReporter takes in a string representation of the preferred -// reporter. -func StringToReporter(s string) Reporter { - switch strings.ToLower(s) { - case "json": - return JSON{} - case "raw": - return Raw{} - case "yaml": - return YAML{} - case "toml": - return TOML{} - case "table": - return Table{} - case "html": - return HTML{} - case "html-table": - return HTMLTable{} - case "markdown": - return Markdown{} - case "custom": - return Custom{} - case "simple": - return Simple{} - default: - return JSON{} - } -} diff --git a/pkg/format/report.go b/pkg/format/report.go deleted file mode 100644 index 2dd372f..0000000 --- a/pkg/format/report.go +++ /dev/null @@ -1,101 +0,0 @@ -package format - -import ( - "encoding/json" - "fmt" - "io" - - "github.com/BurntSushi/toml" - "github.com/brittonhayes/pillager/pkg/templates" - "github.com/zricethezav/gitleaks/v8/report" - "gopkg.in/yaml.v2" -) - -// Reporter is the interface that each of the canonical output formats implement. -type Reporter interface { - Report(io.Writer, []report.Finding) error -} - -type JSON struct{} - -func (j JSON) Report(w io.Writer, findings []report.Finding) error { - encoder := json.NewEncoder(w) - encoder.SetIndent("", "\t") - if err := encoder.Encode(&findings); err != nil { - return err - } - - return nil -} - -type Raw struct{} - -func (r Raw) Report(w io.Writer, findings []report.Finding) error { - encoder := json.NewEncoder(w) - if err := encoder.Encode(&findings); err != nil { - return err - } - - return nil -} - -type YAML struct{} - -func (y YAML) Report(w io.Writer, findings []report.Finding) error { - b, err := yaml.Marshal(&findings) - if err != nil { - return err - } - fmt.Fprintf(w, "%s\n", string(b)) - - return nil -} - -type TOML struct{} - -func (t TOML) Report(w io.Writer, findings []report.Finding) error { - enc := toml.NewEncoder(w) - return enc.Encode(&findings) -} - -type HTML struct{} - -func (h HTML) Report(w io.Writer, findings []report.Finding) error { - return templates.Render(w, templates.HTML, findings) -} - -type HTMLTable struct{} - -func (h HTMLTable) Report(w io.Writer, findings []report.Finding) error { - return templates.Render(w, templates.HTMLTable, findings) -} - -type Markdown struct{} - -func (m Markdown) Report(w io.Writer, findings []report.Finding) error { - return templates.Render(w, templates.Markdown, findings) -} - -type Table struct{} - -func (t Table) Report(w io.Writer, findings []report.Finding) error { - return templates.Render(w, templates.Table, findings) -} - -type Custom struct { - template string -} - -func (c *Custom) WithTemplate(t string) { - c.template = t -} - -func (c Custom) Report(w io.Writer, findings []report.Finding) error { - return templates.Render(w, c.template, findings) -} - -type Simple struct{} - -func (s Simple) Report(w io.Writer, findings []report.Finding) error { - return templates.Render(w, templates.Simple, findings) -} diff --git a/pkg/hunter/README.md b/pkg/hunter/README.md deleted file mode 100644 index 6132d46..0000000 --- a/pkg/hunter/README.md +++ /dev/null @@ -1,156 +0,0 @@ - - -# hunter - -```go -import "github.com/brittonhayes/pillager/pkg/hunter" -``` - -Package hunter contains secret hunting and file scanning tools\. - -## Index - -- [type Config](<#type-config>) - - [func NewConfig(opts ...ConfigOption) *Config](<#func-newconfig>) -- [type ConfigOption](<#type-configoption>) - - [func WithFS(fs afero.Fs) ConfigOption](<#func-withfs>) - - [func WithFormat(reporter format.Reporter) ConfigOption](<#func-withformat>) - - [func WithGitleaksConfig(g config.Config) ConfigOption](<#func-withgitleaksconfig>) - - [func WithLogLevel(level string) ConfigOption](<#func-withloglevel>) - - [func WithRedact(redact bool) ConfigOption](<#func-withredact>) - - [func WithScanPath(path string) ConfigOption](<#func-withscanpath>) - - [func WithTemplate(template string) ConfigOption](<#func-withtemplate>) - - [func WithVerbose(verbose bool) ConfigOption](<#func-withverbose>) - - [func WithWorkers(count int) ConfigOption](<#func-withworkers>) -- [type Hunter](<#type-hunter>) - - [func New(opts ...ConfigOption) (*Hunter, error)](<#func-new>) - - [func (h *Hunter) Hunt() ([]report.Finding, error)](<#func-hunter-hunt>) - - [func (h *Hunter) Report(w io.Writer, findings []report.Finding) error](<#func-hunter-report>) - - -## type [Config]() - -Config takes all of the configurable parameters for a Hunter\. - -```go -type Config struct { - Filesystem afero.Fs - Reporter format.Reporter - Gitleaks config.Config - - ScanPath string - Verbose bool - Redact bool - Debug bool - Workers int - Template string -} -``` - -### func [NewConfig]() - -```go -func NewConfig(opts ...ConfigOption) *Config -``` - -NewConfig creates a Config instance\. - -## type [ConfigOption]() - -ConfigOption is a convenient type alias for func\(\*Config\)\. - -```go -type ConfigOption func(*Config) -``` - -### func [WithFS]() - -```go -func WithFS(fs afero.Fs) ConfigOption -``` - -### func [WithFormat]() - -```go -func WithFormat(reporter format.Reporter) ConfigOption -``` - -### func [WithGitleaksConfig]() - -```go -func WithGitleaksConfig(g config.Config) ConfigOption -``` - -### func [WithLogLevel]() - -```go -func WithLogLevel(level string) ConfigOption -``` - -### func [WithRedact]() - -```go -func WithRedact(redact bool) ConfigOption -``` - -### func [WithScanPath]() - -```go -func WithScanPath(path string) ConfigOption -``` - -### func [WithTemplate]() - -```go -func WithTemplate(template string) ConfigOption -``` - -### func [WithVerbose]() - -```go -func WithVerbose(verbose bool) ConfigOption -``` - -### func [WithWorkers]() - -```go -func WithWorkers(count int) ConfigOption -``` - -## type [Hunter]() - -Hunter is the secret scanner\. - -```go -type Hunter struct { - *Config -} -``` - -### func [New]() - -```go -func New(opts ...ConfigOption) (*Hunter, error) -``` - -New creates an instance of the Hunter\. - -### func \(\*Hunter\) [Hunt]() - -```go -func (h *Hunter) Hunt() ([]report.Finding, error) -``` - -Hunt walks over the filesystem at the configured path\, looking for sensitive information\. - -### func \(\*Hunter\) [Report]() - -```go -func (h *Hunter) Report(w io.Writer, findings []report.Finding) error -``` - -Report prints out the Findings in the preferred output format\. - - - -Generated by [gomarkdoc]() diff --git a/pkg/hunter/config.go b/pkg/hunter/config.go deleted file mode 100644 index 8c25925..0000000 --- a/pkg/hunter/config.go +++ /dev/null @@ -1,144 +0,0 @@ -package hunter - -import ( - "errors" - "os" - "runtime" - - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - - "github.com/brittonhayes/pillager/internal/validate" - - "github.com/brittonhayes/pillager/pkg/format" - "github.com/brittonhayes/pillager/pkg/rules" - - "github.com/zricethezav/gitleaks/v8/config" -) - -// Config takes all of the configurable parameters for a Hunter. -type Config struct { - Reporter format.Reporter - Gitleaks config.Config - ScanPath string - Verbose bool - Redact bool - Debug bool - Workers int - Template string -} - -// ConfigOption is a convenient type alias for func(*Config). -type ConfigOption func(*Config) - -// NewConfig creates a Config instance. -func NewConfig(opts ...ConfigOption) *Config { - var ( - defaultVerbose = false - defaultScanPath = "." - defaultReporter = format.JSON{} - defaultWorkers = runtime.NumCPU() - defaultGitleaks = rules.NewLoader().Load() - defaultTemplate = "" - defaultLogLevel = zerolog.ErrorLevel - ) - - zerolog.SetGlobalLevel(defaultLogLevel) - config := &Config{ - ScanPath: defaultScanPath, - Reporter: defaultReporter, - Workers: defaultWorkers, - Gitleaks: defaultGitleaks, - Verbose: defaultVerbose, - Template: defaultTemplate, - } - - for _, opt := range opts { - opt(config) - } - - if err := config.validate(); err != nil { - log.Fatal().Err(err).Send() - } - - return config -} - -func WithScanPath(path string) ConfigOption { - return func(c *Config) { - if validate.PathExists(path) { - c.ScanPath = path - return - } - - currentDir, err := os.Getwd() - if err != nil { - log.Fatal().Err(err).Msg("failed to get current dir") - } - - log.Error().Msgf("scan path %q not found, defaulting to %q", path, currentDir) - c.ScanPath = currentDir - } -} - -func WithLogLevel(level string) ConfigOption { - return func(c *Config) { - lvl, err := zerolog.ParseLevel(level) - if err != nil { - log.Fatal().Err(err).Send() - } - zerolog.SetGlobalLevel(lvl) - } -} - -func WithVerbose(verbose bool) ConfigOption { - return func(c *Config) { - c.Verbose = verbose - } -} - -func WithWorkers(count int) ConfigOption { - return func(c *Config) { - c.Workers = count - } -} - -func WithRedact(redact bool) ConfigOption { - return func(c *Config) { - c.Redact = redact - } -} - -func WithFormat(reporter format.Reporter) ConfigOption { - return func(c *Config) { - if c.Template != "" { - custom := &format.Custom{} - custom.WithTemplate(c.Template) - c.Reporter = custom - return - } - - c.Reporter = reporter - } -} - -func WithTemplate(template string) ConfigOption { - return func(c *Config) { - c.Reporter = format.Custom{} - c.Template = template - } -} - -func WithGitleaksConfig(g config.Config) ConfigOption { - return func(c *Config) { - c.Gitleaks = g - } -} - -func (c *Config) validate() error { - if c.Gitleaks.Rules == nil { - return errors.New("no gitleaks rules provided") - } - - return nil -} diff --git a/pkg/hunter/docs.go b/pkg/hunter/docs.go deleted file mode 100644 index 3e8881c..0000000 --- a/pkg/hunter/docs.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package hunter contains secret hunting and file scanning tools. -package hunter diff --git a/pkg/hunter/hunter.go b/pkg/hunter/hunter.go deleted file mode 100644 index ae99a6b..0000000 --- a/pkg/hunter/hunter.go +++ /dev/null @@ -1,51 +0,0 @@ -package hunter - -import ( - "io" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/zricethezav/gitleaks/v8/config" - "github.com/zricethezav/gitleaks/v8/detect" - "github.com/zricethezav/gitleaks/v8/report" -) - -// Hunter is the secret scanner. -type Hunter struct { - *Config -} - -// New creates an instance of the Hunter. -func New(opts ...ConfigOption) (*Hunter, error) { - return &Hunter{ - Config: NewConfig(opts...), - }, nil -} - -// Hunt walks over the filesystem at the configured path, looking for sensitive information. -func (h *Hunter) Hunt() ([]report.Finding, error) { - d, err := detect.NewDetectorDefaultConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to setup hunter") - } - - d.Verbose = h.Verbose - d.Redact = h.Redact - if h.Config != nil { - d.Config = config.Config{Allowlist: h.Gitleaks.Allowlist, Rules: h.Gitleaks.Rules} - } - - findings, err := d.DetectFiles(h.ScanPath) - if err != nil { - return nil, errors.Wrap(err, "failed to detect from files") - } - - log.Debug().Bool("verbose", h.Verbose).Msg("scanner created") - - return findings, nil -} - -// Report prints out the Findings in the preferred output format. -func (h *Hunter) Report(w io.Writer, findings []report.Finding) error { - return h.Reporter.Report(w, findings) -} diff --git a/pkg/report/format.go b/pkg/report/format.go new file mode 100644 index 0000000..d82f0a4 --- /dev/null +++ b/pkg/report/format.go @@ -0,0 +1,99 @@ +package report + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/brittonhayes/pillager" + "github.com/brittonhayes/pillager/internal/templates" + "gopkg.in/yaml.v2" +) + +type JSON struct{} + +func (j JSON) Report(w io.Writer, findings []pillager.Finding) error { + encoder := json.NewEncoder(w) + if err := encoder.Encode(&findings); err != nil { + return err + } + + return nil +} + +type JSONPretty struct{} + +func (j JSONPretty) Report(w io.Writer, findings []pillager.Finding) error { + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + if err := encoder.Encode(&findings); err != nil { + return err + } + + return nil +} + +type Raw struct{} + +func (r Raw) Report(w io.Writer, findings []pillager.Finding) error { + encoder := json.NewEncoder(w) + if err := encoder.Encode(&findings); err != nil { + return err + } + + return nil +} + +type YAML struct{} + +func (y YAML) Report(w io.Writer, findings []pillager.Finding) error { + b, err := yaml.Marshal(&findings) + if err != nil { + return err + } + fmt.Fprintf(w, "%s\n", string(b)) + + return nil +} + +type HTML struct{} + +func (h HTML) Report(w io.Writer, findings []pillager.Finding) error { + return Render(w, templates.HTML, findings) +} + +type HTMLTable struct{} + +func (h HTMLTable) Report(w io.Writer, findings []pillager.Finding) error { + return Render(w, templates.HTMLTable, findings) +} + +type Markdown struct{} + +func (m Markdown) Report(w io.Writer, findings []pillager.Finding) error { + return Render(w, templates.Markdown, findings) +} + +type Table struct{} + +func (t Table) Report(w io.Writer, findings []pillager.Finding) error { + return Render(w, templates.Table, findings) +} + +type Custom struct { + template string +} + +func (c *Custom) WithTemplate(t string) { + c.template = t +} + +func (c Custom) Report(w io.Writer, findings []pillager.Finding) error { + return Render(w, c.template, findings) +} + +type Simple struct{} + +func (s Simple) Report(w io.Writer, findings []pillager.Finding) error { + return Render(w, templates.DefaultTemplate, findings) +} diff --git a/pkg/report/report.go b/pkg/report/report.go new file mode 100644 index 0000000..90bf868 --- /dev/null +++ b/pkg/report/report.go @@ -0,0 +1,81 @@ +package report + +import ( + "io" + "strings" + "text/template" + + "encoding/json" + + "github.com/Masterminds/sprig" + "github.com/brittonhayes/pillager" + "github.com/brittonhayes/pillager/internal/templates" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +// Reporter is the interface that each of the canonical output formats implement. +type Reporter interface { + Report(io.Writer, []pillager.Finding) error +} + +// StringToReporter takes in a string representation of the preferred +// reporter. +func StringToReporter(s string) Reporter { + switch strings.ToLower(s) { + case "json": + return JSON{} + case "json-pretty": + return JSONPretty{} + case "raw": + return Raw{} + case "yaml": + return YAML{} + case "table": + return Table{} + case "html": + return HTML{} + case "html-table": + return HTMLTable{} + case "markdown": + return Markdown{} + case "custom": + return Custom{} + case "simple": + return Simple{} + default: + return JSON{} + } +} + +// Render renders a finding in a custom go template format to the provided writer. +func Render(w io.Writer, tpl string, findings []pillager.Finding) error { + t := template.New("custom") + if tpl == "" { + log.Debug().Msg("using default template") + tpl = templates.DefaultTemplate + } + + funcMap := sprig.TxtFuncMap() + funcMap["json"] = func(v interface{}) string { + b, err := json.Marshal(v) + if err != nil { + return "" + } + // Escape quotes and backslashes for HTML attributes + escaped := strings.ReplaceAll(string(b), `"`, `"`) + escaped = strings.ReplaceAll(escaped, `\`, `\\`) + return escaped + } + + t, err := t.Funcs(funcMap).Parse(tpl) + if err != nil { + return errors.Wrap(err, "failed to parse template") + } + + if err := t.Execute(w, findings); err != nil { + return errors.Wrap(err, "Failed to use custom template") + } + + return nil +} diff --git a/pkg/rules/README.md b/pkg/rules/README.md deleted file mode 100755 index 62ca9be..0000000 --- a/pkg/rules/README.md +++ /dev/null @@ -1,97 +0,0 @@ - - -# rules - -```go -import "github.com/brittonhayes/pillager/pkg/rules" -``` - -Package rules enables the parsing of Gitleaks rulesets\. - -## Index - -- [Constants](<#constants>) -- [Variables](<#variables>) -- [type Loader](<#type-loader>) - - [func NewLoader(opts ...LoaderOption) *Loader](<#func-newloader>) - - [func (l *Loader) Load() config.Config](<#func-loader-load>) - - [func (l *Loader) WithStrict() LoaderOption](<#func-loader-withstrict>) -- [type LoaderOption](<#type-loaderoption>) - - [func FromFile(file string) LoaderOption](<#func-fromfile>) - - -## Constants - -ErrReadConfig is the custom error message used if an error is encountered reading the gitleaks config\. - -```go -const ErrReadConfig = "Failed to read gitleaks config" -``` - -## Variables - -These strings contain default configs\. They are initialized at compile time via go:embed\. - -```go -var ( - //go:embed rules_simple.toml - RulesDefault string - - //go:embed rules_strict.toml - RulesStrict string -) -``` - -## type [Loader]() - -Loader represents a gitleaks config loader\. - -```go -type Loader struct { - // contains filtered or unexported fields -} -``` - -### func [NewLoader]() - -```go -func NewLoader(opts ...LoaderOption) *Loader -``` - -NewLoader creates a Loader instance\. - -### func \(\*Loader\) [Load]() - -```go -func (l *Loader) Load() config.Config -``` - -Load parses the gitleaks configuration\. - -### func \(\*Loader\) [WithStrict]() - -```go -func (l *Loader) WithStrict() LoaderOption -``` - -WithStrict enables more strict pillager scanning\. - -## type [LoaderOption]() - -LoaderOption sets a parameter for the gitleaks config loader\. - -```go -type LoaderOption func(*Loader) -``` - -### func [FromFile]() - -```go -func FromFile(file string) LoaderOption -``` - -FromFile decodes a gitleaks config from a local file\. - - - -Generated by [gomarkdoc]() diff --git a/pkg/rules/rules.go b/pkg/rules/rules.go deleted file mode 100644 index fdd5d08..0000000 --- a/pkg/rules/rules.go +++ /dev/null @@ -1,86 +0,0 @@ -// Package rules enables the parsing of Gitleaks rulesets. -package rules - -import ( - _ "embed" - - "github.com/BurntSushi/toml" - "github.com/brittonhayes/pillager/internal/validate" - "github.com/rs/zerolog/log" - - "github.com/zricethezav/gitleaks/v8/config" -) - -// ErrReadConfig is the custom error message used if an error is encountered -// reading the gitleaks config. -const ErrReadConfig = "Failed to read gitleaks config" - -// These strings contain default configs. They are initialized at compile time via go:embed. -var ( - RulesDefault = config.DefaultConfig - - //go:embed rules_strict.toml - RulesStrict string -) - -// Loader represents a gitleaks config loader. -type Loader struct { - loader config.ViperConfig -} - -// LoaderOption sets a parameter for the gitleaks config loader. -type LoaderOption func(*Loader) - -// NewLoader creates a Loader instance. -func NewLoader(opts ...LoaderOption) *Loader { - var loader Loader - if _, err := toml.Decode(RulesDefault, &loader.loader); err != nil { - log.Fatal().Err(err).Msg(ErrReadConfig) - } - - for _, opt := range opts { - opt(&loader) - } - - return &loader -} - -// WithStrict enables more strict pillager scanning. -func (l *Loader) WithStrict() LoaderOption { - return func(l *Loader) { - if _, err := toml.Decode(RulesStrict, &l.loader); err != nil { - log.Fatal().Err(err).Msg(ErrReadConfig) - } - } -} - -// Load parses the gitleaks configuration. -func (l *Loader) Load() config.Config { - config, err := l.loader.Translate() - if err != nil { - log.Fatal().Err(err).Msg(ErrReadConfig) - } - - return config -} - -// WithFile decodes a gitleaks config from a local file. -func WithFile(file string) LoaderOption { - return func(l *Loader) { - if file == "" { - if _, err := toml.Decode(config.DefaultConfig, &l.loader); err != nil { - log.Fatal().Err(err).Msg(ErrReadConfig) - } - return - } - - if validate.PathExists(file) { - if _, err := toml.DecodeFile(file, &l.loader); err != nil { - log.Fatal().Err(err).Msg(ErrReadConfig) - } - return - } - - log.Fatal().Msgf("invalid - rules file '%s' does not exist", file) - } -} diff --git a/pkg/rules/rules_simple.toml b/pkg/rules/rules_simple.toml deleted file mode 100644 index f0fc62d..0000000 --- a/pkg/rules/rules_simple.toml +++ /dev/null @@ -1,29 +0,0 @@ -title = "pillager config" -[[rules]] - description = "AWS Access Key" - regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' - tags = ["key", "AWS"] -[[rules]] - description = "AWS Secret Key" - regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]''' - tags = ["key", "AWS"] -[[rules]] - description = "Github" - regex = '''(?i)github(.{0,20})?(?-i)[0-9a-zA-Z]{35,40}''' - tags = ["key", "Github"] -[[rules]] - description = "Slack" - regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?''' - tags = ["key", "Slack"] -[[rules]] - description = "Asymmetric Private Key" - regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----''' - tags = ["key", "AsymmetricPrivateKey"] -[[rules]] - description = "Slack Webhook" - regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}''' - tags = ["key", "slack"] - -[allowlist] - description = "Allowlisted files" - files = ['''^\.?gitleaks.toml$''', '''(.*?)(png|jpg|gif|doc|docx|pdf|bin|xls|pyc|zip)$''', '''(go.mod|go.sum)$'''] diff --git a/pkg/rules/rules_strict.toml b/pkg/rules/rules_strict.toml deleted file mode 100644 index 32d4a0c..0000000 --- a/pkg/rules/rules_strict.toml +++ /dev/null @@ -1,226 +0,0 @@ -title = "Global gitleaks config" - -[allowlist] - description = "Allowlisted files" - files = ['''^\.?gitleaks.toml$''', # Ignoring this file - '''(.*?)(jpg|gif|doc|pdf|bin)$''', # Ignoring common binaries - '''^(.*?)_test\.go$''', # Ignoring Go test files - '''^(.*?)\.(spec|test)\.(j|t)s$''', # Ignoring JavaScript and TypeScript test files - '''(go.mod|go.sum)$''', # Ignoring Go manifests - '''vendor\.json''', - '''Gopkg\.(lock|toml)''', - '''package-lock\.json''', # Ignoring Node/JS manifests - '''package\.json''', - '''composer\.json''', - '''composer\.lock''', #Ignoring PHP manifests - '''yarn\.lock'''] - paths = ["node_modules", # Ignoring Node dependencies - "vendor", # Ignoring Go dependencies - "test", # Ignoring test directories - "tests"] - regexes = ['''test'''] # Ignoring lines with test - - -[[rules]] - description = "AWS Secret Key" - regex = '''(?i)aws(.{0,20})?(?-i)[0-9a-zA-Z\/+]{40}''' - tags = ["key", "AWS"] - -[[rules]] - description = "AWS MWS key" - regex = '''amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''' - tags = ["key", "AWS", "MWS"] - -[[rules]] - description = "Facebook Secret Key" - regex = '''(?i)(facebook|fb)(.{0,20})?(?-i)[0-9a-f]{32}''' - tags = ["key", "Facebook"] - -[[rules]] - description = "Facebook Client ID" - regex = '''(?i)(facebook|fb)(.{0,20})?[0-9]{13,17}''' - tags = ["key", "Facebook"] - -[[rules]] - description = "Twitter Secret Key" - regex = '''(?i)twitter(.{0,20})?[0-9a-z]{35,44}''' - tags = ["key", "Twitter"] - -[[rules]] - description = "Twitter Client ID" - regex = '''(?i)twitter(.{0,20})?[0-9a-z]{18,25}''' - tags = ["client", "Twitter"] - -[[rules]] - description = "Github" - regex = '''(?i)github(.{0,20})?(?-i)[0-9a-zA-Z]{35,40}''' - tags = ["key", "Github"] - -[[rules]] - description = "LinkedIn Client ID" - regex = '''(?i)linkedin(.{0,20})?(?-i)[0-9a-z]{12}''' - tags = ["client", "LinkedIn"] - -[[rules]] - description = "LinkedIn Secret Key" - regex = '''(?i)linkedin(.{0,20})?[0-9a-z]{16}''' - tags = ["secret", "LinkedIn"] - -[[rules]] - description = "Slack" - regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?''' - tags = ["key", "Slack"] - -[[rules]] - description = "Asymmetric Private Key" - regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----''' - tags = ["key", "AsymmetricPrivateKey"] - -[[rules]] - description = "Google API key" - regex = '''AIza[0-9A-Za-z\\-_]{35}''' - tags = ["key", "Google"] - -[[rules]] - description = "Google (GCP) Service Account" - regex = '''"type": "service_account"''' - tags = ["key", "Google"] - -[[rules]] - description = "Heroku API key" - regex = '''(?i)heroku(.{0,20})?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''' - tags = ["key", "Heroku"] - -[[rules]] - description = "MailChimp API key" - regex = '''(?i)(mailchimp|mc)(.{0,20})?[0-9a-f]{32}-us[0-9]{1,2}''' - tags = ["key", "Mailchimp"] - -[[rules]] - description = "Mailgun API key" - regex = '''((?i)(mailgun|mg)(.{0,20})?)?key-[0-9a-z]{32}''' - tags = ["key", "Mailgun"] - -[[rules]] - description = "PayPal Braintree access token" - regex = '''access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}''' - tags = ["key", "Paypal"] - -[[rules]] - description = "Picatic API key" - regex = '''sk_live_[0-9a-z]{32}''' - tags = ["key", "Picatic"] - -[[rules]] - description = "SendGrid API Key" - regex = '''SG\.[\w_]{16,32}\.[\w_]{16,64}''' - tags = ["key", "SendGrid"] - -[[rules]] - description = "Slack Webhook" - regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}''' - tags = ["key", "slack"] - -[[rules]] - description = "Stripe API key" - regex = '''(?i)stripe(.{0,20})?[sr]k_live_[0-9a-zA-Z]{24}''' - tags = ["key", "Stripe"] - -[[rules]] - description = "Square access token" - regex = '''sq0atp-[0-9A-Za-z\-_]{22}''' - tags = ["key", "square"] - -[[rules]] - description = "Square OAuth secret" - regex = '''sq0csp-[0-9A-Za-z\\-_]{43}''' - tags = ["key", "square"] - -[[rules]] - description = "Twilio API key" - regex = '''(?i)twilio(.{0,20})?SK[0-9a-f]{32}''' - tags = ["key", "twilio"] - -# The following rules check for credentials assigned to variables that its value has an entropy of more than 3 bits. -# To achieve this there's a regexp for each language. The regexp checks for a variable with a suspicious name followed -# by a value assignation (for example, := in Go, = in JS, etc.). Then, looks for a group of non-space characters enclosed -# between quotes. If that group has an entropy higher than 3 bits the rule will trigger. - -[[rules]] - description = "Hardcoded credentials in Go files" - file = '''^(.*?)\.go$''' - regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:\w|\s*?)(?:=|:=)(?:\s*?)[\"'`](.{4,120}?)[\"'`]''' - tags = ["credentials", "hardcoded", "go"] - [[rules.Entropies]] - Min = "3" - Max = "7" - Group = "1" - -[[rules]] - description = "Hardcoded credentials in JavaScript or TypeScript files" - file = '''^(.*?)\.(?:j|t)s$''' - regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:\w|\s*?)(?:=){1}(?:\s{0,10})[\"'`](.*?)[\"'`]''' - tags = ["credentials", "hardcoded", "js"] - [[rules.Entropies]] - Min = "3" - Max = "7" - Group = "1" - -[[rules]] - description = "Hardcoded credentials in PHP files" - file = '''^(.*?)\.php$''' - regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?:=){1}(?:.{0,10})[\"'`](.{4,120})[\"'`]''' - tags = ["credentials", "hardcoded", "php"] - [[rules.Entropies]] - Min = "3" - Max = "7" - Group = "1" - -[[rules]] - description = "Hardcoded credentials in YAML files as quoted strings" - file = '''^(.*?)\.y(a|)ml$''' - regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?::){1}(?:\s{0,10})(?:[\"'](.{4,120})[\"'])''' - tags = ["credentials", "hardcoded", "yaml"] - [[rules.Entropies]] - Min = "3" - Max = "7" - Group = "1" - [rules.allowlist] - description = "Skip YAML Serverless variables, grabbed and concated values, encrypted secrets, and values with jinja2 placeholders" - regexes = ['''\${(?:.)+}''', '''(?i)\(\((?:\s)*?(?:grab|concat)(?:.)*?(?:\s)*?\)\)''', '''(?i)!!enveloped:(?:\S)+''', '''(?:.)*?{{(?:.)*?}}'''] - -[[rules]] - description = "Hardcoded credentials in YAML files as unquoted strings" - file = '''^(.*?)\.y(a|)ml$''' - regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?::){1}(?:\s{0,10})(\S{4,120})''' - tags = ["credentials", "hardcoded", "yaml"] - [[rules.Entropies]] - Min = "3.5" # A higher entropy is required for this type of match, as unquoted can trigger many false positives - Max = "7" - Group = "1" - [rules.allowlist] - description = "Skip YAML Serverless variables, grabbed and concated values, encrypted secrets, and values with jinja2 placeholders" - regexes = ['''\${(?:.)+}''', '''(?i)\(\((?:\s)*?(?:grab|concat)(?:.)*?(?:\s)*?\)\)''', '''(?i)!!enveloped:(?:\S)+''', '''(?:.)*?{{(?:.)*?}}'''] - -[[rules]] - description = "Hardcoded credentials in YAML files as multiline strings" - file = '''^(.*?)\.y(a|)ml$''' - regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?::){1}(?:\s{0,10})(?:\|(?:-|))\n(?:\s{0,10})(\S{4,120})''' - tags = ["credentials", "hardcoded", "yaml"] - [[rules.Entropies]] - Min = "4" - Max = "7" - Group = "1" - -[[rules]] - description = "Hardcoded credentials in HCL files (*.tf)" - file = '''^(.*?)\.tf$''' - regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?:=){1}(?:\s)*?"(.{4,120})"''' - tags = ["credentials", "hardcoded", "hcl"] - [[rules.Entropies]] - Min = "3" - Max = "7" - Group = "1" - [rules.allowlist] - description = "Skip variable substitution" - regexes = ['''\${(?:.)*?}'''] diff --git a/pkg/scanner/config.go b/pkg/scanner/config.go new file mode 100644 index 0000000..8c6dac2 --- /dev/null +++ b/pkg/scanner/config.go @@ -0,0 +1,138 @@ +package scanner + +import ( + "fmt" + "path/filepath" + "runtime" + + "github.com/BurntSushi/toml" + "github.com/brittonhayes/pillager" + "github.com/pkg/errors" + "github.com/spf13/viper" + "github.com/zricethezav/gitleaks/v8/config" +) + +// ConfigLoader handles loading configuration from files +type ConfigLoader struct { + v *viper.Viper +} + +// NewConfigLoader creates a new configuration loader +func NewConfigLoader() *ConfigLoader { + v := viper.New() + v.SetDefault("verbose", false) + v.SetDefault("path", ".") + v.SetDefault("template", "") + v.SetDefault("workers", runtime.NumCPU()) + v.SetDefault("redact", false) + v.SetDefault("reporter", "json") + + return &ConfigLoader{v: v} +} + +func convertDefaultConfig() (*pillager.Options, error) { + var defaultConfig config.Config + + if err := toml.Unmarshal([]byte(config.DefaultConfig), &defaultConfig); err != nil { + return nil, errors.Wrap(err, "failed to parse default rules") + } + + var opts pillager.Options + + opts.Rules = gitleaksToPillagerRules(defaultConfig.Rules) + opts.Allowlist = gitleaksToPillagerAllowlist(defaultConfig.Allowlist) + + return &opts, nil +} + +// LoadConfig attempts to load configuration from a file +func (c *ConfigLoader) LoadConfig(configPath string) (*pillager.Options, error) { + + // Rules and allowlist defaults + defaultConfig, err := convertDefaultConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to convert default rules") + } + + c.v.SetDefault("rules", defaultConfig.Rules) + c.v.SetDefault("allowlist.paths", defaultConfig.Allowlist.Paths) + c.v.SetDefault("allowlist.regexes", defaultConfig.Allowlist.Regexes) + + if configPath != "" { + // Use specified config file + ext := filepath.Ext(configPath) + if ext != ".toml" { + return nil, fmt.Errorf("config file must have an extension .toml") + } + c.v.SetConfigType(ext[1:]) + c.v.SetConfigFile(configPath) + + if err := c.v.ReadInConfig(); err != nil { + return nil, fmt.Errorf("failed to read config file %s: %w", configPath, err) + } + } else { + // Search for config in default locations + c.v.SetConfigName("pillager") + c.v.AddConfigPath(".") + c.v.AddConfigPath("$HOME/.pillager") + c.v.AddConfigPath("$HOME/.config/pillager") + + c.v.ReadInConfig() + } + + // Create options from config + opts := &pillager.Options{ + Path: c.v.GetString("path"), + Workers: c.v.GetInt("workers"), + Verbose: c.v.GetBool("verbose"), + Template: c.v.GetString("template"), + Redact: c.v.GetBool("redact"), + Reporter: c.v.GetString("reporter"), + Allowlist: pillager.Allowlist{ + Paths: c.v.GetStringSlice("allowlist.paths"), + Regexes: c.v.GetStringSlice("allowlist.regexes"), + }, + } + + // Unmarshal rules if they exist + var rules []pillager.Rule + if err := c.v.UnmarshalKey("rules", &rules); err != nil { + return nil, fmt.Errorf("failed to parse rules configuration: %w", err) + } + opts.Rules = rules + + return opts, nil +} + +// MergeWithFlags merges configuration with command line flags +func (c *ConfigLoader) MergeWithFlags(opts *pillager.Options, flags *pillager.Options) { + if flags.Verbose { + opts.Verbose = flags.Verbose + } + if flags.Redact { + opts.Redact = flags.Redact + } + if flags.Workers > 0 { + opts.Workers = flags.Workers + } + if flags.Reporter != "" { + opts.Reporter = flags.Reporter + } + if flags.Template != "" { + opts.Template = flags.Template + } + if flags.Path != "" { + opts.Path = flags.Path + } + // Merge rules if provided + if len(flags.Rules) > 0 { + opts.Rules = flags.Rules + } + // Merge allowlist if provided + if len(flags.Allowlist.Paths) > 0 { + opts.Allowlist.Paths = flags.Allowlist.Paths + } + if len(flags.Allowlist.Regexes) > 0 { + opts.Allowlist.Regexes = flags.Allowlist.Regexes + } +} diff --git a/pkg/scanner/gitleaks.go b/pkg/scanner/gitleaks.go new file mode 100644 index 0000000..00b09eb --- /dev/null +++ b/pkg/scanner/gitleaks.go @@ -0,0 +1,183 @@ +package scanner + +import ( + "regexp" + + "github.com/brittonhayes/pillager" + "github.com/brittonhayes/pillager/pkg/report" + "github.com/pkg/errors" + + "github.com/rs/zerolog/log" + "github.com/zricethezav/gitleaks/v8/config" + "github.com/zricethezav/gitleaks/v8/detect" + output "github.com/zricethezav/gitleaks/v8/report" +) + +// GitleaksScanner implements the Scanner interface using gitleaks +type GitleaksScanner struct { + detector *detect.Detector + reporter string +} + +// NewGitleaksScanner creates a new scanner using gitleaks +func NewGitleaksScanner(options pillager.Options) (Scanner, error) { + scanner := &GitleaksScanner{ + reporter: options.Reporter, + } + + cfg := config.Config{ + Allowlist: convertToGitleaksAllowlist(options.Allowlist), + Rules: pillagerToGitleaksRules(options.Rules), + Path: options.Path, + } + + scanner.detector = detect.NewDetector(cfg) + scanner.detector.Verbose = options.Verbose + scanner.detector.Redact = options.Redact + + return scanner, nil +} + +// Reporter returns the reporter for the scanner +func (g *GitleaksScanner) Reporter() report.Reporter { + return report.StringToReporter(g.reporter) +} + +// ScanPath returns the path that the scanner is scanning +func (g *GitleaksScanner) ScanPath() string { + return g.detector.Config.Path +} + +func (g *GitleaksScanner) Translate(f output.Finding) pillager.Finding { + finding := pillager.Finding{ + Description: f.Description, + StartLine: f.StartLine, + EndLine: f.EndLine, + StartColumn: f.StartColumn, + EndColumn: f.EndColumn, + Match: f.Match, + Secret: f.Secret, + File: f.File, + Entropy: f.Entropy, + RuleID: f.RuleID, + } + return finding +} + +// Scan implements the Scanner interface +func (g *GitleaksScanner) Scan() ([]pillager.Finding, error) { + findings, err := g.detector.DetectFiles(g.ScanPath()) + if err != nil { + return nil, errors.Wrap(err, "failed to scan files") + } + + var pillagerFindings []pillager.Finding + for _, f := range findings { + pillagerFindings = append(pillagerFindings, g.Translate(f)) + } + + if len(pillagerFindings) == 0 { + return pillagerFindings, nil + } + + return pillagerFindings, nil +} + +func convertToGitleaksAllowlist(a pillager.Allowlist) config.Allowlist { + paths := []*regexp.Regexp{} + for _, path := range a.Paths { + p, err := regexp.Compile(path) + if err != nil { + log.Fatal().Err(err). + Str("pattern", path). + Msg("failed to compile allowlist path regex") + } + paths = append(paths, p) + } + + regexes := []*regexp.Regexp{} + for _, regex := range a.Regexes { + r, err := regexp.Compile(regex) + if err != nil { + log.Fatal().Err(err). + Str("pattern", regex). + Msg("failed to compile allowlist regex pattern") + } + regexes = append(regexes, r) + } + + return config.Allowlist{ + Paths: paths, + Regexes: regexes, + } +} + +func gitleaksToPillagerAllowlist(a config.Allowlist) pillager.Allowlist { + paths := []string{} + for _, path := range a.Paths { + paths = append(paths, path.String()) + } + + regexes := []string{} + for _, regex := range a.Regexes { + regexes = append(regexes, regex.String()) + } + + return pillager.Allowlist{Paths: paths, Regexes: regexes} +} + +func gitleaksToPillagerRules(rules []*config.Rule) []pillager.Rule { + converted := []pillager.Rule{} + for _, rule := range rules { + r := gitleaksToPillagerRule(rule) + converted = append(converted, r) + } + return converted +} + +func gitleaksToPillagerRule(rule *config.Rule) pillager.Rule { + return pillager.Rule{ + ID: rule.RuleID, + Description: rule.Description, + Regex: rule.Regex.String(), + Tags: rule.Tags, + Allowlist: gitleaksToPillagerAllowlist(rule.Allowlist), + } +} + +func pillagerToGitleaksRules(rules []pillager.Rule) []*config.Rule { + converted := []*config.Rule{} + for _, rule := range rules { + r := pillagerToGitleaksRule(rule) + converted = append(converted, r) + } + return converted +} + +func pillagerToGitleaksRule(rule pillager.Rule) *config.Rule { + path, err := regexp.Compile(rule.Path) + if err != nil { + log.Fatal().Err(err). + Str("rule_id", rule.ID). + Str("pattern", rule.Path). + Msg("failed to compile rule path regex") + } + + regex, err := regexp.Compile(rule.Regex) + if err != nil { + log.Fatal().Err(err). + Str("rule_id", rule.ID). + Str("pattern", rule.Regex). + Msg("failed to compile rule regex pattern") + } + + return &config.Rule{ + RuleID: rule.ID, + Path: path, + Description: rule.Description, + Regex: regex, + Keywords: rule.Keywords, + Tags: rule.Tags, + Allowlist: convertToGitleaksAllowlist(rule.Allowlist), + } +} diff --git a/pkg/scanner/scanner.go b/pkg/scanner/scanner.go new file mode 100644 index 0000000..1952c26 --- /dev/null +++ b/pkg/scanner/scanner.go @@ -0,0 +1,18 @@ +package scanner + +import ( + "github.com/brittonhayes/pillager" + "github.com/brittonhayes/pillager/pkg/report" +) + +// Scanner defines the interface for secret scanning implementations +type Scanner interface { + // Scan performs the secret scanning operation on the given path + Scan() ([]pillager.Finding, error) + + // Reporter returns the reporter for the scanner + Reporter() report.Reporter + + // ScanPath returns the path that the scanner is scanning + ScanPath() string +} diff --git a/pkg/templates/html-table.tmpl b/pkg/templates/html-table.tmpl deleted file mode 100644 index 5a704cb..0000000 --- a/pkg/templates/html-table.tmpl +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - Pillager - Scan Results - - - -
-
-

- Pillager -

-

- Results of your latest hunt -

-
- - - - - - - - - - {{ range .Leaks }} - - - - - - {{ end }} - -
FileLineLeak
{{.File}}{{.LineNumber}}{{.Offender}}
-
-
-
- - - diff --git a/pkg/templates/html.tmpl b/pkg/templates/html.tmpl deleted file mode 100644 index ed88a9b..0000000 --- a/pkg/templates/html.tmpl +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - Pillager - Scan Results - - - -
-
-

- Pillager -

-

- Results of your latest hunt -

-
- {{ range . }} -
-
-

{{.File}}

-

- Tags: - - {{.Tags}} - -

-

- Leak: - {{.Secret}} -

-

- Line: - {{.StartLine}} -

-
-
- {{end}} -
-
-
- - diff --git a/pkg/templates/markdown.tmpl b/pkg/templates/markdown.tmpl deleted file mode 100644 index 45a75f8..0000000 --- a/pkg/templates/markdown.tmpl +++ /dev/null @@ -1,6 +0,0 @@ -# Results - -{{ range . -}} - ## {{ .File }} - - Location: {{.StartLine}} -{{end}} diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go deleted file mode 100644 index 9bfd622..0000000 --- a/pkg/templates/templates.go +++ /dev/null @@ -1,60 +0,0 @@ -// Package templates contains a compilation of go templates for rendering secret findings. -package templates - -import ( - _ "embed" - "io" - "text/template" - - "github.com/Masterminds/sprig" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/zricethezav/gitleaks/v8/report" -) - -var ( - //go:embed simple.tmpl - Simple string - - //go:embed html.tmpl - HTML string - - //go:embed markdown.tmpl - Markdown string - - //go:embed table.tmpl - Table string - - //go:embed html-table.tmpl - HTMLTable string -) - -// DefaultTemplate is the base template used to format a Finding into the -// custom output format. -const DefaultTemplate = `{{ with . -}} -{{ range . -}} -Line: {{ quote .StartLine}} -File: {{ quote .File }} -Secret: {{ quote .Secret }} ---- -{{ end -}}{{- end}}` - -// Render renders a finding in a custom go template format to the provided writer. -func Render(w io.Writer, tpl string, findings []report.Finding) error { - t := template.New("custom") - if tpl == "" { - log.Debug().Msg("using default template") - tpl = DefaultTemplate - } - - t, err := t.Funcs(sprig.TxtFuncMap()).Parse(tpl) - if err != nil { - return errors.Wrap(err, "failed to parse template") - } - - if err := t.Execute(w, findings); err != nil { - return errors.Wrap(err, "Failed to use custom template") - } - - return nil -} diff --git a/pkg/tui/model/model.go b/pkg/tui/model/model.go index a60867e..45edf59 100644 --- a/pkg/tui/model/model.go +++ b/pkg/tui/model/model.go @@ -4,13 +4,13 @@ import ( "fmt" "os" - "github.com/brittonhayes/pillager/pkg/hunter" + "github.com/brittonhayes/pillager" + "github.com/brittonhayes/pillager/pkg/scanner" "github.com/brittonhayes/pillager/pkg/tui/style" "github.com/charmbracelet/bubbles/help" "github.com/charmbracelet/bubbles/spinner" tea "github.com/charmbracelet/bubbletea" "github.com/evertras/bubble-table/table" - "github.com/zricethezav/gitleaks/v8/report" "golang.org/x/term" ) @@ -22,8 +22,8 @@ type model struct { help help.Model table table.Model - hunter *hunter.Hunter - results []report.Finding + scanner scanner.Scanner + results []pillager.Finding err error width int @@ -51,15 +51,15 @@ type body struct { message string } -type resultsMsg struct{ results []report.Finding } +type resultsMsg struct{ results []pillager.Finding } type errMsg struct{ err error } func (e errMsg) Error() string { - return fmt.Sprintf("🔥 Uh oh! Well that's not good. Looks like something went wrong and the application has exited: \n\n%s\n\n%s", style.Error.Render(e.Error()), "Press [q] to quit.") + return fmt.Sprintf("🔥 Uh oh! Well that's not good. Looks like something went wrong and the application has exited: \n\n%s\n\n%s", style.Error.Render(e.err.Error()), "Press [q] to quit.") } -func NewModel(hunt *hunter.Hunter) model { +func NewModel(scan scanner.Scanner) model { width, height, err := term.GetSize(int(os.Stdout.Fd())) if err != nil { width = 80 @@ -76,7 +76,7 @@ func NewModel(hunt *hunter.Hunter) model { h := help.New() m := model{ - hunter: hunt, + scanner: scan, header: header{ title: "Pillager", subtitle: "Hunt inside the file system for valuable information", @@ -102,11 +102,9 @@ func (m model) Dimensions() (int, int) { return m.width, m.height } -func startScan(h *hunter.Hunter) tea.Cmd { +func startScan(s scanner.Scanner) tea.Cmd { return func() tea.Msg { - h.Debug = false - h.Verbose = false - results, err := h.Hunt() + results, err := s.Scan() if err != nil { // There was an error making our request. Wrap the error we received // in a message and return it. diff --git a/pkg/tui/model/table.go b/pkg/tui/model/table.go index 855eaf2..aa2dcf6 100644 --- a/pkg/tui/model/table.go +++ b/pkg/tui/model/table.go @@ -4,9 +4,9 @@ import ( "path/filepath" "strings" + "github.com/brittonhayes/pillager" "github.com/brittonhayes/pillager/pkg/tui/style" "github.com/evertras/bubble-table/table" - "github.com/zricethezav/gitleaks/v8/report" ) const ( @@ -35,7 +35,7 @@ func newTable(width int) table.Model { return t } -func addRowData(data []report.Finding) []table.Row { +func addRowData(data []pillager.Finding) []table.Row { rows := []table.Row{} for i, entry := range data { diff --git a/pkg/tui/model/view.go b/pkg/tui/model/view.go index 5a38362..72dc921 100644 --- a/pkg/tui/model/view.go +++ b/pkg/tui/model/view.go @@ -4,11 +4,11 @@ import ( "fmt" "strings" + "github.com/brittonhayes/pillager" "github.com/brittonhayes/pillager/pkg/tui/style" "github.com/charmbracelet/bubbles/key" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" - "github.com/zricethezav/gitleaks/v8/report" ) func (m model) Init() tea.Cmd { @@ -57,7 +57,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case key.Matches(msg, m.keymap.Start): m.loading.active = true - return m, tea.Batch(startScan(m.hunter), m.loading.spinner.Tick) + return m, tea.Batch(startScan(m.scanner), m.loading.spinner.Tick) default: return m, m.loading.spinner.Tick } @@ -69,7 +69,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { func (m model) selectedView() string { if m.results != nil && m.body.selected.visible { w := &strings.Builder{} - err := m.hunter.Report(w, []report.Finding{m.selectedRow()}) + err := m.scanner.Reporter().Report(w, []pillager.Finding{m.selectedRow()}) if err != nil { m.err = err } @@ -80,7 +80,7 @@ func (m model) selectedView() string { return "" } -func (m model) selectedRow() report.Finding { +func (m model) selectedRow() pillager.Finding { return m.results[m.table.HighlightedRow().Data[columnKeyID].(int)-1] } @@ -91,9 +91,9 @@ func (m model) View() string { m.body.toast = "" if m.loading.active || m.loading.spinner.Visible() { - m.body.message = fmt.Sprintf("%s Scanning for secrets in %q with %d workers", m.loading.spinner.View(), m.hunter.ScanPath, m.hunter.Workers) + m.body.message = fmt.Sprintf("%s Scanning for secrets in %q", m.loading.spinner.View(), m.scanner.ScanPath()) } else if m.results != nil { - m.body.toast = style.Highlight.Render(fmt.Sprintf("Found %d secrets in path %q", len(m.results), m.hunter.ScanPath)) + m.body.toast = style.Highlight.Render(fmt.Sprintf("Found %d secrets in path %q", len(m.results), m.scanner.ScanPath())) m.body.message = m.table.View() m.body.message += m.selectedView() }