diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f1b219b..202ae23 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,4 +3,4 @@ updates: - package-ecosystem: "gomod" directory: "/" schedule: - interval: "weekly" + interval: "monthly" diff --git a/.github/workflows/build.yml b/.github/workflows/gobuild.yml similarity index 57% rename from .github/workflows/build.yml rename to .github/workflows/gobuild.yml index 4c8e56d..9efdc61 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/gobuild.yml @@ -1,6 +1,6 @@ name: build -on: [ pull_request ] +on: [pull_request] jobs: goreleaser: @@ -16,9 +16,5 @@ jobs: go-version: 1.17 - name: Build executables - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: build --rm-dist --skip-validate - env: - GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} + run: | + go build cmd/pillager/main.go diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index aa01166..04e267b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,6 +1,6 @@ name: reviewdog -on: [ pull_request ] +on: [pull_request] jobs: reviewdog: @@ -10,11 +10,4 @@ jobs: uses: actions/checkout@v1 - name: golangci-lint - uses: reviewdog/action-golangci-lint@v1 - - - name: misspell - uses: reviewdog/action-golangci-lint@v1 - with: - golangci_lint_flags: "--disable-all -E misspell" - tool_name: misspell - level: info + uses: reviewdog/action-golangci-lint@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 958073b..5cd2b6e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: test -on: [ pull_request ] +on: [pull_request] jobs: test: diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..7307ed4 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,15 @@ +linters: + enable: + - gofumpt + - gofmt + - gosimple + - godot + - godox + - dupl + - exhaustive + - funlen + - gocritic + - goprintffuncname + - ifshort + presets: + - unused diff --git a/.goreleaser.yml b/.goreleaser.yml index 7402bc5..c3e7545 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -9,10 +9,12 @@ builds: - windows - darwin ldflags: - - -s -w -X github.com/brittonhayes/pillager/cmd/pillager.version={{.Version}} + - -s -w -X github.com/brittonhayes/pillager/internal/commands/version.version={{.Version}} - -X main.commit={{.Commit}} - -X main.date={{.Date}} - -X main.builtBy=goreleaser + main: "./cmd/pillager" + archives: - replacements: darwin: Darwin @@ -21,17 +23,17 @@ archives: 386: i386 amd64: x86_64 checksum: - name_template: 'checksums.txt' + name_template: "checksums.txt" snapshot: name_template: "{{ .Tag }}-next" changelog: sort: asc filters: exclude: - - '^docs:' - - '^test:' - - '^images:' - - '^_examples:' + - "^docs:" + - "^test:" + - "^images:" + - "^_examples:" scoop: # Template for the url which is determined by the given Token (github or gitlab) url_template: "https://github.com/brittonhayes/pillager/releases/download/{{ .Tag }}/{{ .ArtifactName }}" @@ -64,7 +66,7 @@ brews: # GOARM to specify which 32-bit arm version to use if there are multiple versions # from the build section. Brew formulas support atm only one 32-bit version. # Default is 6 for all artifacts or each id if there a multiple versions. - goarm: 6 + goarm: "6" # GitHub/GitLab repository to push the formula to # Gitea is not supported yet, but the support coming diff --git a/Dockerfile b/Dockerfile index a5daad8..2e3b189 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.16-alpine AS build WORKDIR /src/ COPY . /src/ -RUN CGO_ENABLED=0 go build -o /bin/pillager +RUN CGO_ENABLED=0 go build cmd/pillager/main.go -o /bin/pillager FROM scratch as prod diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..031d5b5 --- /dev/null +++ b/Makefile @@ -0,0 +1,3 @@ +.PHONY: +lint: + golangci-lint run ./... \ No newline at end of file diff --git a/README.md b/README.md index 4b32074..4e86a26 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Pillager -![Image](./images/brand_image_ice.png) +![Image](./logo.png) [![Go Reference](https://pkg.go.dev/badge/github.com/brittonhayes/pillager.svg)](https://pkg.go.dev/github.com/brittonhayes/pillager) @@ -52,7 +52,7 @@ brew install pillager ### Docker Image ``` -docker run --rm -it bjhayes/pillager hunt . +docker run --rm -it bjhayes/pillager hunt . ``` If you're looking for a binary, check the latest releases for the executable that matches your system @@ -111,9 +111,7 @@ pillager hunt . pillager hunt ./example -f json | jq . ``` -> *JSON output is designed to work seamlessly with* -> *the amazing [jq](https://github.com/stedolan/jq)* -> *utility for easy parsing.* +> _JSON output is designed to work seamlessly with_ > _the amazing [jq](https://github.com/stedolan/jq)_ > _utility for easy parsing._
Click to view more output formats @@ -163,7 +161,6 @@ pillager hunt . -t "$(cat templates/simple.tmpl)"
- ### Custom Templates Pillager allows you to use powerful `go text/template` to customize the output format. Here are a few template examples. @@ -209,7 +206,7 @@ awesome [gomarkdoc](https://github.com/princjef/gomarkdoc) tool. **What is Cobra?** -> Cobra is a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. +> Cobra is a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. > Cobra is also an application that will generate your application scaffolding to rapidly develop a Cobra-based application. If you've seen a CLI written in Go before, there's a pretty high chance it was built with Cobra. I can't recommend this @@ -245,5 +242,5 @@ Check out the included [rules.toml](./rules.toml) for a baseline ruleset. > by the MITRE ATT&CK framework. > > [MITRE ATT&CK Technique - T1552,003 - Unsecured Credentials: Bash History ](https://attack.mitre.org/techniques/T1552/003/) -> +> > [MITRE ATT&CK Technique - T1552,001 - Unsecured Credentials: Credentials In Files](https://attack.mitre.org/techniques/T1552/001/) diff --git a/_examples/hunter/main.go b/_examples/hunter/main.go index 51a9d0e..a2feaba 100644 --- a/_examples/hunter/main.go +++ b/_examples/hunter/main.go @@ -1,20 +1,42 @@ package main import ( - "runtime" + "os" + "github.com/brittonhayes/pillager" "github.com/brittonhayes/pillager/pkg/hunter" - "github.com/brittonhayes/pillager/pkg/rules" - "github.com/spf13/afero" + "github.com/rs/zerolog" ) func main() { - // Create a new hunter config - c := hunter.NewConfig(afero.NewOsFs(), ".", true, rules.Load(""), hunter.StringToFormat("JSON"), "", runtime.NumCPU()) + err := example() + if err != nil { + panic(err) + } +} + +func example() error { + opts := []pillager.ConfigOption{ + pillager.WithLogLevel(zerolog.DebugLevel), + } - // Create a new hunter from the config - h := hunter.NewHunter(c) + // Create a new hunter config + p, err := hunter.New(opts...) + if err != nil { + return err + } // Start hunting - _ = h.Hunt() + results, err := p.Hunt() + if err != nil { + return err + } + + // Report results + err = p.Report(os.Stdout, results) + if err != nil { + return err + } + + return nil } diff --git a/_examples/testdata/aws_leak.json b/_examples/testdata/aws_leak.json deleted file mode 100644 index e174c08..0000000 --- a/_examples/testdata/aws_leak.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "lineNumber": 5, - "offender": "AKIAIO5FODNN7EXAMPLE", - "rule": "AWS Access Key", - "file": "../test_data/test_repos/test_dir_1/server.test.py", - "tags": "key, AWS" - } -] diff --git a/_examples/testdata/aws_leak.txt b/_examples/testdata/aws_leak.txt new file mode 100644 index 0000000..6409b08 --- /dev/null +++ b/_examples/testdata/aws_leak.txt @@ -0,0 +1 @@ +AKIAIO5FODNN7EXAMPLE \ No newline at end of file diff --git a/cmd/pillager/main.go b/cmd/pillager/main.go new file mode 100644 index 0000000..729ee97 --- /dev/null +++ b/cmd/pillager/main.go @@ -0,0 +1,9 @@ +package main + +import ( + pillager "github.com/brittonhayes/pillager/internal/commands" +) + +func main() { + pillager.Execute() +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..d296e6b --- /dev/null +++ b/config.go @@ -0,0 +1,141 @@ +//go:generate golangci-lint run ./... +//go:generate gomarkdoc ./pkg/hunter/... +//go:generate gomarkdoc ./pkg/rules/... +//go:generate gomarkdoc ./pkg/format/... +package pillager + +import ( + "errors" + "os" + "runtime" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "github.com/brittonhayes/pillager/internal/validate" + "github.com/brittonhayes/pillager/pkg/format" + "github.com/brittonhayes/pillager/pkg/rules" + "github.com/spf13/afero" + gitleaks "github.com/zricethezav/gitleaks/v7/config" +) + +// Config takes all of the configurable +// parameters for a Hunter. +type Config struct { + Filesystem afero.Fs + Style format.Style + Gitleaks gitleaks.Config + + ScanPath string + Verbose bool + Debug bool + Workers int + Template string +} + +type ConfigOption func(*Config) + +func NewConfig(opts ...ConfigOption) *Config { + var ( + defaultFS = afero.NewOsFs() + defaultVerbose = false + defaultScanPath = "." + defaultStyle = format.StyleJSON + defaultWorkers = runtime.NumCPU() + defaultGitleaks = rules.NewLoader().Load() + defaultTemplate = "" + defaultLogLevel = zerolog.ErrorLevel + ) + + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + zerolog.SetGlobalLevel(defaultLogLevel) + config := &Config{ + ScanPath: defaultScanPath, + Filesystem: defaultFS, + Style: defaultStyle, + Workers: defaultWorkers, + Gitleaks: defaultGitleaks, + Verbose: defaultVerbose, + Template: defaultTemplate, + } + + for _, opt := range opts { + opt(config) + } + + if err := config.validate(); err != nil { + log.Fatal().Err(err).Send() + } + + return config +} + +func WithFS(fs afero.Fs) ConfigOption { + return func(c *Config) { + c.Filesystem = fs + } +} + +func WithScanPath(path string) ConfigOption { + return func(c *Config) { + c.ScanPath = validate.Path(c.Filesystem, c.ScanPath) + } +} + +func WithLogLevel(level string) ConfigOption { + return func(c *Config) { + lvl, err := zerolog.ParseLevel(level) + if err != nil { + log.Fatal().Err(err).Send() + } + zerolog.SetGlobalLevel(lvl) + } +} + +func WithVerbose(verbose bool) ConfigOption { + return func(c *Config) { + c.Verbose = verbose + } +} + +func WithWorkers(count int) ConfigOption { + return func(c *Config) { + c.Workers = count + } +} + +func WithStyle(style format.Style) ConfigOption { + return func(c *Config) { + if c.Template != "" { + c.Style = format.StyleCustom + return + } + + c.Style = style + } +} + +func WithTemplate(template string) ConfigOption { + return func(c *Config) { + c.Style = format.StyleCustom + c.Template = template + } +} + +func WithGitleaksConfig(g gitleaks.Config) ConfigOption { + return func(c *Config) { + c.Gitleaks = g + } +} + +func (c *Config) validate() error { + if c.Filesystem == nil { + return errors.New("missing filesystem in Config") + } + + if c.Gitleaks.Rules == nil { + return errors.New("no gitleaks rules provided") + } + + return nil +} diff --git a/doc.go b/doc.go index 7944fae..947c02b 100644 --- a/doc.go +++ b/doc.go @@ -38,4 +38,4 @@ THE SOFTWARE. // brew tap brittonhayes/homebrew-pillager // brew install pillager // -package main +package pillager diff --git a/go.mod b/go.mod index afcb541..780e309 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,15 @@ go 1.17 require ( github.com/BurntSushi/toml v0.4.1 - github.com/ghodss/yaml v1.0.0 - github.com/gookit/color v1.5.0 + github.com/gookit/color v1.4.2 github.com/mitchellh/go-homedir v1.1.0 + github.com/pkg/errors v0.9.1 + github.com/rs/zerolog v1.26.1 github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.9.0 github.com/zricethezav/gitleaks/v7 v7.6.1 + gopkg.in/yaml.v2 v2.4.0 ) require ( @@ -38,12 +40,11 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect - golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect - golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 // indirect + golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e // indirect + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect golang.org/x/text v0.3.6 // indirect gopkg.in/ini.v1 v1.63.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 3967305..2e49880 100644 --- a/go.sum +++ b/go.sum @@ -97,7 +97,6 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= @@ -185,8 +184,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/gookit/color v1.5.0 h1:1Opow3+BWDwqor78DcJkJCIwnkviFi+rrOANki9BUFw= -github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= +github.com/gookit/color v1.4.2 h1:tXy44JFSFkKnELV6WaMo/lLfu/meqITX3iAV52do7lk= +github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= @@ -292,6 +291,9 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= +github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= @@ -339,6 +341,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zricethezav/gitleaks/v7 v7.6.1 h1:SP0loYvBSpy5YjpgC/2fsBEVo4dncks54yEqaesAXZQ= github.com/zricethezav/gitleaks/v7 v7.6.1/go.mod h1:7b5pE0W3dQhdY9+gbgSBEuzjU5wMAepzj2q1HDh8uNo= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -365,8 +368,9 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -440,8 +444,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 h1:a8jGStKg0XqKDlKqjLrXn0ioF5MH36pT7Z0BRTqLhbk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -529,6 +534,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= @@ -601,6 +607,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/images/brand_image.png b/images/brand_image.png deleted file mode 100644 index 42079b3..0000000 Binary files a/images/brand_image.png and /dev/null differ diff --git a/cmd/pillager/hunt.go b/internal/commands/hunt.go similarity index 59% rename from cmd/pillager/hunt.go rename to internal/commands/hunt.go index d631c7e..38e5abd 100644 --- a/cmd/pillager/hunt.go +++ b/internal/commands/hunt.go @@ -4,23 +4,25 @@ package pillager import ( + "os" "runtime" + "github.com/brittonhayes/pillager" + "github.com/brittonhayes/pillager/pkg/format" "github.com/brittonhayes/pillager/pkg/hunter" - "github.com/brittonhayes/pillager/pkg/rules" - "github.com/spf13/afero" "github.com/spf13/cobra" ) var ( verbose bool + level string rulesConfig string - output string + style string templ string workers int ) -// huntCmd represents the hunt command +// huntCmd represents the hunt command. var huntCmd = &cobra.Command{ Use: "hunt [directory]", Short: "Hunt for loot", @@ -51,15 +53,40 @@ var huntCmd = &cobra.Command{ pillager hunt ./example --template "$(cat templates/simple.tmpl)" `, Args: cobra.ExactArgs(1), - RunE: startHunt(), + RunE: func(cmd *cobra.Command, args []string) error { + h, err := hunter.New( + pillager.WithScanPath(args[0]), + pillager.WithWorkers(workers), + pillager.WithVerbose(verbose), + pillager.WithTemplate(templ), + pillager.WithStyle(format.StringToFormat(style)), + pillager.WithLogLevel(level), + ) + if err != nil { + return err + } + + results, err := h.Hunt() + if err != nil { + return err + } + + err = h.Report(os.Stdout, results) + if err != nil { + return err + } + + return nil + }, } func init() { rootCmd.AddCommand(huntCmd) - huntCmd.Flags().IntVarP(&workers, "workers", "w", runtime.NumCPU(), "number of concurrent workers to create") - huntCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "toggle verbose output") + huntCmd.Flags().IntVarP(&workers, "workers", "w", runtime.NumCPU(), "number of concurrent workers") + huntCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "enable scanner verbose output") + huntCmd.Flags().StringVarP(&level, "log-level", "l", "error", "set logging level") huntCmd.Flags().StringVarP(&rulesConfig, "rules", "r", "", "path to gitleaks rules.toml config") - huntCmd.Flags().StringVarP(&output, "format", "f", "json", "set output format (json, yaml)") + huntCmd.Flags().StringVarP(&style, "format", "f", "json", "set output format (json, yaml)") huntCmd.Flags().StringVarP( &templ, "template", @@ -68,19 +95,3 @@ func init() { "set go text/template string for output format", ) } - -func startHunt() func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - c := hunter.NewConfig( - afero.NewOsFs(), - args[0], - verbose, - rules.Load(rulesConfig), - hunter.StringToFormat(output), - templ, - workers, - ) - h := hunter.NewHunter(c) - return h.Hunt() - } -} diff --git a/cmd/pillager/root.go b/internal/commands/root.go similarity index 93% rename from cmd/pillager/root.go rename to internal/commands/root.go index a772d05..89e67bf 100644 --- a/cmd/pillager/root.go +++ b/internal/commands/root.go @@ -6,13 +6,15 @@ import ( "github.com/gookit/color" "github.com/mitchellh/go-homedir" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) var cfgFile string -// rootCmd represents the base command when called without any subcommands +// rootCmd represents the base command when called without any subcommands. var rootCmd = &cobra.Command{ Use: "pillager", Short: "Pillage systems for sensitive information", @@ -68,6 +70,7 @@ func initConfig() { // If a config file is found, read it in. if err := viper.ReadInConfig(); err == nil { - fmt.Println("Using config file:", viper.ConfigFileUsed()) + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + log.Info().Msgf("Using config file: %q", viper.ConfigFileUsed()) } } diff --git a/cmd/pillager/version.go b/internal/commands/version.go similarity index 100% rename from cmd/pillager/version.go rename to internal/commands/version.go diff --git a/internal/validate/path.go b/internal/validate/path.go index 7de2a5c..df1b680 100644 --- a/internal/validate/path.go +++ b/internal/validate/path.go @@ -1,38 +1,23 @@ package validate import ( - "log" - "os" + "github.com/rs/zerolog/log" "github.com/spf13/afero" ) -var _ Validator = &Validation{} - -type Validation struct{} - -// New creates a new validation -func New() *Validation { - return &Validation{} -} - -type Validator interface { - Path(fs afero.Fs, path string) string -} - // Path checks if a filepath exists and -// returns it if so, otherwise returns a default path -func (v *Validation) Path(fs afero.Fs, path string) string { +// returns it if so, otherwise returns a default path. +func Path(fs afero.Fs, path string) string { ok, err := afero.Exists(fs, path) if err != nil { - log.Printf("ERROR: %s", err.Error()) - os.Exit(1) + log.Fatal().Err(err).Send() } if ok { return path } - log.Fatal("no valid path provided") + log.Fatal().Msg("no valid path provided") return "." } diff --git a/images/brand_image_ice.png b/logo.png similarity index 100% rename from images/brand_image_ice.png rename to logo.png diff --git a/main.go b/main.go deleted file mode 100644 index d68f467..0000000 --- a/main.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "github.com/brittonhayes/pillager/cmd/pillager" -) - -//go:generate golangci-lint run ./... -//go:generate gomarkdoc ./hunter/... -//go:generate gomarkdoc ./rules/... - -func main() { - pillager.Execute() -} diff --git a/pkg/format/README.md b/pkg/format/README.md new file mode 100755 index 0000000..f86e106 --- /dev/null +++ b/pkg/format/README.md @@ -0,0 +1,74 @@ + + +# format + +```go +import "github.com/brittonhayes/pillager/pkg/format" +``` + +## Index + +- [Constants](<#constants>) +- [func RenderTemplate(w io.Writer, tpl string, f scan.Report) error](<#func-rendertemplate>) +- [type Style](<#type-style>) + - [func StringToFormat(s string) Style](<#func-stringtoformat>) + - [func (s Style) String() string](<#func-style-string>) + + +## Constants + +DefaultTemplate is the base template used to format a Finding into the custom output format\. + +```go +const DefaultTemplate = `{{ with . -}} +{{ range .Leaks -}} +Line: {{.LineNumber}} +File: {{ .File }} +Offender: {{ .Offender }} + +{{ end -}}{{- end}}` +``` + +## func [RenderTemplate]() + +```go +func RenderTemplate(w io.Writer, tpl string, f scan.Report) error +``` + +RenderTemplate renders a finding in a custom go template format to the provided writer\. + +## type [Style]() + +```go +type Style int +``` + +```go +const ( + StyleJSON Style = iota + StyleYAML + StyleTable + StyleHTML + StyleHTMLTable + StyleMarkdown + StyleCustom +) +``` + +### func [StringToFormat]() + +```go +func StringToFormat(s string) Style +``` + +StringToFormat takes in a string representation of the preferred output format and returns to enum equivalent\. + +### func \(Style\) [String]() + +```go +func (s Style) String() string +``` + + + +Generated by [gomarkdoc]() diff --git a/pkg/format/format.go b/pkg/format/format.go new file mode 100644 index 0000000..1afbafa --- /dev/null +++ b/pkg/format/format.go @@ -0,0 +1,40 @@ +package format + +import "strings" + +const ( + StyleJSON Style = iota + StyleYAML + StyleTable + StyleHTML + StyleHTMLTable + StyleMarkdown + StyleCustom +) + +type Style int + +func (s Style) String() string { + return [...]string{"json", "yaml", "table", "html", "html-table", "markdown", "custom"}[s] +} + +// StringToFormat takes in a string representation of the preferred +// output format and returns to enum equivalent. +func StringToFormat(s string) Style { + switch strings.ToLower(s) { + case "yaml": + return StyleYAML + case "table": + return StyleTable + case "html": + return StyleHTML + case "html-table": + return StyleHTMLTable + case "markdown": + return StyleMarkdown + case "custom": + return StyleCustom + default: + return StyleJSON + } +} diff --git a/pkg/hunter/template.go b/pkg/format/template.go similarity index 51% rename from pkg/hunter/template.go rename to pkg/format/template.go index 29f8a0d..7c46ffa 100644 --- a/pkg/hunter/template.go +++ b/pkg/format/template.go @@ -1,15 +1,15 @@ -package hunter +package format import ( "html/template" "io" - "log" + "github.com/pkg/errors" "github.com/zricethezav/gitleaks/v7/scan" ) // DefaultTemplate is the base template used to -// format a Finding into the custom output format +// format a Finding into the custom output format. const DefaultTemplate = `{{ with . -}} {{ range .Leaks -}} Line: {{.LineNumber}} @@ -18,16 +18,18 @@ Offender: {{ .Offender }} {{ end -}}{{- end}}` -// RenderTemplate renders a Hound finding in a -// custom go template format to the provided writer -func RenderTemplate(w io.Writer, tpl string, f scan.Report) { +// RenderTemplate renders a finding in a +// custom go template format to the provided writer. +func RenderTemplate(w io.Writer, tpl string, f scan.Report) error { t := template.New("custom") t, err := t.Parse(tpl) if err != nil { - log.Fatal("failed to parse template, ", err.Error()) + return errors.Wrap(err, "failed to parse template") } if err := t.Execute(w, f); err != nil { - log.Fatal("Failed to use custom template, ", err.Error()) + return errors.Wrap(err, "Failed to use custom template") } + + return nil } diff --git a/pkg/hunter/README.md b/pkg/hunter/README.md index 7af0015..c6a7998 100644 --- a/pkg/hunter/README.md +++ b/pkg/hunter/README.md @@ -3,339 +3,52 @@ # hunter ```go -import "github.com/brittonhayes/pillager/hunter" +import "github.com/brittonhayes/pillager/pkg/hunter" ``` Package hunter contains the types\, methods\, and interfaces for the file hunting portion of pillager ## Index -- [Constants](<#constants>) -- [func RenderTemplate(w io.Writer, tpl string, f scan.Report)](<#func-rendertemplate>) -- [type Config](<#type-config>) - - [func NewConfig(fs afero.Fs, path string, verbose bool, gitleaks gitleaks.Config, format Format, template string, workers int) *Config](<#func-newconfig>) - - [func (c *Config) Default() *Config](<#func-config-default>) - - [func (c *Config) Validate() (err error)](<#func-config-validate>) -- [type Configer](<#type-configer>) -- [type Format](<#type-format>) - - [func StringToFormat(s string) Format](<#func-stringtoformat>) - - [func (f Format) String() string](<#func-format-string>) -- [type Hound](<#type-hound>) - - [func NewHound(c *Config) *Hound](<#func-newhound>) - - [func (h *Hound) Howl(findings scan.Report)](<#func-hound-howl>) -- [type Hounder](<#type-hounder>) - [type Hunter](<#type-hunter>) - - [func NewHunter(c *Config) *Hunter](<#func-newhunter>) - - [func (h Hunter) Hunt() error](<#func-hunter-hunt>) -- [type Hunting](<#type-hunting>) + - [func New(opts ...pillager.ConfigOption) (*Hunter, error)](<#func-new>) + - [func (h *Hunter) Hunt() (scan.Report, error)](<#func-hunter-hunt>) + - [func (h *Hunter) Report(w io.Writer, results scan.Report) error](<#func-hunter-report>) -## Constants +## type [Hunter]() -DefaultTemplate is the base template used to format a Finding into the custom output format - -```go -const DefaultTemplate = `{{ with . -}} -{{ range .Leaks -}}Line: {{.LineNumber}} -File: {{ .File }} -Offender: {{ .Offender }} - -{{end}} -{{- end}}` -``` - -## func [RenderTemplate]() - -```go -func RenderTemplate(w io.Writer, tpl string, f scan.Report) -``` - -RenderTemplate renders a Hound finding in a custom go template format to the provided writer - -## type [Config]() - -Config takes all of the configurable parameters for a Hunter - -```go -type Config struct { - System afero.Fs - BasePath string - Verbose bool - Workers int - Gitleaks gitleaks.Config - Format Format - Template string -} -``` - -### func [NewConfig]() - -```go -func NewConfig(fs afero.Fs, path string, verbose bool, gitleaks gitleaks.Config, format Format, template string, workers int) *Config -``` - -NewConfig generates a new config for the Hunter - -### func \(\*Config\) [Default]() - -```go -func (c *Config) Default() *Config -``` - -Default loads the default configuration for the Hunter - -### func \(\*Config\) [Validate]() - -```go -func (c *Config) Validate() (err error) -``` - -## type [Configer]() - -```go -type Configer interface { - Default() *Config - Validate() (err error) -} -``` - -## type [Format]() - -```go -type Format int -``` - -```go -const ( - JSONFormat Format = iota + 1 - YAMLFormat - CustomFormat -) -``` - -### func [StringToFormat]() - -```go -func StringToFormat(s string) Format -``` - -StringToFormat takes in a string representation of the preferred output format and returns to enum equivalent - -### func \(Format\) [String]() - -```go -func (f Format) String() string -``` - -## type [Hound]() - -A Hound performs the file inspection and returns the results - -```go -type Hound struct { - Config *Config - Findings scan.Report `json:"findings"` -} -``` - -### func [NewHound]() - -```go -func NewHound(c *Config) *Hound -``` - -NewHound creates an instance of the Hound type - -### func \(\*Hound\) [Howl]() - -```go -func (h *Hound) Howl(findings scan.Report) -``` - -Howl prints out the Findings from the Hound in the preferred output format - -
Example -

- -Here is an example of utilizing the Howl function on a slice of findings\. The Howl method is the final method in the hunting process\. It takes whatever has been found and outputs it for the user\. - -```go -{ - h := NewHound(&Config{ - System: afero.NewMemMapFs(), - Gitleaks: rules.Load(""), - Format: JSONFormat, - }) - findings := scan.Report{ - Leaks: []scan.Leak{ - {Line: "person@email.com", LineNumber: 16, Offender: "person@email.com", Rule: "Email Addresses"}, - }, - } - - h.Howl(findings) -} -``` - -

-
- -## type [Hounder]() - -The Hounder interface defines the available methods for instances of the Hound type - -```go -type Hounder interface { - Howl(findings scan.Report) -} -``` - -## type [Hunter]() - -Hunter holds the required fields to implement the Hunting interface and utilize the hunter package +Hunter is the secret scanner\. ```go type Hunter struct { - Config *Config - Hound *Hound + *pillager.Config } ``` -### func [NewHunter]() +### func [New]() ```go -func NewHunter(c *Config) *Hunter +func New(opts ...pillager.ConfigOption) (*Hunter, error) ``` -NewHunter creates an instance of the Hunter type +New creates an instance of the Hunter\. -### func \(Hunter\) [Hunt]() +### func \(\*Hunter\) [Hunt]() ```go -func (h Hunter) Hunt() error +func (h *Hunter) Hunt() (scan.Report, error) ``` -Hunt walks over the filesystem at the configured path\, looking for sensitive information +Hunt walks over the filesystem at the configured path\, looking for sensitive information\. -
Example -

- -This method also accepts custom output formats using go template/html\. So if you don't like yaml or json\, you can format to your heart's content\. +### func \(\*Hunter\) [Report]() ```go -{ - fs := afero.NewMemMapFs() - f, err := fs.Create("example.yaml") - if err != nil { - panic(err) - } - defer f.Close() - - _, err = f.Write([]byte(`https://github.com/brittonhayes/pillager`)) - if err != nil { - panic(err) - } - - config := NewConfig(fs, "./", true, rules.Load(""), CustomFormat, DefaultTemplate, 5) - h := NewHunter(config) - _ = h.Hunt() -} +func (h *Hunter) Report(w io.Writer, results scan.Report) error ``` -

-
- -
Example -

- -This is an example of how to run a scan on a single file to look for email addresses\. We're using an in\-memory file system for simplicity\, but this supports using an actual file system as well\. - -```go -{ - fs := afero.NewMemMapFs() - f, err := fs.Create("example.toml") - if err != nil { - panic(err) - } - defer f.Close() - - _, err = f.Write([]byte(`example@email.com`)) - if err != nil { - panic(err) - } - - config := NewConfig(fs, "./", true, rules.Load(""), StringToFormat("yaml"), DefaultTemplate, 5) - h := NewHunter(config) - _ = h.Hunt() -} -``` - -

-
- -
Example -

- -This method accepts json output format as well - -```go -{ - fs := afero.NewMemMapFs() - f, err := fs.Create("fake.json") - if err != nil { - panic(err) - } - defer f.Close() - _, err = f.Write([]byte(`git@github.com:brittonhayes/pillager.git`)) - if err != nil { - panic(err) - } - - config := NewConfig(fs, ".", true, rules.Load(""), JSONFormat, DefaultTemplate, 5) - h := NewHunter(config) - _ = h.Hunt() -} -``` - -

-
- -
Example -

- -Hunter will also look personally identifiable info in TOML - -```go -{ - fs := afero.NewMemMapFs() - f, err := fs.Create("fake.toml") - if err != nil { - panic(err) - } - defer f.Close() - _, err = f.Write([]byte(`fakeperson@example.com`)) - if err != nil { - panic(err) - } - - config := NewConfig(fs, ".", true, rules.Load(""), JSONFormat, DefaultTemplate, 5) - - h := NewHunter(config) - _ = h.Hunt() -} -``` - -

-
- -## type [Hunting]() - -Hunting is the primary API interface for the hunter package - -```go -type Hunting interface { - Hunt() error -} -``` +Report prints out the Findings in the preferred output format\. diff --git a/pkg/hunter/config.go b/pkg/hunter/config.go deleted file mode 100644 index 0284d5f..0000000 --- a/pkg/hunter/config.go +++ /dev/null @@ -1,68 +0,0 @@ -package hunter - -import ( - "fmt" - - "github.com/brittonhayes/pillager/internal/validate" - "github.com/brittonhayes/pillager/pkg/rules" - "github.com/spf13/afero" - gitleaks "github.com/zricethezav/gitleaks/v7/config" -) - -var _ Configer = &Config{} - -// Config takes all of the configurable -// parameters for a Hunter -type Config struct { - System afero.Fs - BasePath string - Verbose bool - Workers int - Gitleaks gitleaks.Config - Format Format - Template string -} - -type Configer interface { - Default() *Config - Validate() (err error) -} - -// NewConfig generates a new config for the Hunter -func NewConfig(fs afero.Fs, path string, verbose bool, gitleaks gitleaks.Config, format Format, template string, workers int) *Config { - p := validate.New().Path(fs, path) - return &Config{ - System: fs, - BasePath: p, - Verbose: verbose, - Gitleaks: gitleaks, - Format: format, - Template: template, - Workers: workers, - } -} - -// Default loads the default configuration -// for the Hunter -func (c *Config) Default() *Config { - fs := afero.NewOsFs() - v := validate.New() - return &Config{ - System: fs, - BasePath: v.Path(fs, "."), - Verbose: false, - Gitleaks: rules.Load(""), - Format: JSONFormat, - } -} - -func (c *Config) Validate() (err error) { - if c.System == nil { - err = fmt.Errorf("missing filesystem in Hunter Config") - } - - if c.Gitleaks.Rules == nil { - err = fmt.Errorf("no gitleaks config provided") - } - return -} diff --git a/pkg/hunter/format.go b/pkg/hunter/format.go deleted file mode 100644 index a0e839d..0000000 --- a/pkg/hunter/format.go +++ /dev/null @@ -1,40 +0,0 @@ -package hunter - -import "strings" - -const ( - JSONFormat Format = iota + 1 - YAMLFormat - TableFormat - HTMLFormat - HTMLTableFormat - MarkdownFormat - CustomFormat -) - -type Format int - -func (f Format) String() string { - return [...]string{"json", "yaml", "table", "html", "html-table", "markdown", "custom"}[f] -} - -// StringToFormat takes in a string representation of the preferred -// output format and returns to enum equivalent -func StringToFormat(s string) Format { - switch strings.ToLower(s) { - case "yaml": - return YAMLFormat - case "table": - return TableFormat - case "html": - return HTMLFormat - case "html-table": - return HTMLTableFormat - case "markdown": - return MarkdownFormat - case "custom": - return CustomFormat - default: - return JSONFormat - } -} diff --git a/pkg/hunter/hound.go b/pkg/hunter/hound.go deleted file mode 100644 index 04ad31f..0000000 --- a/pkg/hunter/hound.go +++ /dev/null @@ -1,73 +0,0 @@ -package hunter - -import ( - "encoding/json" - "fmt" - "log" - "os" - - "github.com/brittonhayes/pillager/templates" - "github.com/ghodss/yaml" - "github.com/zricethezav/gitleaks/v7/scan" -) - -var _ Hounder = &Hound{} - -// The Hounder interface defines the available methods -// for instances of the Hound type -type Hounder interface { - Howl(findings scan.Report) -} - -// A Hound performs the file inspection and returns the results -type Hound struct { - Config *Config - Findings scan.Report `json:"findings"` -} - -// NewHound creates an instance of the Hound type -func NewHound(c *Config) *Hound { - if c == nil { - var config Config - return &Hound{config.Default(), scan.Report{}} - } - if c.System == nil { - log.Fatal("Missing filesystem in Hunter Config") - } - - return &Hound{c, scan.Report{}} -} - -// Howl prints out the Findings from the Hound in the preferred output format -func (h *Hound) Howl(findings scan.Report) { - if h.Config.Template != "" { - h.Config.Format = CustomFormat - } - switch h.Config.Format { - case JSONFormat: - b, err := json.Marshal(&findings.Leaks) - if err != nil { - log.Fatal("Failed to unmarshal findings") - } - fmt.Println(string(b)) - case YAMLFormat: - b, err := yaml.Marshal(&findings.Leaks) - if err != nil { - fmt.Printf("err: %v\n", err) - return - } - fmt.Println(string(b)) - case HTMLFormat: - RenderTemplate(os.Stdout, templates.HTML, findings) - case HTMLTableFormat: - RenderTemplate(os.Stdout, templates.HTMLTable, findings) - case MarkdownFormat: - RenderTemplate(os.Stdout, templates.Markdown, findings) - case TableFormat: - RenderTemplate(os.Stdout, templates.Table, findings) - case CustomFormat: - RenderTemplate(os.Stdout, h.Config.Template, findings) - default: - RenderTemplate(os.Stdout, templates.Simple, findings) - } -} diff --git a/pkg/hunter/hound_test.go b/pkg/hunter/hound_test.go deleted file mode 100644 index 725f619..0000000 --- a/pkg/hunter/hound_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package hunter - -import ( - "github.com/brittonhayes/pillager/pkg/rules" - "github.com/spf13/afero" - "github.com/zricethezav/gitleaks/v7/scan" -) - -// Here is an example of utilizing the Howl function -// on a slice of findings. The Howl method is the final -// method in the hunting process. It takes whatever -// has been found and outputs it for the user. -func ExampleHound_Howl_json() { - h := NewHound(&Config{ - System: afero.NewMemMapFs(), - Gitleaks: rules.Load(""), - Format: JSONFormat, - }) - findings := scan.Report{ - Leaks: []scan.Leak{ - {Line: "person@email.com", LineNumber: 16, Offender: "person@email.com", Rule: "Email Addresses"}, - }, - } - - h.Howl(findings) -} diff --git a/pkg/hunter/hunter.go b/pkg/hunter/hunter.go index 4e8673a..6d3606e 100644 --- a/pkg/hunter/hunter.go +++ b/pkg/hunter/hunter.go @@ -1,62 +1,82 @@ package hunter import ( + "encoding/json" "fmt" - "os" + "io" + "github.com/brittonhayes/pillager" + "github.com/brittonhayes/pillager/pkg/format" + "github.com/brittonhayes/pillager/templates" + "github.com/rs/zerolog/log" "github.com/zricethezav/gitleaks/v7/config" "github.com/zricethezav/gitleaks/v7/options" "github.com/zricethezav/gitleaks/v7/scan" + "gopkg.in/yaml.v2" ) -// Hunter holds the required fields to implement -// the Hunting interface and utilize the hunter package +// Hunter is the secret scanner. type Hunter struct { - Config *Config - Hound *Hound + *pillager.Config } -var _ Hunting = Hunter{} - -// Hunting is the primary API interface for the hunter package -type Hunting interface { - Hunt() error +// New creates an instance of the Hunter. +func New(opts ...pillager.ConfigOption) (*Hunter, error) { + return &Hunter{ + Config: pillager.NewConfig(opts...), + }, nil } -// NewHunter creates an instance of the Hunter type -func NewHunter(c *Config) *Hunter { - if c == nil { - var conf Config - return &Hunter{conf.Default(), NewHound(conf.Default())} - } +// Hunt walks over the filesystem at the configured path, looking for sensitive information. +func (h *Hunter) Hunt() (scan.Report, error) { + opt := options.Options{Path: h.ScanPath, Verbose: h.Verbose, Threads: h.Workers} + conf := config.Config{Allowlist: h.Gitleaks.Allowlist, Rules: h.Gitleaks.Rules} + + scanner := scan.NewNoGitScanner(opt, conf) + log.Debug().Str("style", h.Style.String()).Bool("verbose", h.Verbose).Msg("scanner created") - err := c.Validate() + report, err := scanner.Scan() if err != nil { - fmt.Println(err.Error()) - os.Exit(1) + return scan.Report{}, err } - return &Hunter{c, NewHound(c)} + return report, nil } -// Hunt walks over the filesystem at the configured path, looking for sensitive information -func (h Hunter) Hunt() error { - h.Hound = NewHound(h.Config) - if _, err := os.Stat(h.Config.BasePath); os.IsNotExist(err) { - return fmt.Errorf("config file does not exist") - } +// Report prints out the Findings in the preferred output format. +func (h *Hunter) Report(w io.Writer, results scan.Report) error { + switch h.Style { + case format.StyleJSON: + encoder := json.NewEncoder(w) + err := encoder.Encode(&results.Leaks) + if err != nil { + return err + } - opt := options.Options{Path: h.Config.BasePath, Verbose: h.Config.Verbose, Threads: h.Config.Workers} - conf := config.Config{Allowlist: h.Config.Gitleaks.Allowlist, Rules: h.Config.Gitleaks.Rules} + case format.StyleYAML: + b, err := yaml.Marshal(&results.Leaks) + if err != nil { + return err + } + fmt.Fprintf(w, "%s\n", string(b)) - scanner := scan.NewNoGitScanner(opt, conf) - report, err := scanner.Scan() - if err != nil { - return err - } + case format.StyleHTML: + return format.RenderTemplate(w, templates.HTML, results) + + case format.StyleHTMLTable: + return format.RenderTemplate(w, templates.HTMLTable, results) + + case format.StyleMarkdown: + return format.RenderTemplate(w, templates.Markdown, results) + + case format.StyleTable: + return format.RenderTemplate(w, templates.Table, results) + + case format.StyleCustom: + return format.RenderTemplate(w, h.Template, results) - if !opt.Verbose { - h.Hound.Howl(report) + default: + return format.RenderTemplate(w, templates.Simple, results) } return nil diff --git a/pkg/hunter/hunter_test.go b/pkg/hunter/hunter_test.go deleted file mode 100644 index a70b865..0000000 --- a/pkg/hunter/hunter_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package hunter - -import ( - "github.com/brittonhayes/pillager/pkg/rules" - "github.com/spf13/afero" -) - -// This is an example of how to run a scan on a single file to look for -// email addresses. We're using an in-memory file system for simplicity, -// but this supports using an actual file system as well. -func ExampleHunter_Hunt_email() { - fs := afero.NewMemMapFs() - f, err := fs.Create("example.toml") - if err != nil { - panic(err) - } - defer f.Close() - - _, err = f.Write([]byte(`example@email.com`)) - if err != nil { - panic(err) - } - - config := NewConfig(fs, "./", true, rules.Load(""), StringToFormat("yaml"), DefaultTemplate, 5) - h := NewHunter(config) - _ = h.Hunt() -} - -// This method also accepts custom output formats using -// go template/html. So if you don't like yaml or json, -// you can format to your heart's content. -func ExampleHunter_Hunt_custom_output() { - fs := afero.NewMemMapFs() - f, err := fs.Create("example.yaml") - if err != nil { - panic(err) - } - defer f.Close() - - _, err = f.Write([]byte(`https://github.com/brittonhayes/pillager`)) - if err != nil { - panic(err) - } - - config := NewConfig(fs, "./", true, rules.Load(""), CustomFormat, DefaultTemplate, 5) - h := NewHunter(config) - _ = h.Hunt() -} - -// This method accepts json output format -// as well -func ExampleHunter_Hunt_json() { - fs := afero.NewMemMapFs() - f, err := fs.Create("fake.json") - if err != nil { - panic(err) - } - defer f.Close() - _, err = f.Write([]byte(`git@github.com:brittonhayes/pillager.git`)) - if err != nil { - panic(err) - } - - config := NewConfig(fs, ".", true, rules.Load(""), JSONFormat, DefaultTemplate, 5) - h := NewHunter(config) - _ = h.Hunt() -} - -// Hunter will also look personally identifiable info in TOML -func ExampleHunter_Hunt_toml() { - fs := afero.NewMemMapFs() - f, err := fs.Create("fake.toml") - if err != nil { - panic(err) - } - defer f.Close() - _, err = f.Write([]byte(`fakeperson@example.com`)) - if err != nil { - panic(err) - } - - config := NewConfig(fs, ".", true, rules.Load(""), JSONFormat, DefaultTemplate, 5) - - h := NewHunter(config) - _ = h.Hunt() -} diff --git a/pkg/rules/README.md b/pkg/rules/README.md index c025c2e..e3239dc 100755 --- a/pkg/rules/README.md +++ b/pkg/rules/README.md @@ -3,56 +3,86 @@ # rules ```go -import "github.com/brittonhayes/pillager/rules" +import "github.com/brittonhayes/pillager/pkg/rules" ``` ## Index - [Constants](<#constants>) -- [func Load(filepath string) gitleaks.Config](<#func-load>) +- [Variables](<#variables>) +- [type Loader](<#type-loader>) + - [func NewLoader(opts ...LoaderOption) *Loader](<#func-newloader>) + - [func (l *Loader) Load() gitleaks.Config](<#func-loader-load>) + - [func (l *Loader) WithStrict() LoaderOption](<#func-loader-withstrict>) +- [type LoaderOption](<#type-loaderoption>) + - [func FromFile(file string) LoaderOption](<#func-fromfile>) ## Constants -DefaultConfig is the default ruleset for pillager's hunting parameters\. This can be overridden by providing a rules\.toml file as an argument\. - -```go -const DefaultConfig = ` -title = "pillager config" -[[rules]] - description = "AWS Access Key" - regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' - tags = ["key", "AWS"] -[[rules]] - description = "AWS Secret Key" - regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]''' - tags = ["key", "AWS"] -[[rules]] - description = "Github" - regex = '''(?i)github(.{0,20})?(?-i)[0-9a-zA-Z]{35,40}''' - tags = ["key", "Github"] -[[rules]] - description = "Slack" - regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?''' - tags = ["key", "Slack"] -[[rules]] - description = "Asymmetric Private Key" - regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----''' - tags = ["key", "AsymmetricPrivateKey"] -[[rules]] - description = "Slack Webhook" - regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}''' - tags = ["key", "slack"] -` -``` - -## func [Load]() - -```go -func Load(filepath string) gitleaks.Config -``` - -Load loads the config file into an array of gitleaks rules +```go +const ( + ErrReadConfig = "Failed to read config" +) +``` + +## Variables + +```go +var ( + //go:embed rules_simple.toml + RulesDefault string + + //go:embed rules_strict.toml + RulesStrict string +) +``` + +## type [Loader]() + +```go +type Loader struct { + // contains filtered or unexported fields +} +``` + +### func [NewLoader]() + +```go +func NewLoader(opts ...LoaderOption) *Loader +``` + +NewLoader creates a configuration loader\. + +### func \(\*Loader\) [Load]() + +```go +func (l *Loader) Load() gitleaks.Config +``` + +Load parses the gitleaks configuration\. + +### func \(\*Loader\) [WithStrict]() + +```go +func (l *Loader) WithStrict() LoaderOption +``` + +WithStrict enables more strict pillager scanning\. + +## type [LoaderOption]() + +```go +type LoaderOption func(*Loader) +``` + +### func [FromFile]() + +```go +func FromFile(file string) LoaderOption +``` + +FromFile decodes the configuration from a local file\. diff --git a/pkg/rules/rules.go b/pkg/rules/rules.go index 1ead21f..f676c6c 100644 --- a/pkg/rules/rules.go +++ b/pkg/rules/rules.go @@ -1,32 +1,74 @@ package rules import ( - "log" + _ "embed" "github.com/BurntSushi/toml" + "github.com/rs/zerolog/log" gitleaks "github.com/zricethezav/gitleaks/v7/config" ) -// Load loads the config file into an array of gitleaks rules -func Load(filepath string) gitleaks.Config { - var ( - config gitleaks.TomlLoader - err error - ) - - if filepath != "" { - _, err = toml.DecodeFile(filepath, &config) - } else { - _, err = toml.Decode(DefaultConfig, &config) - } +const ( + ErrReadConfig = "Failed to read config" +) + +var ( + //go:embed rules_simple.toml + RulesDefault string + + //go:embed rules_strict.toml + RulesStrict string +) + +type Loader struct { + loader gitleaks.TomlLoader +} + +type LoaderOption func(*Loader) + +// NewLoader creates a configuration +// loader. +func NewLoader(opts ...LoaderOption) *Loader { + var loader Loader + _, err := toml.Decode(RulesDefault, &loader.loader) if err != nil { - log.Fatal("Failed to read in config ", err.Error()) + log.Fatal().Err(err).Msg(ErrReadConfig) } - c, err := config.Parse() + for _, opt := range opts { + opt(&loader) + } + + return &loader +} + +// WithStrict enables more strict pillager scanning. +func (l *Loader) WithStrict() LoaderOption { + return func(l *Loader) { + _, err := toml.Decode(RulesStrict, &l.loader) + if err != nil { + log.Fatal().Err(err).Msg(ErrReadConfig) + } + } +} + +// Load parses the gitleaks configuration. +func (l *Loader) Load() gitleaks.Config { + config, err := l.loader.Parse() if err != nil { - log.Fatal("Failed to parse in toml config") + log.Fatal().Err(err).Msg(ErrReadConfig) } - return c + return config +} + +// FromFile decodes the configuration +// from a local file. +func FromFile(file string) LoaderOption { + return func(l *Loader) { + _, err := toml.DecodeFile(file, &l.loader) + if err != nil { + log.Fatal().Err(err).Msg(ErrReadConfig) + } + } } diff --git a/pkg/rules/default.go b/pkg/rules/rules_simple.toml similarity index 74% rename from pkg/rules/default.go rename to pkg/rules/rules_simple.toml index e57d833..f0fc62d 100644 --- a/pkg/rules/default.go +++ b/pkg/rules/rules_simple.toml @@ -1,8 +1,3 @@ -package rules - -// DefaultConfig is the default ruleset for pillager's hunting parameters. -// This can be overridden by providing a rules.toml file as an argument. -const DefaultConfig = ` title = "pillager config" [[rules]] description = "AWS Access Key" @@ -31,7 +26,4 @@ title = "pillager config" [allowlist] description = "Allowlisted files" - files = ['''^\.?gitleaks.toml$''', - '''(.*?)(png|jpg|gif|doc|docx|pdf|bin|xls|pyc|zip)$''', - '''(go.mod|go.sum)$'''] -` + files = ['''^\.?gitleaks.toml$''', '''(.*?)(png|jpg|gif|doc|docx|pdf|bin|xls|pyc|zip)$''', '''(go.mod|go.sum)$'''] diff --git a/pkg/rules/rules_strict.toml b/pkg/rules/rules_strict.toml new file mode 100644 index 0000000..32d4a0c --- /dev/null +++ b/pkg/rules/rules_strict.toml @@ -0,0 +1,226 @@ +title = "Global gitleaks config" + +[allowlist] + description = "Allowlisted files" + files = ['''^\.?gitleaks.toml$''', # Ignoring this file + '''(.*?)(jpg|gif|doc|pdf|bin)$''', # Ignoring common binaries + '''^(.*?)_test\.go$''', # Ignoring Go test files + '''^(.*?)\.(spec|test)\.(j|t)s$''', # Ignoring JavaScript and TypeScript test files + '''(go.mod|go.sum)$''', # Ignoring Go manifests + '''vendor\.json''', + '''Gopkg\.(lock|toml)''', + '''package-lock\.json''', # Ignoring Node/JS manifests + '''package\.json''', + '''composer\.json''', + '''composer\.lock''', #Ignoring PHP manifests + '''yarn\.lock'''] + paths = ["node_modules", # Ignoring Node dependencies + "vendor", # Ignoring Go dependencies + "test", # Ignoring test directories + "tests"] + regexes = ['''test'''] # Ignoring lines with test + + +[[rules]] + description = "AWS Secret Key" + regex = '''(?i)aws(.{0,20})?(?-i)[0-9a-zA-Z\/+]{40}''' + tags = ["key", "AWS"] + +[[rules]] + description = "AWS MWS key" + regex = '''amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''' + tags = ["key", "AWS", "MWS"] + +[[rules]] + description = "Facebook Secret Key" + regex = '''(?i)(facebook|fb)(.{0,20})?(?-i)[0-9a-f]{32}''' + tags = ["key", "Facebook"] + +[[rules]] + description = "Facebook Client ID" + regex = '''(?i)(facebook|fb)(.{0,20})?[0-9]{13,17}''' + tags = ["key", "Facebook"] + +[[rules]] + description = "Twitter Secret Key" + regex = '''(?i)twitter(.{0,20})?[0-9a-z]{35,44}''' + tags = ["key", "Twitter"] + +[[rules]] + description = "Twitter Client ID" + regex = '''(?i)twitter(.{0,20})?[0-9a-z]{18,25}''' + tags = ["client", "Twitter"] + +[[rules]] + description = "Github" + regex = '''(?i)github(.{0,20})?(?-i)[0-9a-zA-Z]{35,40}''' + tags = ["key", "Github"] + +[[rules]] + description = "LinkedIn Client ID" + regex = '''(?i)linkedin(.{0,20})?(?-i)[0-9a-z]{12}''' + tags = ["client", "LinkedIn"] + +[[rules]] + description = "LinkedIn Secret Key" + regex = '''(?i)linkedin(.{0,20})?[0-9a-z]{16}''' + tags = ["secret", "LinkedIn"] + +[[rules]] + description = "Slack" + regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?''' + tags = ["key", "Slack"] + +[[rules]] + description = "Asymmetric Private Key" + regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----''' + tags = ["key", "AsymmetricPrivateKey"] + +[[rules]] + description = "Google API key" + regex = '''AIza[0-9A-Za-z\\-_]{35}''' + tags = ["key", "Google"] + +[[rules]] + description = "Google (GCP) Service Account" + regex = '''"type": "service_account"''' + tags = ["key", "Google"] + +[[rules]] + description = "Heroku API key" + regex = '''(?i)heroku(.{0,20})?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''' + tags = ["key", "Heroku"] + +[[rules]] + description = "MailChimp API key" + regex = '''(?i)(mailchimp|mc)(.{0,20})?[0-9a-f]{32}-us[0-9]{1,2}''' + tags = ["key", "Mailchimp"] + +[[rules]] + description = "Mailgun API key" + regex = '''((?i)(mailgun|mg)(.{0,20})?)?key-[0-9a-z]{32}''' + tags = ["key", "Mailgun"] + +[[rules]] + description = "PayPal Braintree access token" + regex = '''access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}''' + tags = ["key", "Paypal"] + +[[rules]] + description = "Picatic API key" + regex = '''sk_live_[0-9a-z]{32}''' + tags = ["key", "Picatic"] + +[[rules]] + description = "SendGrid API Key" + regex = '''SG\.[\w_]{16,32}\.[\w_]{16,64}''' + tags = ["key", "SendGrid"] + +[[rules]] + description = "Slack Webhook" + regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}''' + tags = ["key", "slack"] + +[[rules]] + description = "Stripe API key" + regex = '''(?i)stripe(.{0,20})?[sr]k_live_[0-9a-zA-Z]{24}''' + tags = ["key", "Stripe"] + +[[rules]] + description = "Square access token" + regex = '''sq0atp-[0-9A-Za-z\-_]{22}''' + tags = ["key", "square"] + +[[rules]] + description = "Square OAuth secret" + regex = '''sq0csp-[0-9A-Za-z\\-_]{43}''' + tags = ["key", "square"] + +[[rules]] + description = "Twilio API key" + regex = '''(?i)twilio(.{0,20})?SK[0-9a-f]{32}''' + tags = ["key", "twilio"] + +# The following rules check for credentials assigned to variables that its value has an entropy of more than 3 bits. +# To achieve this there's a regexp for each language. The regexp checks for a variable with a suspicious name followed +# by a value assignation (for example, := in Go, = in JS, etc.). Then, looks for a group of non-space characters enclosed +# between quotes. If that group has an entropy higher than 3 bits the rule will trigger. + +[[rules]] + description = "Hardcoded credentials in Go files" + file = '''^(.*?)\.go$''' + regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:\w|\s*?)(?:=|:=)(?:\s*?)[\"'`](.{4,120}?)[\"'`]''' + tags = ["credentials", "hardcoded", "go"] + [[rules.Entropies]] + Min = "3" + Max = "7" + Group = "1" + +[[rules]] + description = "Hardcoded credentials in JavaScript or TypeScript files" + file = '''^(.*?)\.(?:j|t)s$''' + regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:\w|\s*?)(?:=){1}(?:\s{0,10})[\"'`](.*?)[\"'`]''' + tags = ["credentials", "hardcoded", "js"] + [[rules.Entropies]] + Min = "3" + Max = "7" + Group = "1" + +[[rules]] + description = "Hardcoded credentials in PHP files" + file = '''^(.*?)\.php$''' + regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?:=){1}(?:.{0,10})[\"'`](.{4,120})[\"'`]''' + tags = ["credentials", "hardcoded", "php"] + [[rules.Entropies]] + Min = "3" + Max = "7" + Group = "1" + +[[rules]] + description = "Hardcoded credentials in YAML files as quoted strings" + file = '''^(.*?)\.y(a|)ml$''' + regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?::){1}(?:\s{0,10})(?:[\"'](.{4,120})[\"'])''' + tags = ["credentials", "hardcoded", "yaml"] + [[rules.Entropies]] + Min = "3" + Max = "7" + Group = "1" + [rules.allowlist] + description = "Skip YAML Serverless variables, grabbed and concated values, encrypted secrets, and values with jinja2 placeholders" + regexes = ['''\${(?:.)+}''', '''(?i)\(\((?:\s)*?(?:grab|concat)(?:.)*?(?:\s)*?\)\)''', '''(?i)!!enveloped:(?:\S)+''', '''(?:.)*?{{(?:.)*?}}'''] + +[[rules]] + description = "Hardcoded credentials in YAML files as unquoted strings" + file = '''^(.*?)\.y(a|)ml$''' + regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?::){1}(?:\s{0,10})(\S{4,120})''' + tags = ["credentials", "hardcoded", "yaml"] + [[rules.Entropies]] + Min = "3.5" # A higher entropy is required for this type of match, as unquoted can trigger many false positives + Max = "7" + Group = "1" + [rules.allowlist] + description = "Skip YAML Serverless variables, grabbed and concated values, encrypted secrets, and values with jinja2 placeholders" + regexes = ['''\${(?:.)+}''', '''(?i)\(\((?:\s)*?(?:grab|concat)(?:.)*?(?:\s)*?\)\)''', '''(?i)!!enveloped:(?:\S)+''', '''(?:.)*?{{(?:.)*?}}'''] + +[[rules]] + description = "Hardcoded credentials in YAML files as multiline strings" + file = '''^(.*?)\.y(a|)ml$''' + regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?::){1}(?:\s{0,10})(?:\|(?:-|))\n(?:\s{0,10})(\S{4,120})''' + tags = ["credentials", "hardcoded", "yaml"] + [[rules.Entropies]] + Min = "4" + Max = "7" + Group = "1" + +[[rules]] + description = "Hardcoded credentials in HCL files (*.tf)" + file = '''^(.*?)\.tf$''' + regex = '''(?i)(?:secret|key|signature|password|pwd|pass|token)(?:.{0,20})(?:=){1}(?:\s)*?"(.{4,120})"''' + tags = ["credentials", "hardcoded", "hcl"] + [[rules.Entropies]] + Min = "3" + Max = "7" + Group = "1" + [rules.allowlist] + description = "Skip variable substitution" + regexes = ['''\${(?:.)*?}'''] diff --git a/waypoint.hcl b/waypoint.hcl deleted file mode 100644 index fb78d68..0000000 --- a/waypoint.hcl +++ /dev/null @@ -1,46 +0,0 @@ -project = "pillager" - -app "cli" { - - labels = { - "maintainer" = "Britton Hayes" - "github" = "https://github.com/brittonhayes/pillager" - } - - build { - use "docker" {} - - registry { - use "docker" { - image = "bjhayes/pillager" - tag = "latest" - } - } - hook { - when = "before" - command = [ - "golangci-lint", - "run", - "./..."] - on_failure = "fail" - } - - hook { - when = "before" - command = [ - "go", - "test", - "-v", - "./..."] - on_failure = "fail" - } - } - - deploy { - use "docker" {} - } - - url { - auto_hostname = false - } -}