Project Simplification (#58)

* Large refactor of pillager application. This refactor prioritizes code
readability and modularity.

* fix: fixes lint error

* fix: more consistent logging

* fix: merged go.mod

* fix: updated goreleaser build path

* fix: update reviewdog

* fix: github action and docker build

Co-authored-by: brittonhayes <[email protected]>

.github/dependabot.yml

@@ -3,4 +3,4 @@ updates:
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"

.github/workflows/build.yml → .github/workflows/gobuild.yml

@@ -1,6 +1,6 @@
 name: build
 
-on: [ pull_request ]
+on: [pull_request]
 
 jobs:
   goreleaser:
@@ -16,9 +16,5 @@ jobs:
           go-version: 1.17
 
       - name: Build executables
-        uses: goreleaser/goreleaser-action@v2
-        with:
-          version: latest
-          args: build --rm-dist --skip-validate
-        env:
-          GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }}
+        run: |
+          go build cmd/pillager/main.go

.github/workflows/lint.yml

@@ -1,6 +1,6 @@
 name: reviewdog
 
-on: [ pull_request ]
+on: [pull_request]
 
 jobs:
   reviewdog:
@@ -10,11 +10,4 @@ jobs:
         uses: actions/checkout@v1
 
       - name: golangci-lint
-        uses: reviewdog/action-golangci-lint@v1
-
-      - name: misspell
-        uses: reviewdog/action-golangci-lint@v1
-        with:
-          golangci_lint_flags: "--disable-all -E misspell"
-          tool_name: misspell
-          level: info
+        uses: reviewdog/action-golangci-lint@v2

.github/workflows/test.yml

@@ -1,6 +1,6 @@
 name: test
 
-on: [ pull_request ]
+on: [pull_request]
 
 jobs:
   test:

.golangci.yaml

@@ -0,0 +1,15 @@
+linters:
+  enable:
+    - gofumpt
+    - gofmt
+    - gosimple
+    - godot
+    - godox
+    - dupl
+    - exhaustive
+    - funlen
+    - gocritic
+    - goprintffuncname
+    - ifshort
+  presets:
+    - unused

.goreleaser.yml

@@ -9,10 +9,12 @@ builds:
       - windows
       - darwin
     ldflags:
-      - -s -w -X github.com/brittonhayes/pillager/cmd/pillager.version={{.Version}}
+      - -s -w -X github.com/brittonhayes/pillager/internal/commands/version.version={{.Version}}
       - -X main.commit={{.Commit}}
       - -X main.date={{.Date}}
       - -X main.builtBy=goreleaser
+    main: "./cmd/pillager"
+
 archives:
   - replacements:
       darwin: Darwin
@@ -21,17 +23,17 @@ archives:
       386: i386
       amd64: x86_64
 checksum:
-  name_template: 'checksums.txt'
+  name_template: "checksums.txt"
 snapshot:
   name_template: "{{ .Tag }}-next"
 changelog:
   sort: asc
   filters:
     exclude:
-      - '^docs:'
-      - '^test:'
-      - '^images:'
-      - '^_examples:'
+      - "^docs:"
+      - "^test:"
+      - "^images:"
+      - "^_examples:"
 scoop:
   # Template for the url which is determined by the given Token (github or gitlab)
   url_template: "https://github.com/brittonhayes/pillager/releases/download/{{ .Tag }}/{{ .ArtifactName }}"
@@ -64,7 +66,7 @@ brews:
     # GOARM to specify which 32-bit arm version to use if there are multiple versions
     # from the build section. Brew formulas support atm only one 32-bit version.
     # Default is 6 for all artifacts or each id if there a multiple versions.
-    goarm: 6
+    goarm: "6"
 
     # GitHub/GitLab repository to push the formula to
     # Gitea is not supported yet, but the support coming

Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.16-alpine AS build
 
 WORKDIR /src/
 COPY . /src/
-RUN CGO_ENABLED=0 go build -o /bin/pillager
+RUN CGO_ENABLED=0 go build cmd/pillager/main.go -o /bin/pillager
 
 FROM scratch as prod
 

Makefile

@@ -0,0 +1,3 @@
+.PHONY:
+lint:
+	golangci-lint run ./...

README.md

@@ -1,6 +1,6 @@
 # Pillager
 
-![Image](./images/brand_image_ice.png)
+![Image](./logo.png)
 
 [![Go Reference](https://pkg.go.dev/badge/github.com/brittonhayes/pillager.svg)](https://pkg.go.dev/github.com/brittonhayes/pillager)
 
@@ -52,7 +52,7 @@ brew install pillager
 ### Docker Image
 
 ```
-docker run --rm -it bjhayes/pillager hunt . 
+docker run --rm -it bjhayes/pillager hunt .
 ```
 
 If you're looking for a binary, check the latest releases for the executable that matches your system
@@ -111,9 +111,7 @@ pillager hunt .
 pillager hunt ./example -f json | jq .
 ```
 
-> *JSON output is designed to work seamlessly with*
-> *the amazing [jq](https://github.com/stedolan/jq)*
-> *utility for easy parsing.*
+> _JSON output is designed to work seamlessly with_ > _the amazing [jq](https://github.com/stedolan/jq)_ > _utility for easy parsing._
 
 <details>
 <summary>Click to view more output formats</summary>
@@ -163,7 +161,6 @@ pillager hunt . -t "$(cat templates/simple.tmpl)"
 
 </details>
 
-
 ### Custom Templates
 
 Pillager allows you to use powerful `go text/template` to customize the output format. Here are a few template examples.
@@ -209,7 +206,7 @@ awesome [gomarkdoc](https://github.com/princjef/gomarkdoc) tool.
 
 **What is Cobra?**
 
-> Cobra is a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. 
+> Cobra is a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools.
 > Cobra is also an application that will generate your application scaffolding to rapidly develop a Cobra-based application.
 
 If you've seen a CLI written in Go before, there's a pretty high chance it was built with Cobra. I can't recommend this
@@ -245,5 +242,5 @@ Check out the included [rules.toml](./rules.toml) for a baseline ruleset.
 > by the MITRE ATT&CK framework.
 >
 > [MITRE ATT&CK Technique - T1552,003 - Unsecured Credentials: Bash History ](https://attack.mitre.org/techniques/T1552/003/)
-> 
+>
 > [MITRE ATT&CK Technique - T1552,001 - Unsecured Credentials: Credentials In Files](https://attack.mitre.org/techniques/T1552/001/)

_examples/hunter/main.go

@@ -1,20 +1,42 @@
 package main
 
 import (
-	"runtime"
+	"os"
 
+	"github.com/brittonhayes/pillager"
 	"github.com/brittonhayes/pillager/pkg/hunter"
-	"github.com/brittonhayes/pillager/pkg/rules"
-	"github.com/spf13/afero"
+	"github.com/rs/zerolog"
 )
 
 func main() {
-	// Create a new hunter config
-	c := hunter.NewConfig(afero.NewOsFs(), ".", true, rules.Load(""), hunter.StringToFormat("JSON"), "", runtime.NumCPU())
+	err := example()
+	if err != nil {
+		panic(err)
+	}
+}
+
+func example() error {
+	opts := []pillager.ConfigOption{
+		pillager.WithLogLevel(zerolog.DebugLevel),
+	}
 
-	// Create a new hunter from the config
-	h := hunter.NewHunter(c)
+	// Create a new hunter config
+	p, err := hunter.New(opts...)
+	if err != nil {
+		return err
+	}
 
 	// Start hunting
-	_ = h.Hunt()
+	results, err := p.Hunt()
+	if err != nil {
+		return err
+	}
+
+	// Report results
+	err = p.Report(os.Stdout, results)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }

_examples/testdata/aws_leak.txt

@@ -0,0 +1 @@
+AKIAIO5FODNN7EXAMPLE

cmd/pillager/main.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	pillager "github.com/brittonhayes/pillager/internal/commands"
+)
+
+func main() {
+	pillager.Execute()
+}

config.go

@@ -0,0 +1,141 @@
+//go:generate golangci-lint run ./...
+//go:generate gomarkdoc ./pkg/hunter/...
+//go:generate gomarkdoc ./pkg/rules/...
+//go:generate gomarkdoc ./pkg/format/...
+package pillager
+
+import (
+	"errors"
+	"os"
+	"runtime"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+
+	"github.com/brittonhayes/pillager/internal/validate"
+	"github.com/brittonhayes/pillager/pkg/format"
+	"github.com/brittonhayes/pillager/pkg/rules"
+	"github.com/spf13/afero"
+	gitleaks "github.com/zricethezav/gitleaks/v7/config"
+)
+
+// Config takes all of the configurable
+// parameters for a Hunter.
+type Config struct {
+	Filesystem afero.Fs
+	Style      format.Style
+	Gitleaks   gitleaks.Config
+
+	ScanPath string
+	Verbose  bool
+	Debug    bool
+	Workers  int
+	Template string
+}
+
+type ConfigOption func(*Config)
+
+func NewConfig(opts ...ConfigOption) *Config {
+	var (
+		defaultFS       = afero.NewOsFs()
+		defaultVerbose  = false
+		defaultScanPath = "."
+		defaultStyle    = format.StyleJSON
+		defaultWorkers  = runtime.NumCPU()
+		defaultGitleaks = rules.NewLoader().Load()
+		defaultTemplate = ""
+		defaultLogLevel = zerolog.ErrorLevel
+	)
+
+	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+	zerolog.SetGlobalLevel(defaultLogLevel)
+	config := &Config{
+		ScanPath:   defaultScanPath,
+		Filesystem: defaultFS,
+		Style:      defaultStyle,
+		Workers:    defaultWorkers,
+		Gitleaks:   defaultGitleaks,
+		Verbose:    defaultVerbose,
+		Template:   defaultTemplate,
+	}
+
+	for _, opt := range opts {
+		opt(config)
+	}
+
+	if err := config.validate(); err != nil {
+		log.Fatal().Err(err).Send()
+	}
+
+	return config
+}
+
+func WithFS(fs afero.Fs) ConfigOption {
+	return func(c *Config) {
+		c.Filesystem = fs
+	}
+}
+
+func WithScanPath(path string) ConfigOption {
+	return func(c *Config) {
+		c.ScanPath = validate.Path(c.Filesystem, c.ScanPath)
+	}
+}
+
+func WithLogLevel(level string) ConfigOption {
+	return func(c *Config) {
+		lvl, err := zerolog.ParseLevel(level)
+		if err != nil {
+			log.Fatal().Err(err).Send()
+		}
+		zerolog.SetGlobalLevel(lvl)
+	}
+}
+
+func WithVerbose(verbose bool) ConfigOption {
+	return func(c *Config) {
+		c.Verbose = verbose
+	}
+}
+
+func WithWorkers(count int) ConfigOption {
+	return func(c *Config) {
+		c.Workers = count
+	}
+}
+
+func WithStyle(style format.Style) ConfigOption {
+	return func(c *Config) {
+		if c.Template != "" {
+			c.Style = format.StyleCustom
+			return
+		}
+
+		c.Style = style
+	}
+}
+
+func WithTemplate(template string) ConfigOption {
+	return func(c *Config) {
+		c.Style = format.StyleCustom
+		c.Template = template
+	}
+}
+
+func WithGitleaksConfig(g gitleaks.Config) ConfigOption {
+	return func(c *Config) {
+		c.Gitleaks = g
+	}
+}
+
+func (c *Config) validate() error {
+	if c.Filesystem == nil {
+		return errors.New("missing filesystem in Config")
+	}
+
+	if c.Gitleaks.Rules == nil {
+		return errors.New("no gitleaks rules provided")
+	}
+
+	return nil
+}