Initial CLI w/ partially implemented Git source and demo detector (#1)

Makefile

@@ -0,0 +1,32 @@
+PROTOS_IMAGE=us-docker.pkg.dev/thog-artifacts/public/go-ci-1.17-1
+
+.PHONY: check
+.PHONY: test
+.PHONY: test-race
+.PHONY: run
+.PHONY: install
+.PHONY: protos
+.PHONY: protos-windows
+.PHONY: vendor
+
+install:
+	CGO_ENABLED=0 go install .
+
+check:
+	go fmt $(shell go list ./... | grep -v /vendor/)
+	go vet $(shell go list ./... | grep -v /vendor/)
+
+test:
+	CGO_ENABLED=0 go test $(shell go list ./... | grep -v /vendor/)
+
+test-race:
+	CGO_ENABLED=1 go test -race $(shell go list ./... | grep -v /vendor/)
+
+bench:
+	CGO_ENABLED=0 go test $(shell go list ./pkg/secrets/... | grep -v /vendor/) -benchmem -run=xxx -bench .
+
+run:
+	CGO_ENABLED=0 go run . git file://.
+
+protos:
+	docker run -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"

go.mod

@@ -0,0 +1,67 @@
+module github.com/trufflesecurity/trufflehog
+
+go 1.17
+
+require (
+	cloud.google.com/go/secretmanager v1.0.0
+	github.com/bradleyfalzon/ghinstallation v1.1.1
+	github.com/envoyproxy/protoc-gen-validate v0.6.2
+	github.com/go-errors/errors v1.4.1
+	github.com/go-git/go-git/v5 v5.4.2
+	github.com/google/go-github/v41 v41.0.0
+	github.com/h2non/filetype v1.1.3
+	github.com/hashicorp/go-retryablehttp v0.7.0
+	github.com/joho/godotenv v1.4.0
+	github.com/kylelemons/godebug v1.1.0
+	github.com/mattn/go-colorable v0.1.12
+	github.com/pkg/errors v0.9.1
+	github.com/sirupsen/logrus v1.8.1
+	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
+	github.com/xanzy/go-gitlab v0.54.3
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368
+	google.golang.org/protobuf v1.27.1
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6
+)
+
+require (
+	cloud.google.com/go v0.94.1 // indirect
+	github.com/Microsoft/go-winio v0.4.16 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
+	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/go-github/v29 v29.0.2 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/xanzy/ssh-agent v0.3.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/mod v0.5.0 // indirect
+	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect
+	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	golang.org/x/tools v0.1.5 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/api v0.57.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/grpc v1.40.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+)

main.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"strconv"
+
+	"github.com/sirupsen/logrus"
+	"github.com/trufflesecurity/trufflehog/pkg/decoders"
+	"github.com/trufflesecurity/trufflehog/pkg/engine"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
+)
+
+func main() {
+
+	cli := kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
+	debug := cli.Flag("debug", "Run in debug mode").Bool()
+	jsonOut := cli.Flag("json", "Output in JSON format.").Bool()
+	concurrency := cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
+	verification := cli.Flag("verification", "Verify the results.").Bool()
+	// rules := cli.Flag("rules", "Path to file with custom rules.").String()
+
+	gitScan := cli.Command("git", "Find credentials in git repositories.")
+	gitScanURI := gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()
+	// gitScanIncludePaths := gitScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+	// gitScanExcludePaths := gitScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+	// gitScanSinceCommit := gitScan.Flag("since_commit", "Commit to start scan from.").String()
+	gitScanBranch := gitScan.Flag("branch", "Branch to scan.").String()
+	// gitScanMaxDepth := gitScan.Flag("max_depth", "Maximum depth of commits to scan.").Int()
+	gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
+	gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
+	gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
+
+	githubScan := cli.Command("github", "Find credentials in GitHub repositories.")
+	// githubScanTarget := githubScan.Arg("target", "GitHub target. Can be a repository, user or organization.").Required().String()
+	// githubScanToken := githubScan.Flag("token", "GitHub token.").String()
+
+	gitlabScan := cli.Command("gitlab", "Find credentials in GitLab repositories.")
+	// gitlabScanTarget := gitlabScan.Arg("target", "GitLab target. Can be a repository, user or organization.").Required().String()
+	// gitlabScanToken := gitlabScan.Flag("token", "GitLab token.").String()
+
+	// bitbucketScan := cli.Command("bitbucket", "Find credentials in Bitbucket repositories.")
+	// bitbucketScanTarget := bitbucketScan.Arg("target", "Bitbucket target. Can be a repository, user or organization.").Required().String()
+	// bitbucketScanToken := bitbucketScan.Flag("token", "Bitbucket token.").String()
+
+	// filesystemScan := cli.Command("filesystem", "Find credentials in filesystem.")
+	// filesystemScanPath := filesystemScan.Arg("path", "Path to scan.").Required().String()
+	// filesystemScanRecursive := filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
+	// filesystemScanIncludePaths := filesystemScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+	// filesystemScanExcludePaths := filesystemScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+
+	cmd := kingpin.MustParse(cli.Parse(os.Args[1:]))
+
+	if *jsonOut {
+		logrus.SetFormatter(&logrus.JSONFormatter{})
+	}
+	if *debug {
+		logrus.SetLevel(logrus.DebugLevel)
+	} else {
+		logrus.SetLevel(logrus.InfoLevel)
+	}
+
+	ctx := context.TODO()
+	e := engine.Start(ctx,
+		engine.WithConcurrency(*concurrency),
+		engine.WithDecoders(decoders.DefaultDecoders()...),
+		engine.WithDetectors(*verification, engine.DefaultDetectors()...),
+	)
+
+	switch cmd {
+	case gitScan.FullCommand():
+		err := e.ScanGit(ctx, *gitScanURI, *gitScanBranch, "HEAD")
+		if err != nil {
+			logrus.WithError(err).Fatal("Failed to scan git.")
+		}
+	case githubScan.FullCommand():
+		log.Fatal("github not implemented")
+	case gitlabScan.FullCommand():
+		log.Fatal("gitlab not implemented")
+	}
+
+	// deal with the results from e.ResultsChan()
+	for r := range e.ResultsChan() {
+		if *jsonOut {
+			// todo - add parity to trufflehog's existing output for git
+			// source
+			out, err := json.Marshal(r)
+			if err != nil {
+				logrus.WithError(err).Fatal("could not marshal result")
+			}
+			fmt.Println(string(out))
+		} else {
+			fmt.Printf("%+v\n", r)
+		}
+	}
+
+}

pkg/common/depaware.go

@@ -0,0 +1,5 @@
+package common
+
+import (
+	_ "github.com/tailscale/depaware/depaware"
+)

pkg/common/http.go

@@ -0,0 +1,130 @@
+package common
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	retryablehttp "github.com/hashicorp/go-retryablehttp"
+)
+
+var caCerts = []string{
+	// 	CN = ISRG Root X1
+	// TODO: Expires Monday, June 4, 2035 at 4:04:38 AM Pacific
+	`
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----	
+`,
+	// 	CN = ISRG Root X2
+	// TODO: Expires September 17, 2040 at 9:00:00 AM Pacific Daylight Time
+	`
+-----BEGIN CERTIFICATE-----
+MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw
+CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg
+R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00
+MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT
+ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw
+EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW
++1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9
+ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T
+AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI
+zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW
+tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1
+/q4AaOeMSQ+2b1tbFfLn
+-----END CERTIFICATE-----
+`,
+}
+
+func PinnedCertPool() *x509.CertPool {
+	trustedCerts := x509.NewCertPool()
+	for _, cert := range caCerts {
+		trustedCerts.AppendCertsFromPEM([]byte(strings.TrimSpace(cert)))
+	}
+	return trustedCerts
+}
+
+type CustomTransport struct {
+	T http.RoundTripper
+}
+
+func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.Header.Add("User-Agent", "TruffleHog")
+	return t.T.RoundTrip(req)
+}
+
+func NewCustomTransport(T http.RoundTripper) *CustomTransport {
+	if T == nil {
+		T = http.DefaultTransport
+	}
+	return &CustomTransport{T}
+}
+
+func PinnedRetryableHttpClient() *http.Client {
+	httpClient := retryablehttp.NewClient()
+	httpClient.Logger = nil
+	httpClient.HTTPClient.Transport = NewCustomTransport(&http.Transport{
+		TLSClientConfig: &tls.Config{
+			RootCAs: PinnedCertPool(),
+		},
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	})
+	return httpClient.StandardClient()
+}
+
+func RetryableHttpClient() *http.Client {
+	httpClient := retryablehttp.NewClient()
+	httpClient.RetryMax = 3
+	httpClient.Logger = nil
+	httpClient.HTTPClient.Timeout = 3 * time.Second
+	httpClient.HTTPClient.Transport = NewCustomTransport(nil)
+	return httpClient.StandardClient()
+}
+
+func SaneHttpClient() *http.Client {
+	httpClient := &http.Client{}
+	httpClient.Timeout = time.Second * 2
+	httpClient.Transport = NewCustomTransport(nil)
+	return httpClient
+}

pkg/common/secrets.go

@@ -0,0 +1,55 @@
+package common
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	secretmanager "cloud.google.com/go/secretmanager/apiv1"
+	"github.com/joho/godotenv"
+	"github.com/pkg/errors"
+	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
+)
+
+type Secret struct{ kv map[string]string }
+
+func (s *Secret) MustGetField(name string) string {
+	val, ok := s.kv[name]
+	if !ok {
+		panic(errors.Errorf("field %s not found", name))
+	}
+	return val
+}
+
+func GetTestSecret(ctx context.Context) (secret *Secret, err error) {
+	return GetSecret(ctx, "trufflehog-testing", "test")
+}
+
+func GetSecret(ctx context.Context, gcpProject, name string) (secret *Secret, err error) {
+	ctx, cancel := context.WithTimeout(ctx, time.Second*10)
+	defer cancel()
+
+	parent := fmt.Sprintf("projects/%s/secrets/%s/versions/latest", gcpProject, name)
+
+	client, err := secretmanager.NewClient(ctx)
+	if err != nil {
+		return nil, errors.Errorf("failed to create secretmanager client: %v", err)
+	}
+	defer client.Close()
+
+	req := &secretmanagerpb.AccessSecretVersionRequest{
+		Name: parent,
+	}
+
+	result, err := client.AccessSecretVersion(ctx, req)
+	if err != nil {
+		return nil, errors.Errorf("failed to access secret version: %v", err)
+	}
+
+	data, err := godotenv.Unmarshal(string(result.Payload.Data))
+	if err != nil {
+		return nil, err
+	}
+
+	return &Secret{kv: data}, nil
+}