From 5aca42213edf474a8597fbfe4f466569c0f7d2d1 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 25 Jul 2024 13:40:34 +0600 Subject: [PATCH 01/12] feat: enhance secret scanning for specific binary files From 62fda696ccd4491cb8b2273fddbc2c088b79f11d Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 1 Aug 2024 17:58:25 +0600 Subject: [PATCH 02/12] feat: add pyc files --- pkg/fanal/analyzer/secret/secret.go | 27 +++++++++++++++++++++------ pkg/fanal/utils/utils.go | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/pkg/fanal/analyzer/secret/secret.go b/pkg/fanal/analyzer/secret/secret.go index d2627a840c1b..2668d7a08828 100644 --- a/pkg/fanal/analyzer/secret/secret.go +++ b/pkg/fanal/analyzer/secret/secret.go @@ -54,6 +54,9 @@ var ( ".gz", ".gzip", ".tar", + } + + allowedExt = []string{ ".pyc", } ) @@ -63,6 +66,10 @@ func init() { analyzer.RegisterAnalyzer(NewSecretAnalyzer(secret.Scanner{}, "")) } +func isAllowedBinary(filename string) bool { + return slices.Contains(allowedExt, filepath.Ext(filename)) +} + // SecretAnalyzer is an analyzer for secrets type SecretAnalyzer struct { scanner secret.Scanner @@ -96,7 +103,7 @@ func (a *SecretAnalyzer) Init(opt analyzer.AnalyzerOptions) error { func (a *SecretAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { // Do not scan binaries binary, err := utils.IsBinary(input.Content, input.Info.Size()) - if binary || err != nil { + if err != nil || (binary && !isAllowedBinary(input.FilePath)) { return nil, nil } @@ -104,12 +111,20 @@ func (a *SecretAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput log.WithPrefix("secret").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.FilePath(input.FilePath), log.Int64("size (MB)", size/1048576)) } - content, err := io.ReadAll(input.Content) - if err != nil { - return nil, xerrors.Errorf("read error %s: %w", input.FilePath, err) - } + var content []byte - content = bytes.ReplaceAll(content, []byte("\r"), []byte("")) + if !binary { + content, err = io.ReadAll(input.Content) + if err != nil { + return nil, xerrors.Errorf("read error %s: %w", input.FilePath, err) + } + content = bytes.ReplaceAll(content, []byte("\r"), []byte("")) + } else { + content, err = utils.ExtractPrintableBytes(input.Content) + if err != nil { + return nil, xerrors.Errorf("binary read error %s: %w", input.FilePath, err) + } + } filePath := input.FilePath // Files extracted from the image have an empty input.Dir. diff --git a/pkg/fanal/utils/utils.go b/pkg/fanal/utils/utils.go index 1c5ea54b2612..66c05e1825bb 100644 --- a/pkg/fanal/utils/utils.go +++ b/pkg/fanal/utils/utils.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "unicode" xio "github.com/aquasecurity/trivy/pkg/x/io" ) @@ -93,3 +94,22 @@ func IsBinary(content xio.ReadSeekerAt, fileSize int64) (bool, error) { return false, nil } + +func ExtractPrintableBytes(content xio.ReadSeekerAt) ([]byte, error) { + var printalbe []byte + current := make([]byte, 1) + + for { + _, err := content.Read(current) + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + if unicode.IsPrint(rune(current[0])) { + printalbe = append(printalbe, current[0]) + } + } + + return printalbe, nil +} From 910f84e25fd5627e2e331d7b01a2c18b7d7a3885 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 1 Aug 2024 18:23:16 +0600 Subject: [PATCH 03/12] refactor: rename array of allowed extensions --- pkg/fanal/analyzer/secret/secret.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/fanal/analyzer/secret/secret.go b/pkg/fanal/analyzer/secret/secret.go index 2668d7a08828..621eb2694b9a 100644 --- a/pkg/fanal/analyzer/secret/secret.go +++ b/pkg/fanal/analyzer/secret/secret.go @@ -56,7 +56,7 @@ var ( ".tar", } - allowedExt = []string{ + allowedBinaries = []string{ ".pyc", } ) @@ -67,7 +67,7 @@ func init() { } func isAllowedBinary(filename string) bool { - return slices.Contains(allowedExt, filepath.Ext(filename)) + return slices.Contains(allowedBinaries, filepath.Ext(filename)) } // SecretAnalyzer is an analyzer for secrets From b9a8a58c9086b93d5c350a616852b825ec90215c Mon Sep 17 00:00:00 2001 From: afdesk Date: Wed, 14 Aug 2024 10:42:20 +0600 Subject: [PATCH 04/12] skip unprintable chars --- pkg/fanal/utils/utils.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/fanal/utils/utils.go b/pkg/fanal/utils/utils.go index 66c05e1825bb..8df8297817ff 100644 --- a/pkg/fanal/utils/utils.go +++ b/pkg/fanal/utils/utils.go @@ -99,6 +99,8 @@ func ExtractPrintableBytes(content xio.ReadSeekerAt) ([]byte, error) { var printalbe []byte current := make([]byte, 1) + wasReadable := false + for { _, err := content.Read(current) if err == io.EOF { @@ -107,8 +109,15 @@ func ExtractPrintableBytes(content xio.ReadSeekerAt) ([]byte, error) { return nil, err } if unicode.IsPrint(rune(current[0])) { + if !wasReadable { + printalbe = append(printalbe, byte('\n')) + wasReadable = true + } printalbe = append(printalbe, current[0]) + } else { + wasReadable = false } + } return printalbe, nil From ee234ad8e582a438192f26adce76aa2c5bbfd69b Mon Sep 17 00:00:00 2001 From: afdesk Date: Wed, 14 Aug 2024 11:41:59 +0600 Subject: [PATCH 05/12] splite unprintable chars with spaces --- pkg/fanal/utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fanal/utils/utils.go b/pkg/fanal/utils/utils.go index 8df8297817ff..80917371a1dc 100644 --- a/pkg/fanal/utils/utils.go +++ b/pkg/fanal/utils/utils.go @@ -110,7 +110,7 @@ func ExtractPrintableBytes(content xio.ReadSeekerAt) ([]byte, error) { } if unicode.IsPrint(rune(current[0])) { if !wasReadable { - printalbe = append(printalbe, byte('\n')) + printalbe = append(printalbe, byte(' ')) wasReadable = true } printalbe = append(printalbe, current[0]) From 06cb7a028d2214001a7f6f0ed5ba04403802f20e Mon Sep 17 00:00:00 2001 From: afdesk Date: Wed, 14 Aug 2024 11:42:19 +0600 Subject: [PATCH 06/12] test: add python binary --- pkg/fanal/analyzer/secret/secret_test.go | 39 ++++++++++++++++++ .../secret/testdata/secret.cpython-310.pyc | Bin 0 -> 226 bytes 2 files changed, 39 insertions(+) create mode 100755 pkg/fanal/analyzer/secret/testdata/secret.cpython-310.pyc diff --git a/pkg/fanal/analyzer/secret/secret_test.go b/pkg/fanal/analyzer/secret/secret_test.go index 7cba1d137e8f..ae5fdd23cd08 100644 --- a/pkg/fanal/analyzer/secret/secret_test.go +++ b/pkg/fanal/analyzer/secret/secret_test.go @@ -95,6 +95,30 @@ func TestSecretAnalyzer(t *testing.T) { }, }, } + wantFindingGH_PAT := types.SecretFinding{ + RuleID: "github-fine-grained-pat", + Category: "GitHub", + Title: "GitHub Fine-grained personal access tokens", + Severity: "CRITICAL", + StartLine: 1, + EndLine: 1, + Match: " o e\xabfx \xe3 @ s d Z e e d S ) \xda]*********************************************************************************************N) \xda secret1\xda print\xa9", + Code: types.Code{ + Lines: []types.Line{ + { + Number: 1, + Content: " o e\xabfx \xe3 @ s d Z e e d S ) \xda]*********************************************************************************************N) \xda secret1\xda print\xa9", + IsCause: true, + Annotation: "", + Truncated: false, + Highlighted: " o e\xabfx \xe3 @ s d Z e e d S ) \xda]*********************************************************************************************N) \xda secret1\xda print\xa9", + FirstCause: true, + LastCause: true, + }, + }, + }, + } + tests := []struct { name string configPath string @@ -153,6 +177,21 @@ func TestSecretAnalyzer(t *testing.T) { filePath: "testdata/binaryfile", want: nil, }, + { + name: "python binary file", + configPath: "testdata/skip-tests-config.yaml", + filePath: "testdata/secret.cpython-310.pyc", + want: &analyzer.AnalysisResult{ + Secrets: []types.Secret{ + { + FilePath: "/testdata/secret.cpython-310.pyc", + Findings: []types.SecretFinding{ + wantFindingGH_PAT, + }, + }, + }, + }, + }, } for _, tt := range tests { diff --git a/pkg/fanal/analyzer/secret/testdata/secret.cpython-310.pyc b/pkg/fanal/analyzer/secret/testdata/secret.cpython-310.pyc new file mode 100755 index 0000000000000000000000000000000000000000..2bb659e7eb21abd1fed3da8d1f085600619d6f4b GIT binary patch literal 226 zcmd1j<>g`kf~l#i(<*@UV-N=!FabFZKwK;UBvKfn7*ZKi8JZax8B!R788n%0#inPL zWRxbw7bKR%8yY&fxVreddm0qydZb1gR(g7bC%VKZd6tJ1dsQZeB)a>Ag{D`fx>Xh# zI;KWsS45O#Mmjr(W*3&arFy3MmPa}Hc!rvIxca9?IfeN+R+jkzZD21>O)g3;F}%fE tP?VWhvXY^Q1?Uto@rzqeA0n$)P Date: Wed, 14 Aug 2024 12:16:07 +0600 Subject: [PATCH 07/12] update docs --- docs/docs/scanner/secret.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/docs/scanner/secret.md b/docs/docs/scanner/secret.md index 49d8c8488ece..ea74ae64c930 100644 --- a/docs/docs/scanner/secret.md +++ b/docs/docs/scanner/secret.md @@ -3,7 +3,9 @@ Trivy scans any container image, filesystem and git repository to detect exposed secrets like passwords, api keys, and tokens. Secret scanning is enabled by default. -Trivy will scan every plaintext file, according to builtin rules or configuration. There are plenty of builtin rules: +Trivy will scan every plaintext file, according to builtin rules or configuration. Also, Trivy can detect secrets in compiled Python files (`.pyc`). + +There are plenty of builtin rules: - AWS access key - GCP service account From 9619cf7c4c07407ccc2748f19cb5b7733f9b0e92 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 22 Aug 2024 13:27:39 +0600 Subject: [PATCH 08/12] skip printing binary files --- pkg/fanal/analyzer/secret/secret.go | 1 + pkg/fanal/analyzer/secret/secret_test.go | 16 +--------------- pkg/fanal/secret/scanner.go | 19 +++++++++++++++++-- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pkg/fanal/analyzer/secret/secret.go b/pkg/fanal/analyzer/secret/secret.go index 621eb2694b9a..da90ee507403 100644 --- a/pkg/fanal/analyzer/secret/secret.go +++ b/pkg/fanal/analyzer/secret/secret.go @@ -137,6 +137,7 @@ func (a *SecretAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput result := a.scanner.Scan(secret.ScanArgs{ FilePath: filePath, Content: content, + Binary: binary, }) if len(result.Findings) == 0 { diff --git a/pkg/fanal/analyzer/secret/secret_test.go b/pkg/fanal/analyzer/secret/secret_test.go index ae5fdd23cd08..5187ebc341bf 100644 --- a/pkg/fanal/analyzer/secret/secret_test.go +++ b/pkg/fanal/analyzer/secret/secret_test.go @@ -102,21 +102,7 @@ func TestSecretAnalyzer(t *testing.T) { Severity: "CRITICAL", StartLine: 1, EndLine: 1, - Match: " o e\xabfx \xe3 @ s d Z e e d S ) \xda]*********************************************************************************************N) \xda secret1\xda print\xa9", - Code: types.Code{ - Lines: []types.Line{ - { - Number: 1, - Content: " o e\xabfx \xe3 @ s d Z e e d S ) \xda]*********************************************************************************************N) \xda secret1\xda print\xa9", - IsCause: true, - Annotation: "", - Truncated: false, - Highlighted: " o e\xabfx \xe3 @ s d Z e e d S ) \xda]*********************************************************************************************N) \xda secret1\xda print\xa9", - FirstCause: true, - LastCause: true, - }, - }, - }, + Match: "Binary file \"/testdata/secret.cpython-310.pyc\" matches a rule \"GitHub Fine-grained personal access tokens\"", } tests := []struct { diff --git a/pkg/fanal/secret/scanner.go b/pkg/fanal/secret/scanner.go index b8591a69228c..e59ee61f7915 100644 --- a/pkg/fanal/secret/scanner.go +++ b/pkg/fanal/secret/scanner.go @@ -366,6 +366,7 @@ func NewScanner(config *Config) Scanner { type ScanArgs struct { FilePath string Content []byte + Binary bool } type Match struct { @@ -435,8 +436,22 @@ func (s *Scanner) Scan(args ScanArgs) types.Secret { } } - for _, match := range matched { - findings = append(findings, toFinding(match.Rule, match.Location, censored)) + if args.Binary { + for _, match := range matched { + findings = append(findings, types.SecretFinding{ + RuleID: match.Rule.ID, + Category: match.Rule.Category, + Severity: lo.Ternary(match.Rule.Severity == "", "UNKNOWN", match.Rule.Severity), + Title: match.Rule.Title, + Match: fmt.Sprintf("Binary file %q matches a rule %q", args.FilePath, match.Rule.Title), + StartLine: 1, + EndLine: 1, + }) + } + } else { + for _, match := range matched { + findings = append(findings, toFinding(match.Rule, match.Location, censored)) + } } if len(findings) == 0 { From 965a73db342dca65744bb22d440e9ff468b787be Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 22 Aug 2024 13:33:17 +0600 Subject: [PATCH 09/12] read printable chars by line with minimum length --- pkg/fanal/utils/utils.go | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/pkg/fanal/utils/utils.go b/pkg/fanal/utils/utils.go index 80917371a1dc..863a2674b16c 100644 --- a/pkg/fanal/utils/utils.go +++ b/pkg/fanal/utils/utils.go @@ -96,10 +96,11 @@ func IsBinary(content xio.ReadSeekerAt, fileSize int64) (bool, error) { } func ExtractPrintableBytes(content xio.ReadSeekerAt) ([]byte, error) { - var printalbe []byte - current := make([]byte, 1) + const minLength = 4 // Minimum length of strings to extract + var result []byte + var currentPrintableLine []byte - wasReadable := false + current := make([]byte, 1) // buffer for 1 byte reading for { _, err := content.Read(current) @@ -109,16 +110,20 @@ func ExtractPrintableBytes(content xio.ReadSeekerAt) ([]byte, error) { return nil, err } if unicode.IsPrint(rune(current[0])) { - if !wasReadable { - printalbe = append(printalbe, byte(' ')) - wasReadable = true - } - printalbe = append(printalbe, current[0]) - } else { - wasReadable = false + currentPrintableLine = append(currentPrintableLine, current[0]) + continue } - + if len(currentPrintableLine) > minLength { + // add a space between printable lines to separate them + currentPrintableLine = append(currentPrintableLine, ' ') + result = append(result, currentPrintableLine...) + } + currentPrintableLine = nil } - - return printalbe, nil + if len(currentPrintableLine) > minLength { + // add a space between printable lines to separate them + currentPrintableLine = append(currentPrintableLine, ' ') + result = append(result, currentPrintableLine...) + } + return result, nil } From 77d8c3ee5d5edb27bceb730191c437c1d76bcb14 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 29 Aug 2024 12:45:29 +0600 Subject: [PATCH 10/12] refactor: rename --- pkg/fanal/analyzer/secret/secret.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/fanal/analyzer/secret/secret.go b/pkg/fanal/analyzer/secret/secret.go index da90ee507403..bdb3e96d1428 100644 --- a/pkg/fanal/analyzer/secret/secret.go +++ b/pkg/fanal/analyzer/secret/secret.go @@ -66,7 +66,7 @@ func init() { analyzer.RegisterAnalyzer(NewSecretAnalyzer(secret.Scanner{}, "")) } -func isAllowedBinary(filename string) bool { +func allowedBinary(filename string) bool { return slices.Contains(allowedBinaries, filepath.Ext(filename)) } @@ -103,7 +103,7 @@ func (a *SecretAnalyzer) Init(opt analyzer.AnalyzerOptions) error { func (a *SecretAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { // Do not scan binaries binary, err := utils.IsBinary(input.Content, input.Info.Size()) - if err != nil || (binary && !isAllowedBinary(input.FilePath)) { + if err != nil || (binary && !allowedBinary(input.FilePath)) { return nil, nil } From d12e32c3b58069d5719e531e263864d4ef7e339b Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 29 Aug 2024 12:49:23 +0600 Subject: [PATCH 11/12] refactor: remove spagetti code --- pkg/fanal/secret/scanner.go | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/pkg/fanal/secret/scanner.go b/pkg/fanal/secret/scanner.go index e59ee61f7915..274ef9f772f3 100644 --- a/pkg/fanal/secret/scanner.go +++ b/pkg/fanal/secret/scanner.go @@ -435,23 +435,14 @@ func (s *Scanner) Scan(args ScanArgs) types.Secret { censored = censorLocation(loc, censored) } } - - if args.Binary { - for _, match := range matched { - findings = append(findings, types.SecretFinding{ - RuleID: match.Rule.ID, - Category: match.Rule.Category, - Severity: lo.Ternary(match.Rule.Severity == "", "UNKNOWN", match.Rule.Severity), - Title: match.Rule.Title, - Match: fmt.Sprintf("Binary file %q matches a rule %q", args.FilePath, match.Rule.Title), - StartLine: 1, - EndLine: 1, - }) - } - } else { - for _, match := range matched { - findings = append(findings, toFinding(match.Rule, match.Location, censored)) + for _, match := range matched { + finding := toFinding(match.Rule, match.Location, censored) + // Rewrite unreadable fields for binary files + if args.Binary { + finding.Match = fmt.Sprintf("Binary file %q matches a rule %q", args.FilePath, match.Rule.Title) + finding.Code = types.Code{} } + findings = append(findings, finding) } if len(findings) == 0 { From 906a59b95c15f76f3812b46cbc90e06e3da3bc2e Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 30 Sep 2024 16:04:21 +0400 Subject: [PATCH 12/12] refactor: use bytes.Buffer Signed-off-by: knqyf263 --- pkg/fanal/utils/utils.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/pkg/fanal/utils/utils.go b/pkg/fanal/utils/utils.go index 863a2674b16c..58b44510db89 100644 --- a/pkg/fanal/utils/utils.go +++ b/pkg/fanal/utils/utils.go @@ -2,6 +2,7 @@ package utils import ( "bufio" + "bytes" "fmt" "io" "math" @@ -10,6 +11,8 @@ import ( "path/filepath" "unicode" + "golang.org/x/xerrors" + xio "github.com/aquasecurity/trivy/pkg/x/io" ) @@ -98,32 +101,33 @@ func IsBinary(content xio.ReadSeekerAt, fileSize int64) (bool, error) { func ExtractPrintableBytes(content xio.ReadSeekerAt) ([]byte, error) { const minLength = 4 // Minimum length of strings to extract var result []byte - var currentPrintableLine []byte + currentPrintableLine := new(bytes.Buffer) current := make([]byte, 1) // buffer for 1 byte reading for { - _, err := content.Read(current) - if err == io.EOF { + if n, err := content.Read(current); err == io.EOF { break + } else if n != 1 { + continue } else if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to read a byte: %w", err) } if unicode.IsPrint(rune(current[0])) { - currentPrintableLine = append(currentPrintableLine, current[0]) + _ = currentPrintableLine.WriteByte(current[0]) continue } - if len(currentPrintableLine) > minLength { - // add a space between printable lines to separate them - currentPrintableLine = append(currentPrintableLine, ' ') - result = append(result, currentPrintableLine...) + if currentPrintableLine.Len() > minLength { + // add a newline between printable lines to separate them + _ = currentPrintableLine.WriteByte('\n') + result = append(result, currentPrintableLine.Bytes()...) } - currentPrintableLine = nil + currentPrintableLine.Reset() } - if len(currentPrintableLine) > minLength { - // add a space between printable lines to separate them - currentPrintableLine = append(currentPrintableLine, ' ') - result = append(result, currentPrintableLine...) + if currentPrintableLine.Len() > minLength { + // add a newline between printable lines to separate them + _ = currentPrintableLine.WriteByte('\n') + result = append(result, currentPrintableLine.Bytes()...) } return result, nil }