From ace262159929ba9dfffa3a0648289e02390e8608 Mon Sep 17 00:00:00 2001 From: afdesk Date: Fri, 16 Aug 2024 15:00:32 +0600 Subject: [PATCH 01/11] feat(pip): use minimum version in range --- pkg/dependency/parser/python/pip/parse.go | 38 +++++++++++++++++-- pkg/fanal/analyzer/language/python/pip/pip.go | 16 +++++--- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/pkg/dependency/parser/python/pip/parse.go b/pkg/dependency/parser/python/pip/parse.go index 0d9e040f952b..0b4b3f7b72f4 100644 --- a/pkg/dependency/parser/python/pip/parse.go +++ b/pkg/dependency/parser/python/pip/parse.go @@ -25,15 +25,38 @@ const ( ) type Parser struct { - logger *log.Logger + logger *log.Logger + useMinVersion bool } -func NewParser() *Parser { +func NewParser(useMinVersion bool) *Parser { return &Parser{ - logger: log.WithPrefix("pip"), + logger: log.WithPrefix("pip"), + useMinVersion: useMinVersion, } } +/* +~= 2.2 +>= 2.2, == 2.* + +~= 1.4.5 +>= 1.4.5, == 1.4.* +*/ + +func splitLine(line string) []string { + result := strings.Split(line, "~=") + if len(result) == 2 { + return result + } + result = strings.Split(line, ">=") + if len(result) == 2 { + return result + } + + return strings.Split(line, "==") +} + func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) { // `requirements.txt` can use byte order marks (BOM) // e.g. on Windows `requirements.txt` can use UTF-16LE with BOM @@ -53,7 +76,14 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc line = rStripByKey(line, commentMarker) line = rStripByKey(line, endColon) line = rStripByKey(line, hashMarker) - s := strings.Split(line, "==") + var s []string + + if p.useMinVersion { + s = splitLine(line) + } else { + s = strings.Split(line, "==") + } + if len(s) != 2 { continue } diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index e08a90e7a70b..670dd195bb50 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -38,14 +38,16 @@ var pythonExecNames = []string{ } type pipLibraryAnalyzer struct { - logger *log.Logger - metadataParser packaging.Parser + logger *log.Logger + metadataParser packaging.Parser + detectionPriority types.DetectionPriority } -func newPipLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { +func newPipLibraryAnalyzer(opts analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { return pipLibraryAnalyzer{ - logger: log.WithPrefix("pip"), - metadataParser: *packaging.NewParser(), + logger: log.WithPrefix("pip"), + metadataParser: *packaging.NewParser(), + detectionPriority: opts.DetectionPriority, }, nil } @@ -62,8 +64,10 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn return true } + useMinVersion := a.detectionPriority == types.PriorityComprehensive + if err = fsutils.WalkDir(input.FS, ".", required, func(pathPath string, d fs.DirEntry, r io.Reader) error { - app, err := language.Parse(types.Pip, pathPath, r, pip.NewParser()) + app, err := language.Parse(types.Pip, pathPath, r, pip.NewParser(useMinVersion)) if err != nil { return xerrors.Errorf("unable to parse requirements.txt: %w", err) } From c2c20663058af1df8e53a6c63fd71166d0b242b0 Mon Sep 17 00:00:00 2001 From: afdesk Date: Tue, 20 Aug 2024 07:28:39 +0600 Subject: [PATCH 02/11] fix: tests --- pkg/dependency/parser/python/pip/parse_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/dependency/parser/python/pip/parse_test.go b/pkg/dependency/parser/python/pip/parse_test.go index 3a13c5272cc8..4a1137409976 100644 --- a/pkg/dependency/parser/python/pip/parse_test.go +++ b/pkg/dependency/parser/python/pip/parse_test.go @@ -12,9 +12,10 @@ import ( func TestParse(t *testing.T) { tests := []struct { - name string - filePath string - want []ftypes.Package + name string + filePath string + useMinVersion bool + want []ftypes.Package }{ { name: "happy path", @@ -73,7 +74,7 @@ func TestParse(t *testing.T) { f, err := os.Open(tt.filePath) require.NoError(t, err) - got, _, err := NewParser().Parse(f) + got, _, err := NewParser(tt.useMinVersion).Parse(f) require.NoError(t, err) assert.Equal(t, tt.want, got) From 6b38903282be5d7a5c09297eaf7ef8455c1132c5 Mon Sep 17 00:00:00 2001 From: afdesk Date: Tue, 20 Aug 2024 09:43:39 +0600 Subject: [PATCH 03/11] test: add a test case --- .../parser/python/pip/parse_test.go | 6 +++++ .../parser/python/pip/parse_testcase.go | 22 +++++++++++++++++++ .../pip/testdata/requirements_compatible.txt | 2 ++ 3 files changed, 30 insertions(+) create mode 100644 pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt diff --git a/pkg/dependency/parser/python/pip/parse_test.go b/pkg/dependency/parser/python/pip/parse_test.go index 4a1137409976..97964ea77a4a 100644 --- a/pkg/dependency/parser/python/pip/parse_test.go +++ b/pkg/dependency/parser/python/pip/parse_test.go @@ -67,6 +67,12 @@ func TestParse(t *testing.T) { filePath: "testdata/requirements_with_templating_engine.txt", want: nil, }, + { + name: "compatible versions", + filePath: "testdata/requirements_compatible.txt", + useMinVersion: true, + want: requirementsCompatibleVersions, + }, } for _, tt := range tests { diff --git a/pkg/dependency/parser/python/pip/parse_testcase.go b/pkg/dependency/parser/python/pip/parse_testcase.go index e8192ee1775d..c84769e6f48d 100644 --- a/pkg/dependency/parser/python/pip/parse_testcase.go +++ b/pkg/dependency/parser/python/pip/parse_testcase.go @@ -3,6 +3,28 @@ package pip import ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" var ( + requirementsCompatibleVersions = []ftypes.Package{ + { + Name: "keyring", + Version: "4.1.1", + Locations: []ftypes.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, + }, + { + Name: "Mopidy-Dirble", + Version: "1.1", + Locations: []ftypes.Location{ + { + StartLine: 2, + EndLine: 2, + }, + }, + }, + } requirementsFlask = []ftypes.Package{ { Name: "click", diff --git a/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt b/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt new file mode 100644 index 000000000000..fd7b0545c6d2 --- /dev/null +++ b/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt @@ -0,0 +1,2 @@ +keyring >= 4.1.1 # Minimum version 4.1.1 +Mopidy-Dirble ~= 1.1 # Compatible release. Same as >= 1.1, == 1.* \ No newline at end of file From a837b339f78e6403f5e5ab68f33d55482df7a307 Mon Sep 17 00:00:00 2001 From: afdesk Date: Tue, 20 Aug 2024 09:53:17 +0600 Subject: [PATCH 04/11] remove dirty comments --- pkg/dependency/parser/python/pip/parse.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/dependency/parser/python/pip/parse.go b/pkg/dependency/parser/python/pip/parse.go index 0b4b3f7b72f4..02e1d75ec5d1 100644 --- a/pkg/dependency/parser/python/pip/parse.go +++ b/pkg/dependency/parser/python/pip/parse.go @@ -36,14 +36,6 @@ func NewParser(useMinVersion bool) *Parser { } } -/* -~= 2.2 ->= 2.2, == 2.* - -~= 1.4.5 ->= 1.4.5, == 1.4.* -*/ - func splitLine(line string) []string { result := strings.Split(line, "~=") if len(result) == 2 { From 7c849cea9237f40210e98b1bf886d49521a67944 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 22 Aug 2024 09:20:47 +0600 Subject: [PATCH 05/11] refactor splitting --- pkg/dependency/parser/python/pip/parse.go | 28 +++++++++-------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/pkg/dependency/parser/python/pip/parse.go b/pkg/dependency/parser/python/pip/parse.go index 02e1d75ec5d1..08d76b919cfc 100644 --- a/pkg/dependency/parser/python/pip/parse.go +++ b/pkg/dependency/parser/python/pip/parse.go @@ -35,18 +35,18 @@ func NewParser(useMinVersion bool) *Parser { useMinVersion: useMinVersion, } } - -func splitLine(line string) []string { - result := strings.Split(line, "~=") - if len(result) == 2 { - return result +func (p *Parser) splitLine(line string) []string { + separators := []string{"~=", ">=", "=="} + // Without useMinVersion check only `==` + if !p.useMinVersion { + separators = []string{"=="} } - result = strings.Split(line, ">=") - if len(result) == 2 { - return result + for _, sep := range separators { + if result := strings.Split(line, sep); len(result) == 2 { + return result + } } - - return strings.Split(line, "==") + return nil } func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) { @@ -68,14 +68,8 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc line = rStripByKey(line, commentMarker) line = rStripByKey(line, endColon) line = rStripByKey(line, hashMarker) - var s []string - - if p.useMinVersion { - s = splitLine(line) - } else { - s = strings.Split(line, "==") - } + s := p.splitLine(line) if len(s) != 2 { continue } From 78bf35c8a3b7ebd3aef95e5102d46d4d6c9dcb58 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 22 Aug 2024 09:34:55 +0600 Subject: [PATCH 06/11] replace `*` to `0` for minimum versions --- pkg/dependency/parser/python/pip/parse.go | 3 +++ pkg/dependency/parser/python/pip/parse_testcase.go | 10 ++++++++++ .../python/pip/testdata/requirements_compatible.txt | 3 ++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/dependency/parser/python/pip/parse.go b/pkg/dependency/parser/python/pip/parse.go index 08d76b919cfc..ad04580aaa7f 100644 --- a/pkg/dependency/parser/python/pip/parse.go +++ b/pkg/dependency/parser/python/pip/parse.go @@ -73,6 +73,9 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc if len(s) != 2 { continue } + if p.useMinVersion && strings.Index(s[1], "*") > -1 { + s[1] = strings.Replace(s[1], "*", "0", 1) + } if !isValidName(s[0]) || !isValidVersion(s[1]) { p.logger.Debug("Invalid package name/version in requirements.txt.", log.String("line", text)) diff --git a/pkg/dependency/parser/python/pip/parse_testcase.go b/pkg/dependency/parser/python/pip/parse_testcase.go index c84769e6f48d..e4a8d83d7117 100644 --- a/pkg/dependency/parser/python/pip/parse_testcase.go +++ b/pkg/dependency/parser/python/pip/parse_testcase.go @@ -24,6 +24,16 @@ var ( }, }, }, + { + Name: "python-gitlab", + Version: "2.0.0", + Locations: []ftypes.Location{ + { + StartLine: 3, + EndLine: 3, + }, + }, + }, } requirementsFlask = []ftypes.Package{ { diff --git a/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt b/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt index fd7b0545c6d2..5327100a6adf 100644 --- a/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt +++ b/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt @@ -1,2 +1,3 @@ keyring >= 4.1.1 # Minimum version 4.1.1 -Mopidy-Dirble ~= 1.1 # Compatible release. Same as >= 1.1, == 1.* \ No newline at end of file +Mopidy-Dirble ~= 1.1 # Compatible release. Same as >= 1.1, == 1.* +python-gitlab==2.0.* \ No newline at end of file From adb7ef5e112423499eb8a3a116655784c461ebe8 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 22 Aug 2024 10:48:15 +0600 Subject: [PATCH 07/11] update a trailling `*` --- pkg/dependency/parser/python/pip/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dependency/parser/python/pip/parse.go b/pkg/dependency/parser/python/pip/parse.go index ad04580aaa7f..ac5d29510478 100644 --- a/pkg/dependency/parser/python/pip/parse.go +++ b/pkg/dependency/parser/python/pip/parse.go @@ -73,7 +73,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc if len(s) != 2 { continue } - if p.useMinVersion && strings.Index(s[1], "*") > -1 { + if p.useMinVersion && strings.Index(s[1], ".*") > -1 { s[1] = strings.Replace(s[1], "*", "0", 1) } From 5e0b5941ad0626fb67d90879341f10800d2c9679 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 22 Aug 2024 10:48:21 +0600 Subject: [PATCH 08/11] add docs --- docs/docs/coverage/language/python.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/docs/coverage/language/python.md b/docs/docs/coverage/language/python.md index 7a697b87d250..27b776ec2d75 100644 --- a/docs/docs/coverage/language/python.md +++ b/docs/docs/coverage/language/python.md @@ -23,7 +23,7 @@ The following table provides an outline of the features Trivy offers. | Package manager | File | Transitive dependencies | Dev dependencies | [Dependency graph][dependency-graph] | Position | [Detection Priority][detection-priority] | |-----------------|------------------|:-----------------------:|:----------------:|:------------------------------------:|:--------:|:----------------------------------------:| -| pip | requirements.txt | - | Include | - | ✓ | - | +| pip | requirements.txt | - | Include | - | ✓ | ✓ | | Pipenv | Pipfile.lock | ✓ | Include | - | ✓ | Not needed | | Poetry | poetry.lock | ✓ | Exclude | ✓ | - | Not needed | @@ -42,8 +42,17 @@ Trivy parses your files generated by package managers in filesystem/repository s ### pip #### Dependency detection -Trivy only parses [version specifiers](https://packaging.python.org/en/latest/specifications/version-specifiers/#id5) with `==` comparison operator and without `.*`. -To convert unsupported version specifiers - use the `pip freeze` command. +By default, Trivy only parses [version specifiers](https://packaging.python.org/en/latest/specifications/version-specifiers/#id5) with `==` comparison operator and without `.*`. + +Using the [--detection-priority comprehensive](#detection-priority) option ensures that the tool establishes a minimum version, which is particularly useful in scenarios where identifying the exact version is challenging. +In such case Trivy parses specifiers `>=`,`~=` and a trailing `.*`. + +``` +keyring >= 4.1.1 # Minimum version 4.1.1 +Mopidy-Dirble ~= 1.1 # Minimum version 1.1 +python-gitlab==2.0.* # Minimum version 2.0.0 +``` +Also, there is a way to convert unsupported version specifiers - use the `pip freeze` command. ```bash $ cat requirements.txt From 3a4550206218b547a2085f94171e691b1b96c5c9 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 22 Aug 2024 12:43:33 +0600 Subject: [PATCH 09/11] remove unused check --- pkg/dependency/parser/python/pip/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dependency/parser/python/pip/parse.go b/pkg/dependency/parser/python/pip/parse.go index ac5d29510478..512c903d908e 100644 --- a/pkg/dependency/parser/python/pip/parse.go +++ b/pkg/dependency/parser/python/pip/parse.go @@ -73,7 +73,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc if len(s) != 2 { continue } - if p.useMinVersion && strings.Index(s[1], ".*") > -1 { + if p.useMinVersion { s[1] = strings.Replace(s[1], "*", "0", 1) } From e796df2de224fcefa95a6c82b464dd395145bf7d Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 22 Aug 2024 13:37:37 +0600 Subject: [PATCH 10/11] fix: trim suffix --- pkg/dependency/parser/python/pip/parse.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/dependency/parser/python/pip/parse.go b/pkg/dependency/parser/python/pip/parse.go index 512c903d908e..981b7def69c3 100644 --- a/pkg/dependency/parser/python/pip/parse.go +++ b/pkg/dependency/parser/python/pip/parse.go @@ -73,8 +73,8 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc if len(s) != 2 { continue } - if p.useMinVersion { - s[1] = strings.Replace(s[1], "*", "0", 1) + if p.useMinVersion && strings.HasSuffix(s[1], ".*") { + s[1] = strings.TrimSuffix(s[1], "*") + "0" } if !isValidName(s[0]) || !isValidVersion(s[1]) { From b5511bf9b5919d3fe9d92e6abe23e35c372efe55 Mon Sep 17 00:00:00 2001 From: afdesk Date: Thu, 22 Aug 2024 13:50:11 +0600 Subject: [PATCH 11/11] test: add wrong versions --- .../parser/python/pip/testdata/requirements_compatible.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt b/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt index 5327100a6adf..dbcde5b7ab10 100644 --- a/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt +++ b/pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt @@ -1,3 +1,5 @@ keyring >= 4.1.1 # Minimum version 4.1.1 Mopidy-Dirble ~= 1.1 # Compatible release. Same as >= 1.1, == 1.* -python-gitlab==2.0.* \ No newline at end of file +python-gitlab==2.0.* +django==5.*.* # this dep should be skipped +django==4.*.1 \ No newline at end of file