From 74d9d29c923543589d68453bcbb3be522880b581 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Tue, 4 Jun 2024 16:27:25 +0600 Subject: [PATCH 1/2] fix(poetry): compare pkg names from `poetry.lock` and `pyproject.toml` in lowercase --- .../analyzer/language/python/poetry/poetry.go | 15 +++++++++++++-- .../python/poetry/testdata/happy/pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/poetry/poetry.go b/pkg/fanal/analyzer/language/python/poetry/poetry.go index 600f53bafc92..cab008e4bd93 100644 --- a/pkg/fanal/analyzer/language/python/poetry/poetry.go +++ b/pkg/fanal/analyzer/language/python/poetry/poetry.go @@ -7,7 +7,9 @@ import ( "io/fs" "os" "path/filepath" + "strings" + "github.com/samber/lo" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/dependency/parser/python/poetry" @@ -102,9 +104,18 @@ func (a poetryAnalyzer) mergePyProject(fsys fs.FS, dir string, app *types.Applic return xerrors.Errorf("unable to parse %s: %w", path, err) } + // Convert package name to lower case + p = lo.MapKeys(p, func(_ any, key string) string { + return strings.ToLower(key) + }) + // Identify the direct/transitive dependencies for i, pkg := range app.Packages { - // Identify the direct/transitive dependencies - if _, ok := p[pkg.Name]; ok { + // There are cases where package names from `poetry.lock` and `pyproject.toml` use different case. + // e.g. `pyproject.toml` uses `Flask`, but `poetry.lock` uses `flask` + // cf. https://github.com/aquasecurity/trivy/issues/6817 + // So we need to compare lowercase package names. + pkgName := strings.ToLower(pkg.Name) + if _, ok := p[pkgName]; ok { app.Packages[i].Relationship = types.RelationshipDirect } else { app.Packages[i].Indirect = true diff --git a/pkg/fanal/analyzer/language/python/poetry/testdata/happy/pyproject.toml b/pkg/fanal/analyzer/language/python/poetry/testdata/happy/pyproject.toml index 2df330973ff2..9d23492736e4 100644 --- a/pkg/fanal/analyzer/language/python/poetry/testdata/happy/pyproject.toml +++ b/pkg/fanal/analyzer/language/python/poetry/testdata/happy/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Trivy"] [tool.poetry.dependencies] python = "^3.9" -flask = "^1.0" +Flask = "^1.0" requests = {version = "2.28.1", optional = true} [tool.poetry.dev-dependencies] From 954c78e0a57cfc4f4de476411d4293bd59a6b35d Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Wed, 5 Jun 2024 09:59:23 +0600 Subject: [PATCH 2/2] refactor: Use NormalizePkgName for packages from `pyproject.toml` --- pkg/dependency/parser/python/poetry/parse.go | 6 ++++-- .../analyzer/language/python/poetry/poetry.go | 18 +++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/pkg/dependency/parser/python/poetry/parse.go b/pkg/dependency/parser/python/poetry/parse.go index e691a0cb2532..8b06942d3b6b 100644 --- a/pkg/dependency/parser/python/poetry/parse.go +++ b/pkg/dependency/parser/python/poetry/parse.go @@ -105,7 +105,7 @@ func (p *Parser) parseDependencies(deps map[string]any, pkgVersions map[string][ } func (p *Parser) parseDependency(name string, versRange any, pkgVersions map[string][]string) (string, error) { - name = normalizePkgName(name) + name = NormalizePkgName(name) vers, ok := pkgVersions[name] if !ok { return "", xerrors.Errorf("no version found for %q", name) @@ -149,9 +149,11 @@ func matchVersion(currentVersion, constraint string) (bool, error) { return c.Check(v), nil } -func normalizePkgName(name string) string { +// NormalizePkgName normalizes the package name based on pep-0426 +func NormalizePkgName(name string) string { // The package names don't use `_`, `.` or upper case, but dependency names can contain them. // We need to normalize those names. + // cf. https://peps.python.org/pep-0426/#name name = strings.ToLower(name) // e.g. https://github.com/python-poetry/poetry/blob/c8945eb110aeda611cc6721565d7ad0c657d453a/poetry.lock#L819 name = strings.ReplaceAll(name, "_", "-") // e.g. https://github.com/python-poetry/poetry/blob/c8945eb110aeda611cc6721565d7ad0c657d453a/poetry.lock#L50 name = strings.ReplaceAll(name, ".", "-") // e.g. https://github.com/python-poetry/poetry/blob/c8945eb110aeda611cc6721565d7ad0c657d453a/poetry.lock#L816 diff --git a/pkg/fanal/analyzer/language/python/poetry/poetry.go b/pkg/fanal/analyzer/language/python/poetry/poetry.go index cab008e4bd93..b16ba88481c5 100644 --- a/pkg/fanal/analyzer/language/python/poetry/poetry.go +++ b/pkg/fanal/analyzer/language/python/poetry/poetry.go @@ -7,7 +7,6 @@ import ( "io/fs" "os" "path/filepath" - "strings" "github.com/samber/lo" "golang.org/x/xerrors" @@ -104,18 +103,9 @@ func (a poetryAnalyzer) mergePyProject(fsys fs.FS, dir string, app *types.Applic return xerrors.Errorf("unable to parse %s: %w", path, err) } - // Convert package name to lower case - p = lo.MapKeys(p, func(_ any, key string) string { - return strings.ToLower(key) - }) // Identify the direct/transitive dependencies for i, pkg := range app.Packages { - // There are cases where package names from `poetry.lock` and `pyproject.toml` use different case. - // e.g. `pyproject.toml` uses `Flask`, but `poetry.lock` uses `flask` - // cf. https://github.com/aquasecurity/trivy/issues/6817 - // So we need to compare lowercase package names. - pkgName := strings.ToLower(pkg.Name) - if _, ok := p[pkgName]; ok { + if _, ok := p[pkg.Name]; ok { app.Packages[i].Relationship = types.RelationshipDirect } else { app.Packages[i].Indirect = true @@ -138,5 +128,11 @@ func (a poetryAnalyzer) parsePyProject(fsys fs.FS, path string) (map[string]any, if err != nil { return nil, err } + + // Packages from `pyproject.toml` can use uppercase characters, `.` and `_`. + parsed = lo.MapKeys(parsed, func(_ any, pkgName string) string { + return poetry.NormalizePkgName(pkgName) + }) + return parsed, nil }