From b9751cf9ff2325008313df7051a51dfcfdc58bcd Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 10:54:16 +0600 Subject: [PATCH 01/24] refactor(pip): use `PostAnalyze` --- pkg/fanal/analyzer/language/python/pip/pip.go | 41 ++++++++++++++++--- .../analyzer/language/python/pip/pip_test.go | 35 ++++++---------- .../requirements.txt} | 0 .../pip/testdata/{ => happy}/requirements.txt | 0 4 files changed, 48 insertions(+), 28 deletions(-) rename pkg/fanal/analyzer/language/python/pip/testdata/{not-related.txt => empty/requirements.txt} (100%) rename pkg/fanal/analyzer/language/python/pip/testdata/{ => happy}/requirements.txt (100%) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 380fcbf4936c..50aec46f5062 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -2,6 +2,9 @@ package pip import ( "context" + "github.com/aquasecurity/trivy/pkg/utils/fsutils" + "io" + "io/fs" "os" "path/filepath" @@ -14,19 +17,45 @@ import ( ) func init() { - analyzer.RegisterAnalyzer(&pipLibraryAnalyzer{}) + analyzer.RegisterPostAnalyzer(analyzer.TypePip, newPipLibraryAnalyzer) } const version = 1 type pipLibraryAnalyzer struct{} -func (a pipLibraryAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { - res, err := language.Analyze(types.Pip, input.FilePath, input.Content, pip.NewParser()) - if err != nil { - return nil, xerrors.Errorf("unable to parse requirements.txt: %w", err) +func newPipLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { + return pipLibraryAnalyzer{}, nil +} + +func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { + var apps []types.Application + // We only saved the `requirement.txt` files + required := func(_ string, _ fs.DirEntry) bool { + return true + } + + if err := fsutils.WalkDir(input.FS, ".", required, func(pathPath string, d fs.DirEntry, r io.Reader) error { + app, err := language.Parse(types.Pip, pathPath, r, pip.NewParser()) + if err != nil { + return xerrors.Errorf("unable to parse requirements.txt: %w", err) + } + + if app == nil { + return nil + } + + // TODO insert licenses + + apps = append(apps, *app) + return nil + }); err != nil { + return nil, xerrors.Errorf("pip walt error: %w", err) } - return res, nil + + return &analyzer.AnalysisResult{ + Applications: apps, + }, nil } func (a pipLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool { diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index 62fd13953f9b..2788682150e0 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -14,19 +14,19 @@ import ( func Test_pipAnalyzer_Analyze(t *testing.T) { tests := []struct { - name string - inputFile string - want *analyzer.AnalysisResult - wantErr string + name string + dir string + want *analyzer.AnalysisResult + wantErr string }{ { - name: "happy path", - inputFile: "testdata/requirements.txt", + name: "happy path", + dir: "testdata/happy", want: &analyzer.AnalysisResult{ Applications: []types.Application{ { Type: types.Pip, - FilePath: "testdata/requirements.txt", + FilePath: "requirements.txt", Packages: types.Packages{ { Name: "click", @@ -64,29 +64,20 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { }, }, { - name: "happy path with not related filename", - inputFile: "testdata/not-related.txt", - want: nil, + name: "happy path with not related filename", + dir: "testdata/empty", + want: &analyzer.AnalysisResult{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - f, err := os.Open(tt.inputFile) + a, err := newPipLibraryAnalyzer(analyzer.AnalyzerOptions{}) require.NoError(t, err) - defer f.Close() - a := pipLibraryAnalyzer{} - ctx := context.Background() - got, err := a.Analyze(ctx, analyzer.AnalysisInput{ - FilePath: tt.inputFile, - Content: f, + got, err := a.PostAnalyze(context.Background(), analyzer.PostAnalysisInput{ + FS: os.DirFS(tt.dir), }) - if tt.wantErr != "" { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.wantErr) - return - } require.NoError(t, err) assert.Equal(t, tt.want, got) }) diff --git a/pkg/fanal/analyzer/language/python/pip/testdata/not-related.txt b/pkg/fanal/analyzer/language/python/pip/testdata/empty/requirements.txt similarity index 100% rename from pkg/fanal/analyzer/language/python/pip/testdata/not-related.txt rename to pkg/fanal/analyzer/language/python/pip/testdata/empty/requirements.txt diff --git a/pkg/fanal/analyzer/language/python/pip/testdata/requirements.txt b/pkg/fanal/analyzer/language/python/pip/testdata/happy/requirements.txt similarity index 100% rename from pkg/fanal/analyzer/language/python/pip/testdata/requirements.txt rename to pkg/fanal/analyzer/language/python/pip/testdata/happy/requirements.txt From cad6440e708cc88b8d902007b9dc246c3eaedb9e Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 12:03:42 +0600 Subject: [PATCH 02/24] feat(pip): detect licenses from VIRTUAL_ENV --- pkg/fanal/analyzer/language/python/pip/pip.go | 72 +++++++++- .../analyzer/language/python/pip/pip_test.go | 13 +- .../Flask-2.0.0.dist-info/METADATA | 124 ++++++++++++++++++ .../click-8.0.0.dist-info/METADATA | 109 +++++++++++++++ 4 files changed, 314 insertions(+), 4 deletions(-) create mode 100644 pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA create mode 100644 pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 50aec46f5062..94af2aa4868e 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -2,11 +2,16 @@ package pip import ( "context" + "github.com/aquasecurity/trivy/pkg/dependency" + "github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging" + "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/utils/fsutils" + xio "github.com/aquasecurity/trivy/pkg/x/io" "io" "io/fs" "os" "path/filepath" + "strings" "golang.org/x/xerrors" @@ -22,14 +27,49 @@ func init() { const version = 1 -type pipLibraryAnalyzer struct{} +type pipLibraryAnalyzer struct { + logger *log.Logger + metadataParser packaging.Parser +} func newPipLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { - return pipLibraryAnalyzer{}, nil + return pipLibraryAnalyzer{ + logger: log.WithPrefix("pip"), + metadataParser: *packaging.NewParser(), + }, nil } func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { var apps []types.Application + var licenses = make(map[string][]string) + + if libDir, err := findLibDir(); err != nil || libDir == "" { + a.logger.Warn("Unable to find python `lib` directory. License detection are skipped.", log.Err(err)) + } else { + requiredMetadata := func(filePath string, _ fs.DirEntry) bool { + return strings.HasSuffix(filepath.Dir(filePath), ".dist-info") && filepath.Base(filePath) == "METADATA" + } + + // Detect licenses from python lib directory + if err = fsutils.WalkDir(os.DirFS(libDir), ".", requiredMetadata, func(path string, d fs.DirEntry, r io.Reader) error { + rs, err := xio.NewReadSeekerAt(r) + if err != nil { + return xerrors.Errorf("Unable to convert reader: %w", err) + } + + metadataPkg, _, err := a.metadataParser.Parse(rs) + if err != nil { + return xerrors.Errorf("metadata parse error: %w", err) + } + + // METADATA file contains info about only 1 package + licenses[packageID(metadataPkg[0].Name, metadataPkg[0].Version)] = metadataPkg[0].Licenses + return nil + }); err != nil { + return nil, xerrors.Errorf("walk python lib dir error: %w", err) + } + } + // We only saved the `requirement.txt` files required := func(_ string, _ fs.DirEntry) bool { return true @@ -45,7 +85,13 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn return nil } - // TODO insert licenses + // Fill licenses + for i, pkg := range app.Packages { + pkgID := packageID(pkg.Name, pkg.Version) + if lics, ok := licenses[pkgID]; ok { + app.Packages[i].Licenses = lics + } + } apps = append(apps, *app) return nil @@ -70,3 +116,23 @@ func (a pipLibraryAnalyzer) Type() analyzer.Type { func (a pipLibraryAnalyzer) Version() int { return version } + +func findLibDir() (string, error) { + // VIRTUAL_ENV + if venv := os.Getenv("VIRTUAL_ENV"); venv != "" { + libDir := filepath.Join(venv, "lib") + if _, err := os.Stat(libDir); os.IsNotExist(err) { + return "", xerrors.Errorf("Unable to detect `lib` dir for %q venv: %w", venv, err) + } + return libDir, nil + } + + //find bins + + // default dir + return "", nil +} + +func packageID(name, ver string) string { + return dependency.ID(types.Pip, name, ver) +} diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index 2788682150e0..0e7635089541 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -16,12 +16,14 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { tests := []struct { name string dir string + venv string want *analyzer.AnalysisResult wantErr string }{ { - name: "happy path", + name: "happy path with licenses from venv", dir: "testdata/happy", + venv: "testdata", want: &analyzer.AnalysisResult{ Applications: []types.Application{ { @@ -37,6 +39,9 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { EndLine: 1, }, }, + Licenses: []string{ + "BSD License", + }, }, { Name: "Flask", @@ -47,6 +52,9 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { EndLine: 2, }, }, + Licenses: []string{ + "BSD License", + }, }, { Name: "itsdangerous", @@ -71,6 +79,9 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.venv != "" { + t.Setenv("VIRTUAL_ENV", tt.venv) + } a, err := newPipLibraryAnalyzer(analyzer.AnalyzerOptions{}) require.NoError(t, err) diff --git a/pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA b/pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA new file mode 100644 index 000000000000..5df0132dde56 --- /dev/null +++ b/pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA @@ -0,0 +1,124 @@ +Metadata-Version: 2.1 +Name: Flask +Version: 2.0.0 +Summary: A simple framework for building complex web applications. +Home-page: https://palletsprojects.com/p/flask +Author: Armin Ronacher +Author-email: armin.ronacher@active-4.com +Maintainer: Pallets +Maintainer-email: contact@palletsprojects.com +License: BSD-3-Clause +Project-URL: Donate, https://palletsprojects.com/donate +Project-URL: Documentation, https://flask.palletsprojects.com/ +Project-URL: Changes, https://flask.palletsprojects.com/changes/ +Project-URL: Source Code, https://github.com/pallets/flask/ +Project-URL: Issue Tracker, https://github.com/pallets/flask/issues/ +Project-URL: Twitter, https://twitter.com/PalletsTeam +Project-URL: Chat, https://discord.gg/pallets +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Framework :: Flask +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content +Classifier: Topic :: Internet :: WWW/HTTP :: WSGI +Classifier: Topic :: Internet :: WWW/HTTP :: WSGI :: Application +Classifier: Topic :: Software Development :: Libraries :: Application Frameworks +Requires-Python: >=3.6 +Description-Content-Type: text/x-rst +Requires-Dist: Werkzeug (>=2.0) +Requires-Dist: Jinja2 (>=3.0) +Requires-Dist: itsdangerous (>=2.0) +Requires-Dist: click (>=7.1.2) +Provides-Extra: async +Requires-Dist: asgiref (>=3.2) ; extra == 'async' +Provides-Extra: dotenv +Requires-Dist: python-dotenv ; extra == 'dotenv' + +Flask +===== + +Flask is a lightweight `WSGI`_ web application framework. It is designed +to make getting started quick and easy, with the ability to scale up to +complex applications. It began as a simple wrapper around `Werkzeug`_ +and `Jinja`_ and has become one of the most popular Python web +application frameworks. + +Flask offers suggestions, but doesn't enforce any dependencies or +project layout. It is up to the developer to choose the tools and +libraries they want to use. There are many extensions provided by the +community that make adding new functionality easy. + +.. _WSGI: https://wsgi.readthedocs.io/ +.. _Werkzeug: https://werkzeug.palletsprojects.com/ +.. _Jinja: https://jinja.palletsprojects.com/ + + +Installing +---------- + +Install and update using `pip`_: + +.. code-block:: text + + $ pip install -U Flask + +.. _pip: https://pip.pypa.io/en/stable/quickstart/ + + +A Simple Example +---------------- + +.. code-block:: python + + # save this as app.py + from flask import Flask + + app = Flask(__name__) + + @app.route("/") + def hello(): + return "Hello, World!" + +.. code-block:: text + + $ flask run + * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit) + + +Contributing +------------ + +For guidance on setting up a development environment and how to make a +contribution to Flask, see the `contributing guidelines`_. + +.. _contributing guidelines: https://github.com/pallets/flask/blob/master/CONTRIBUTING.rst + + +Donate +------ + +The Pallets organization develops and supports Flask and the libraries +it uses. In order to grow the community of contributors and users, and +allow the maintainers to devote more time to the projects, `please +donate today`_. + +.. _please donate today: https://palletsprojects.com/donate + + +Links +----- + +- Documentation: https://flask.palletsprojects.com/ +- Changes: https://flask.palletsprojects.com/changes/ +- PyPI Releases: https://pypi.org/project/Flask/ +- Source Code: https://github.com/pallets/flask/ +- Issue Tracker: https://github.com/pallets/flask/issues/ +- Website: https://palletsprojects.com/p/flask/ +- Twitter: https://twitter.com/PalletsTeam +- Chat: https://discord.gg/pallets + + diff --git a/pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA b/pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA new file mode 100644 index 000000000000..5e67f0446795 --- /dev/null +++ b/pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA @@ -0,0 +1,109 @@ +Metadata-Version: 2.1 +Name: click +Version: 8.0.0 +Summary: Composable command line interface toolkit +Home-page: https://palletsprojects.com/p/click/ +Author: Armin Ronacher +Author-email: armin.ronacher@active-4.com +Maintainer: Pallets +Maintainer-email: contact@palletsprojects.com +License: BSD-3-Clause +Project-URL: Donate, https://palletsprojects.com/donate +Project-URL: Documentation, https://click.palletsprojects.com/ +Project-URL: Changes, https://click.palletsprojects.com/changes/ +Project-URL: Source Code, https://github.com/pallets/click/ +Project-URL: Issue Tracker, https://github.com/pallets/click/issues/ +Project-URL: Twitter, https://twitter.com/PalletsTeam +Project-URL: Chat, https://discord.gg/pallets +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Requires-Python: >=3.6 +Description-Content-Type: text/x-rst +Requires-Dist: colorama ; platform_system == "Windows" + +\$ click\_ +========== + +Click is a Python package for creating beautiful command line interfaces +in a composable way with as little code as necessary. It's the "Command +Line Interface Creation Kit". It's highly configurable but comes with +sensible defaults out of the box. + +It aims to make the process of writing command line tools quick and fun +while also preventing any frustration caused by the inability to +implement an intended CLI API. + +Click in three points: + +- Arbitrary nesting of commands +- Automatic help page generation +- Supports lazy loading of subcommands at runtime + + +Installing +---------- + +Install and update using `pip`_: + +.. code-block:: text + + $ pip install -U click + +.. _pip: https://pip.pypa.io/en/stable/quickstart/ + + +A Simple Example +---------------- + +.. code-block:: python + + import click + + @click.command() + @click.option("--count", default=1, help="Number of greetings.") + @click.option("--name", prompt="Your name", help="The person to greet.") + def hello(count, name): + """Simple program that greets NAME for a total of COUNT times.""" + for _ in range(count): + click.echo(f"Hello, {name}!") + + if __name__ == '__main__': + hello() + +.. code-block:: text + + $ python hello.py --count=3 + Your name: Click + Hello, Click! + Hello, Click! + Hello, Click! + + +Donate +------ + +The Pallets organization develops and supports Click and other popular +packages. In order to grow the community of contributors and users, and +allow the maintainers to devote more time to the projects, `please +donate today`_. + +.. _please donate today: https://palletsprojects.com/donate + + +Links +----- + +- Documentation: https://click.palletsprojects.com/ +- Changes: https://click.palletsprojects.com/changes/ +- PyPI Releases: https://pypi.org/project/click/ +- Source Code: https://github.com/pallets/click +- Issue Tracker: https://github.com/pallets/click/issues +- Website: https://palletsprojects.com/p/click +- Twitter: https://twitter.com/PalletsTeam +- Chat: https://discord.gg/pallets + + From c54611687cab11fcd1f87769ca5af4ce0a916f21 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 14:45:36 +0600 Subject: [PATCH 03/24] feat(pip): add check python executable files --- pkg/fanal/analyzer/language/python/pip/pip.go | 84 ++++++++++-- .../Flask-2.0.0.dist-info/METADATA | 0 .../click-8.0.0.dist-info/METADATA | 0 .../Flask-2.0.0.dist-info/METADATA | 124 ++++++++++++++++++ .../click-8.0.0.dist-info/METADATA | 109 +++++++++++++++ 5 files changed, 307 insertions(+), 10 deletions(-) rename pkg/fanal/analyzer/language/python/pip/testdata/{lib/python3.10 => libs/common-dir/lib}/site-packages/Flask-2.0.0.dist-info/METADATA (100%) rename pkg/fanal/analyzer/language/python/pip/testdata/{lib/python3.10 => libs/common-dir/lib}/site-packages/click-8.0.0.dist-info/METADATA (100%) create mode 100644 pkg/fanal/analyzer/language/python/pip/testdata/libs/python-dir/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA create mode 100644 pkg/fanal/analyzer/language/python/pip/testdata/libs/python-dir/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 94af2aa4868e..015c99ffb4e6 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -10,6 +10,7 @@ import ( "io" "io/fs" "os" + "os/exec" "path/filepath" "strings" @@ -27,6 +28,8 @@ func init() { const version = 1 +var pythonExecNames = []string{"python3", "python", "python2", "python.exe"} + type pipLibraryAnalyzer struct { logger *log.Logger metadataParser packaging.Parser @@ -43,8 +46,8 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn var apps []types.Application var licenses = make(map[string][]string) - if libDir, err := findLibDir(); err != nil || libDir == "" { - a.logger.Warn("Unable to find python `lib` directory. License detection are skipped.", log.Err(err)) + if libDir, err := getPythonSitePackagesDir(); err != nil || libDir == "" { + a.logger.Warn("Unable to find python `lib` directory. License detection is skipped.", log.Err(err)) } else { requiredMetadata := func(filePath string, _ fs.DirEntry) bool { return strings.HasSuffix(filepath.Dir(filePath), ".dist-info") && filepath.Base(filePath) == "METADATA" @@ -117,22 +120,83 @@ func (a pipLibraryAnalyzer) Version() int { return version } -func findLibDir() (string, error) { - // VIRTUAL_ENV +func packageID(name, ver string) string { + return dependency.ID(types.Pip, name, ver) +} + +func getPythonSitePackagesDir() (string, error) { + // check VIRTUAL_ENV first if venv := os.Getenv("VIRTUAL_ENV"); venv != "" { libDir := filepath.Join(venv, "lib") if _, err := os.Stat(libDir); os.IsNotExist(err) { return "", xerrors.Errorf("Unable to detect `lib` dir for %q venv: %w", venv, err) } - return libDir, nil + + spDir, err := sitePackagesDir(libDir) + if err != nil { + return "", xerrors.Errorf("Unable to detect `site-packages` dir for %q venv: %w", spDir, err) + } + + if spDir != "" { + return spDir, nil + } + } + + // Find path to Python executable + pythonExecPath, err := getPythonExecutablePath() + if err != nil { + return "", err + } + pythonExecDir := filepath.Dir(pythonExecPath) + + // Search for a directory starting with "python" in the lib directory + libDir := filepath.Join(pythonExecDir, "..", "lib") + spDir, err := sitePackagesDir(libDir) + if err != nil { + return "", xerrors.Errorf("Unable to detect `site-packages` dir for %q: %w", pythonExecPath, err) + } + if spDir != "" { + return spDir, nil } - //find bins + // Try another common pattern if the Python library directory is not found + spDir = filepath.Join(pythonExecDir, "..", "..", "lib", "site-packages") + _, err = os.Stat(spDir) + if os.IsNotExist(err) { + return "", xerrors.Errorf("site-packages directory not found") + } - // default dir - return "", nil + return spDir, nil } -func packageID(name, ver string) string { - return dependency.ID(types.Pip, name, ver) +func getPythonExecutablePath() (string, error) { + for _, execName := range pythonExecNames { + // Get the absolute path of the python command + pythonPath, err := exec.LookPath(execName) + if err != nil { + continue + } + return pythonPath, nil + } + return "", xerrors.Errorf("Unable to find path to Python executable") +} + +func sitePackagesDir(libDir string) (string, error) { + entries, err := os.ReadDir(libDir) + if err != nil { + if !os.IsNotExist(err) { + return "", xerrors.Errorf("failed to read lib directory: %w", err) + } + return "", nil + } + + // Use latest python dir + var spDir string + for _, entry := range entries { + if entry.IsDir() && strings.HasPrefix(entry.Name(), "python") { + // Found a directory starting with "python", assume it's the Python library directory + spDir = filepath.Join(libDir, entry.Name(), "site-packages") + } + } + return spDir, nil } diff --git a/pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA b/pkg/fanal/analyzer/language/python/pip/testdata/libs/common-dir/lib/site-packages/Flask-2.0.0.dist-info/METADATA similarity index 100% rename from pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA rename to pkg/fanal/analyzer/language/python/pip/testdata/libs/common-dir/lib/site-packages/Flask-2.0.0.dist-info/METADATA diff --git a/pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA b/pkg/fanal/analyzer/language/python/pip/testdata/libs/common-dir/lib/site-packages/click-8.0.0.dist-info/METADATA similarity index 100% rename from pkg/fanal/analyzer/language/python/pip/testdata/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA rename to pkg/fanal/analyzer/language/python/pip/testdata/libs/common-dir/lib/site-packages/click-8.0.0.dist-info/METADATA diff --git a/pkg/fanal/analyzer/language/python/pip/testdata/libs/python-dir/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA b/pkg/fanal/analyzer/language/python/pip/testdata/libs/python-dir/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA new file mode 100644 index 000000000000..5df0132dde56 --- /dev/null +++ b/pkg/fanal/analyzer/language/python/pip/testdata/libs/python-dir/lib/python3.10/site-packages/Flask-2.0.0.dist-info/METADATA @@ -0,0 +1,124 @@ +Metadata-Version: 2.1 +Name: Flask +Version: 2.0.0 +Summary: A simple framework for building complex web applications. +Home-page: https://palletsprojects.com/p/flask +Author: Armin Ronacher +Author-email: armin.ronacher@active-4.com +Maintainer: Pallets +Maintainer-email: contact@palletsprojects.com +License: BSD-3-Clause +Project-URL: Donate, https://palletsprojects.com/donate +Project-URL: Documentation, https://flask.palletsprojects.com/ +Project-URL: Changes, https://flask.palletsprojects.com/changes/ +Project-URL: Source Code, https://github.com/pallets/flask/ +Project-URL: Issue Tracker, https://github.com/pallets/flask/issues/ +Project-URL: Twitter, https://twitter.com/PalletsTeam +Project-URL: Chat, https://discord.gg/pallets +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Framework :: Flask +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content +Classifier: Topic :: Internet :: WWW/HTTP :: WSGI +Classifier: Topic :: Internet :: WWW/HTTP :: WSGI :: Application +Classifier: Topic :: Software Development :: Libraries :: Application Frameworks +Requires-Python: >=3.6 +Description-Content-Type: text/x-rst +Requires-Dist: Werkzeug (>=2.0) +Requires-Dist: Jinja2 (>=3.0) +Requires-Dist: itsdangerous (>=2.0) +Requires-Dist: click (>=7.1.2) +Provides-Extra: async +Requires-Dist: asgiref (>=3.2) ; extra == 'async' +Provides-Extra: dotenv +Requires-Dist: python-dotenv ; extra == 'dotenv' + +Flask +===== + +Flask is a lightweight `WSGI`_ web application framework. It is designed +to make getting started quick and easy, with the ability to scale up to +complex applications. It began as a simple wrapper around `Werkzeug`_ +and `Jinja`_ and has become one of the most popular Python web +application frameworks. + +Flask offers suggestions, but doesn't enforce any dependencies or +project layout. It is up to the developer to choose the tools and +libraries they want to use. There are many extensions provided by the +community that make adding new functionality easy. + +.. _WSGI: https://wsgi.readthedocs.io/ +.. _Werkzeug: https://werkzeug.palletsprojects.com/ +.. _Jinja: https://jinja.palletsprojects.com/ + + +Installing +---------- + +Install and update using `pip`_: + +.. code-block:: text + + $ pip install -U Flask + +.. _pip: https://pip.pypa.io/en/stable/quickstart/ + + +A Simple Example +---------------- + +.. code-block:: python + + # save this as app.py + from flask import Flask + + app = Flask(__name__) + + @app.route("/") + def hello(): + return "Hello, World!" + +.. code-block:: text + + $ flask run + * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit) + + +Contributing +------------ + +For guidance on setting up a development environment and how to make a +contribution to Flask, see the `contributing guidelines`_. + +.. _contributing guidelines: https://github.com/pallets/flask/blob/master/CONTRIBUTING.rst + + +Donate +------ + +The Pallets organization develops and supports Flask and the libraries +it uses. In order to grow the community of contributors and users, and +allow the maintainers to devote more time to the projects, `please +donate today`_. + +.. _please donate today: https://palletsprojects.com/donate + + +Links +----- + +- Documentation: https://flask.palletsprojects.com/ +- Changes: https://flask.palletsprojects.com/changes/ +- PyPI Releases: https://pypi.org/project/Flask/ +- Source Code: https://github.com/pallets/flask/ +- Issue Tracker: https://github.com/pallets/flask/issues/ +- Website: https://palletsprojects.com/p/flask/ +- Twitter: https://twitter.com/PalletsTeam +- Chat: https://discord.gg/pallets + + diff --git a/pkg/fanal/analyzer/language/python/pip/testdata/libs/python-dir/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA b/pkg/fanal/analyzer/language/python/pip/testdata/libs/python-dir/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA new file mode 100644 index 000000000000..5e67f0446795 --- /dev/null +++ b/pkg/fanal/analyzer/language/python/pip/testdata/libs/python-dir/lib/python3.10/site-packages/click-8.0.0.dist-info/METADATA @@ -0,0 +1,109 @@ +Metadata-Version: 2.1 +Name: click +Version: 8.0.0 +Summary: Composable command line interface toolkit +Home-page: https://palletsprojects.com/p/click/ +Author: Armin Ronacher +Author-email: armin.ronacher@active-4.com +Maintainer: Pallets +Maintainer-email: contact@palletsprojects.com +License: BSD-3-Clause +Project-URL: Donate, https://palletsprojects.com/donate +Project-URL: Documentation, https://click.palletsprojects.com/ +Project-URL: Changes, https://click.palletsprojects.com/changes/ +Project-URL: Source Code, https://github.com/pallets/click/ +Project-URL: Issue Tracker, https://github.com/pallets/click/issues/ +Project-URL: Twitter, https://twitter.com/PalletsTeam +Project-URL: Chat, https://discord.gg/pallets +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Requires-Python: >=3.6 +Description-Content-Type: text/x-rst +Requires-Dist: colorama ; platform_system == "Windows" + +\$ click\_ +========== + +Click is a Python package for creating beautiful command line interfaces +in a composable way with as little code as necessary. It's the "Command +Line Interface Creation Kit". It's highly configurable but comes with +sensible defaults out of the box. + +It aims to make the process of writing command line tools quick and fun +while also preventing any frustration caused by the inability to +implement an intended CLI API. + +Click in three points: + +- Arbitrary nesting of commands +- Automatic help page generation +- Supports lazy loading of subcommands at runtime + + +Installing +---------- + +Install and update using `pip`_: + +.. code-block:: text + + $ pip install -U click + +.. _pip: https://pip.pypa.io/en/stable/quickstart/ + + +A Simple Example +---------------- + +.. code-block:: python + + import click + + @click.command() + @click.option("--count", default=1, help="Number of greetings.") + @click.option("--name", prompt="Your name", help="The person to greet.") + def hello(count, name): + """Simple program that greets NAME for a total of COUNT times.""" + for _ in range(count): + click.echo(f"Hello, {name}!") + + if __name__ == '__main__': + hello() + +.. code-block:: text + + $ python hello.py --count=3 + Your name: Click + Hello, Click! + Hello, Click! + Hello, Click! + + +Donate +------ + +The Pallets organization develops and supports Click and other popular +packages. In order to grow the community of contributors and users, and +allow the maintainers to devote more time to the projects, `please +donate today`_. + +.. _please donate today: https://palletsprojects.com/donate + + +Links +----- + +- Documentation: https://click.palletsprojects.com/ +- Changes: https://click.palletsprojects.com/changes/ +- PyPI Releases: https://pypi.org/project/click/ +- Source Code: https://github.com/pallets/click +- Issue Tracker: https://github.com/pallets/click/issues +- Website: https://palletsprojects.com/p/click +- Twitter: https://twitter.com/PalletsTeam +- Chat: https://discord.gg/pallets + + From 8739155d74af27412d96e3537b271cbba0324f39 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 14:45:41 +0600 Subject: [PATCH 04/24] update tests --- .../analyzer/language/python/pip/pip_test.go | 110 +++++++++++++++--- 1 file changed, 96 insertions(+), 14 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index 0e7635089541..384512b1d4e6 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -3,6 +3,8 @@ package pip import ( "context" "os" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -13,17 +15,83 @@ import ( ) func Test_pipAnalyzer_Analyze(t *testing.T) { + resultWithLicenses := &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.Pip, + FilePath: "requirements.txt", + Packages: types.Packages{ + { + Name: "click", + Version: "8.0.0", + Locations: []types.Location{ + { + StartLine: 1, + EndLine: 1, + }, + }, + Licenses: []string{ + "BSD License", + }, + }, + { + Name: "Flask", + Version: "2.0.0", + Locations: []types.Location{ + { + StartLine: 2, + EndLine: 2, + }, + }, + Licenses: []string{ + "BSD License", + }, + }, + { + Name: "itsdangerous", + Version: "2.0.0", + Locations: []types.Location{ + { + StartLine: 3, + EndLine: 3, + }, + }, + }, + }, + }, + }, + } + tests := []struct { - name string - dir string - venv string - want *analyzer.AnalysisResult - wantErr string + name string + dir string + venv string + pythonExecDir string + want *analyzer.AnalysisResult + wantErr string }{ { - name: "happy path with licenses from venv", - dir: "testdata/happy", - venv: "testdata", + name: "happy path with licenses from venv", + dir: filepath.Join("testdata", "happy"), + venv: filepath.Join("testdata", "libs", "python-dir"), + pythonExecDir: filepath.Join("testdata", "libs", "python-dir", "bin"), + want: resultWithLicenses, + }, + { + name: "happy path with licenses from python dir", + dir: filepath.Join("testdata", "happy"), + pythonExecDir: filepath.Join("testdata", "libs", "python-dir", "bin"), + want: resultWithLicenses, + }, + { + name: "happy path with licenses from common dir", + dir: filepath.Join("testdata", "happy"), + pythonExecDir: filepath.Join("testdata", "libs", "common-dir", "foo", "bar"), + want: resultWithLicenses, + }, + { + name: "happy path without licenses", + dir: filepath.Join("testdata", "happy"), want: &analyzer.AnalysisResult{ Applications: []types.Application{ { @@ -39,9 +107,6 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { EndLine: 1, }, }, - Licenses: []string{ - "BSD License", - }, }, { Name: "Flask", @@ -52,9 +117,6 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { EndLine: 2, }, }, - Licenses: []string{ - "BSD License", - }, }, { Name: "itsdangerous", @@ -82,6 +144,26 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { if tt.venv != "" { t.Setenv("VIRTUAL_ENV", tt.venv) } + if tt.pythonExecDir != "" { + err := os.MkdirAll(tt.pythonExecDir, os.ModePerm) + require.NoError(t, err) + defer func() { + if strings.HasSuffix(tt.pythonExecDir, "bar") { // for `happy path with licenses from common dir` test + tt.pythonExecDir = filepath.Dir(tt.pythonExecDir) + } + err = os.RemoveAll(tt.pythonExecDir) + require.NoError(t, err) + }() + + // create temp python3 Executable + err = os.WriteFile(filepath.Join(tt.pythonExecDir, "python3"), nil, 0755) + require.NoError(t, err) + + absPath, err := filepath.Abs(tt.pythonExecDir) + require.NoError(t, err) + t.Setenv("PATH", absPath) + } + a, err := newPipLibraryAnalyzer(analyzer.AnalyzerOptions{}) require.NoError(t, err) From 87fb037c6ab22321f2499ecab36ad596d5a4df08 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 15:00:45 +0600 Subject: [PATCH 05/24] test: add Test_getPythonExecutablePath --- .../analyzer/language/python/pip/pip_test.go | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index 384512b1d4e6..985802686023 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -202,3 +202,54 @@ func Test_pipAnalyzer_Required(t *testing.T) { }) } } + +func Test_getPythonExecutablePath(t *testing.T) { + tests := []struct { + name string + execName string + wantErr string + }{ + { + name: "happy path with `python` filename", + execName: "python", + }, + { + name: "happy path with `python3` filename", + execName: "python3", + }, + { + name: "happy path with `python2` filename", + execName: "python2", + }, + { + name: "happy path with `python.exe` filename", + execName: "python.exe", + }, + { + name: "sad path. Python executable not found", + execName: "python-wrong", + wantErr: "Unable to find path to Python executable", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + binDir := filepath.Join(tmpDir, "bin") + err := os.MkdirAll(binDir, os.ModePerm) + require.NoError(t, err) + + err = os.WriteFile(filepath.Join(binDir, tt.execName), nil, 0755) + require.NoError(t, err) + + t.Setenv("PATH", binDir) + + path, err := getPythonExecutablePath() + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, tt.execName, filepath.Base(path)) + }) + } +} From a505ca7ea7656c4d242d68289fa20450ca2a030a Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 15:12:20 +0600 Subject: [PATCH 06/24] fix linter errors --- pkg/fanal/analyzer/language/python/pip/pip.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 015c99ffb4e6..4f747d91fcb6 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -2,11 +2,6 @@ package pip import ( "context" - "github.com/aquasecurity/trivy/pkg/dependency" - "github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging" - "github.com/aquasecurity/trivy/pkg/log" - "github.com/aquasecurity/trivy/pkg/utils/fsutils" - xio "github.com/aquasecurity/trivy/pkg/x/io" "io" "io/fs" "os" @@ -16,10 +11,15 @@ import ( "golang.org/x/xerrors" + "github.com/aquasecurity/trivy/pkg/dependency" + "github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging" "github.com/aquasecurity/trivy/pkg/dependency/parser/python/pip" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/utils/fsutils" + xio "github.com/aquasecurity/trivy/pkg/x/io" ) func init() { From 0e8b8e2235dec5946f53c3552bf4bb848b040cd9 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 15:30:36 +0600 Subject: [PATCH 07/24] docs(coverage): update python page --- docs/docs/coverage/language/python.md | 35 ++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/docs/docs/coverage/language/python.md b/docs/docs/coverage/language/python.md index ce792842b978..2266cb9b484b 100644 --- a/docs/docs/coverage/language/python.md +++ b/docs/docs/coverage/language/python.md @@ -3,20 +3,20 @@ Trivy supports three types of Python package managers: `pip`, `Pipenv` and `Poetry`. The following scanners are supported for package managers. -| Package manager | SBOM | Vulnerability | License | -| --------------- | :---: | :-----------: | :-----: | -| pip | ✓ | ✓ | - | -| Pipenv | ✓ | ✓ | - | -| Poetry | ✓ | ✓ | - | +| Package manager | SBOM | Vulnerability | License | +|-----------------|:----:|:-------------:|:-------:| +| pip | ✓ | ✓ | ✓ | +| Pipenv | ✓ | ✓ | - | +| Poetry | ✓ | ✓ | - | In addition, Trivy supports three formats of Python packages: `egg`, `wheel` and `conda`. The following scanners are supported for Python packages. -| Packaging | SBOM | Vulnerability | License | -| --------- | :---: | :-----------: | :-----: | -| Egg | ✓ | ✓ | ✓ | -| Wheel | ✓ | ✓ | ✓ | -| Conda | ✓ | - | - | +| Packaging | SBOM | Vulnerability | License | +|-----------|:----:|:-------------:|:-------:| +| Egg | ✓ | ✓ | ✓ | +| Wheel | ✓ | ✓ | ✓ | +| Conda | ✓ | - | - | The following table provides an outline of the features Trivy offers. @@ -40,6 +40,8 @@ See [here](./index.md) for the detail. Trivy parses your files generated by package managers in filesystem/repository scanning. ### pip + +#### dependency detection Trivy only parses [version specifiers](https://packaging.python.org/en/latest/specifications/version-specifiers/#id4) with `==` comparison operator and without `.*`. To convert unsupported version specifiers - use the `pip freeze` command. @@ -91,7 +93,16 @@ urllib3==1.26.15 `requirements.txt` files don't contain information about dependencies used for development. Trivy could detect vulnerabilities on the development packages, which not affect your production environment. -License detection is not supported for `pip`. +#### license detection + +`requirements.txt` files don't contain information about licenses. +Therefore, Trivy checks `METADATA` files from `lib/site-packages` directory. + +Trivy uses 3 ways to detect `site-packages` directory: + +- Checks `VIRTUAL_ENV` enveroment. +- detects path to `python`[^1] binary and checks `../lib/pythonX.Y/site-packages` directory. +- detects path to `python`[^1] binary and checks `../../lib/site-packages` directory. ### Pipenv Trivy parses `Pipfile.lock`. @@ -116,4 +127,6 @@ Trivy looks for `*.egg-info`, `*.egg-info/PKG-INFO`, `*.egg` and `EGG-INFO/PKG-I ### Wheel Trivy looks for `.dist-info/META-DATA` to identify Python packages. +[^1]: Trivy checks `python`, `python3`, `python2` and `python.exe` file names. + [dependency-graph]: ../../configuration/reporting.md#show-origins-of-vulnerable-dependencies From 18627a1f4a967e48da124b2d507e86ec353ac484 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 15:46:15 +0600 Subject: [PATCH 08/24] fix tests --- pkg/fanal/artifact/local/fs_test.go | 128 ++++++++++++++-------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/pkg/fanal/artifact/local/fs_test.go b/pkg/fanal/artifact/local/fs_test.go index 1d4029578ca9..2cee794c85b2 100644 --- a/pkg/fanal/artifact/local/fs_test.go +++ b/pkg/fanal/artifact/local/fs_test.go @@ -47,7 +47,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:afc2bc421aac8c61d89d4dd1c1865efb5441e3877c8a4c919232729d7c574dab", + BlobID: "sha256:e480047f53bccb8a9107a424fab43452dc59df641022a300d34326639254a0cf", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, OS: types.OS{ @@ -82,9 +82,9 @@ func TestArtifact_Inspect(t *testing.T) { want: artifact.Reference{ Name: "host", Type: artifact.TypeFilesystem, - ID: "sha256:afc2bc421aac8c61d89d4dd1c1865efb5441e3877c8a4c919232729d7c574dab", + ID: "sha256:e480047f53bccb8a9107a424fab43452dc59df641022a300d34326639254a0cf", BlobIDs: []string{ - "sha256:afc2bc421aac8c61d89d4dd1c1865efb5441e3877c8a4c919232729d7c574dab", + "sha256:e480047f53bccb8a9107a424fab43452dc59df641022a300d34326639254a0cf", }, }, }, @@ -125,7 +125,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:afc2bc421aac8c61d89d4dd1c1865efb5441e3877c8a4c919232729d7c574dab", + BlobID: "sha256:e480047f53bccb8a9107a424fab43452dc59df641022a300d34326639254a0cf", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, OS: types.OS{ @@ -175,7 +175,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", + BlobID: "sha256:c91a594202ba114ce4d067d9cac40b545c7ba2a96520c9ca8050b437020c3ab9", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, Applications: []types.Application{ @@ -203,9 +203,9 @@ func TestArtifact_Inspect(t *testing.T) { want: artifact.Reference{ Name: "testdata/requirements.txt", Type: artifact.TypeFilesystem, - ID: "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", + ID: "sha256:c91a594202ba114ce4d067d9cac40b545c7ba2a96520c9ca8050b437020c3ab9", BlobIDs: []string{ - "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", + "sha256:c91a594202ba114ce4d067d9cac40b545c7ba2a96520c9ca8050b437020c3ab9", }, }, }, @@ -216,7 +216,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", + BlobID: "sha256:c91a594202ba114ce4d067d9cac40b545c7ba2a96520c9ca8050b437020c3ab9", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, Applications: []types.Application{ @@ -244,9 +244,9 @@ func TestArtifact_Inspect(t *testing.T) { want: artifact.Reference{ Name: "testdata/requirements.txt", Type: artifact.TypeFilesystem, - ID: "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", + ID: "sha256:c91a594202ba114ce4d067d9cac40b545c7ba2a96520c9ca8050b437020c3ab9", BlobIDs: []string{ - "sha256:0c41376dbbc0dbf18e9dbd36c5de85627007dcf9357fd98f191864d48dd35537", + "sha256:c91a594202ba114ce4d067d9cac40b545c7ba2a96520c9ca8050b437020c3ab9", }, }, }, @@ -341,9 +341,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/single-failure", Type: artifact.TypeFilesystem, - ID: "sha256:1a6ce0acc3b57eb6c830c96fcd868fec1eb4d3b57ad51e481c76d85f22870a65", + ID: "sha256:5e4ca8ffaa89f49c69acfbac6d7cebb0a372b07d4c4c9876f9a58043c8ee56e9", BlobIDs: []string{ - "sha256:1a6ce0acc3b57eb6c830c96fcd868fec1eb4d3b57ad51e481c76d85f22870a65", + "sha256:5e4ca8ffaa89f49c69acfbac6d7cebb0a372b07d4c4c9876f9a58043c8ee56e9", }, }, }, @@ -426,9 +426,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/multiple-failures", Type: artifact.TypeFilesystem, - ID: "sha256:afc20cf0fc99c62bbc79b00cb9fbc70ba7ee76c946a6d560639ba9279344787d", + ID: "sha256:1832cedd0d8baeb172c7297acb56417c0dec454c280a79ee2a8f413e1ffca192", BlobIDs: []string{ - "sha256:afc20cf0fc99c62bbc79b00cb9fbc70ba7ee76c946a6d560639ba9279344787d", + "sha256:1832cedd0d8baeb172c7297acb56417c0dec454c280a79ee2a8f413e1ffca192", }, }, }, @@ -456,9 +456,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/no-results", Type: artifact.TypeFilesystem, - ID: "sha256:1827b6a0b0a17e0d623a2045e9d9c331ef613390eda2fed823969ee0dd730257", + ID: "sha256:00e01cf7bf052dbf8f02739d281675eff16f5ddf004256472cdddb55cf974fd6", BlobIDs: []string{ - "sha256:1827b6a0b0a17e0d623a2045e9d9c331ef613390eda2fed823969ee0dd730257", + "sha256:00e01cf7bf052dbf8f02739d281675eff16f5ddf004256472cdddb55cf974fd6", }, }, }, @@ -505,9 +505,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/passed", Type: artifact.TypeFilesystem, - ID: "sha256:eec58ef10d1b04df4af76b2472e615f8c27e253a16f90d7542670a7001d88915", + ID: "sha256:9ee9fe14c3fcff202bc8c72f9e20a3035442ae394ec5cd201e1cb29d2111b4e1", BlobIDs: []string{ - "sha256:eec58ef10d1b04df4af76b2472e615f8c27e253a16f90d7542670a7001d88915", + "sha256:9ee9fe14c3fcff202bc8c72f9e20a3035442ae394ec5cd201e1cb29d2111b4e1", }, }, }, @@ -571,9 +571,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/busted-relative-paths/child/main.tf", Type: artifact.TypeFilesystem, - ID: "sha256:8db34d644bfb98077180616caeab0b41a26d9029a47a23d4b36e1d6e45584919", + ID: "sha256:c3e6c9e68cd7a9900cd8aa690e1d5af174ddcc00ddb9438a858c3e329b9ea8f4", BlobIDs: []string{ - "sha256:8db34d644bfb98077180616caeab0b41a26d9029a47a23d4b36e1d6e45584919", + "sha256:c3e6c9e68cd7a9900cd8aa690e1d5af174ddcc00ddb9438a858c3e329b9ea8f4", }, }, }, @@ -621,9 +621,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/tfvar-outside/tf", Type: artifact.TypeFilesystem, - ID: "sha256:eec58ef10d1b04df4af76b2472e615f8c27e253a16f90d7542670a7001d88915", + ID: "sha256:9ee9fe14c3fcff202bc8c72f9e20a3035442ae394ec5cd201e1cb29d2111b4e1", BlobIDs: []string{ - "sha256:eec58ef10d1b04df4af76b2472e615f8c27e253a16f90d7542670a7001d88915", + "sha256:9ee9fe14c3fcff202bc8c72f9e20a3035442ae394ec5cd201e1cb29d2111b4e1", }, }, }, @@ -711,9 +711,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/relative-paths/child", Type: artifact.TypeFilesystem, - ID: "sha256:f13b89447db61be1c1e4099ef18aec7272091f8f2d3581643a9d1fabc74eda83", + ID: "sha256:44d28d3fc115f1f8dd3f8978c7e9432ba255dd82661c3810178276897e2d8fb7", BlobIDs: []string{ - "sha256:f13b89447db61be1c1e4099ef18aec7272091f8f2d3581643a9d1fabc74eda83", + "sha256:44d28d3fc115f1f8dd3f8978c7e9432ba255dd82661c3810178276897e2d8fb7", }, }, }, @@ -830,9 +830,9 @@ func TestTerraformPlanSnapshotMisconfScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraformplan/snapshots/single-failure", Type: artifact.TypeFilesystem, - ID: "sha256:732c38451bde877a94d1ff5b6f2019655bed04fd24b1169de195eeee1199045e", + ID: "sha256:39babdcb854331c58dc1e20c20dc67c2b4bda23895bf2861cc72d187de2bc716", BlobIDs: []string{ - "sha256:732c38451bde877a94d1ff5b6f2019655bed04fd24b1169de195eeee1199045e", + "sha256:39babdcb854331c58dc1e20c20dc67c2b4bda23895bf2861cc72d187de2bc716", }, }, }, @@ -906,9 +906,9 @@ func TestTerraformPlanSnapshotMisconfScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraformplan/snapshots/multiple-failures", Type: artifact.TypeFilesystem, - ID: "sha256:752c0b470adfcfe7e21892cf4c6fc3bc28dba873c9c1696f40b71b7a51ad7231", + ID: "sha256:32debc3c2857404ceeec978a938609a4b20f1c53c304603815955885377f286c", BlobIDs: []string{ - "sha256:752c0b470adfcfe7e21892cf4c6fc3bc28dba873c9c1696f40b71b7a51ad7231", + "sha256:32debc3c2857404ceeec978a938609a4b20f1c53c304603815955885377f286c", }, }, }, @@ -946,9 +946,9 @@ func TestTerraformPlanSnapshotMisconfScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraformplan/snapshots/passed", Type: artifact.TypeFilesystem, - ID: "sha256:8947704a08f54ab1df32cd905d6bca72edf1785a42702968bafa331172da7176", + ID: "sha256:2432b64e4583676ec1e94903d31c4faf79a1e0f4ed49bf24aef2bf9b44517ca2", BlobIDs: []string{ - "sha256:8947704a08f54ab1df32cd905d6bca72edf1785a42702968bafa331172da7176", + "sha256:2432b64e4583676ec1e94903d31c4faf79a1e0f4ed49bf24aef2bf9b44517ca2", }, }, }, @@ -1061,9 +1061,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/single-failure/src", Type: artifact.TypeFilesystem, - ID: "sha256:889a94522970c6e55f1f7543914b2f0131f79f9c4526445fb95309f64a9947d7", + ID: "sha256:43bb7ce5253686cf96a68e6d4bd30c33e163019ae2893968602da9e5e86b1aab", BlobIDs: []string{ - "sha256:889a94522970c6e55f1f7543914b2f0131f79f9c4526445fb95309f64a9947d7", + "sha256:43bb7ce5253686cf96a68e6d4bd30c33e163019ae2893968602da9e5e86b1aab", }, }, }, @@ -1145,9 +1145,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/multiple-failures/src", Type: artifact.TypeFilesystem, - ID: "sha256:17c9c72a759856445e6d3847b2d5ed90c3bad3e4ee50cea0c812ef53c179f8ca", + ID: "sha256:4609d64f57187099483e76a979d2f74862a092a5f42a9945e6f1242c372a811c", BlobIDs: []string{ - "sha256:17c9c72a759856445e6d3847b2d5ed90c3bad3e4ee50cea0c812ef53c179f8ca", + "sha256:4609d64f57187099483e76a979d2f74862a092a5f42a9945e6f1242c372a811c", }, }, }, @@ -1177,9 +1177,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/no-results/src", Type: artifact.TypeFilesystem, - ID: "sha256:26c76a2cb55cb0ef2c3a2dd79e237bddb508ca2c4cefdb103698a1972c8a9c2d", + ID: "sha256:a3d40c3290bf45542f9c0aa364f2c16aca06cdccb4868a2e442b7e6a56c6157a", BlobIDs: []string{ - "sha256:26c76a2cb55cb0ef2c3a2dd79e237bddb508ca2c4cefdb103698a1972c8a9c2d", + "sha256:a3d40c3290bf45542f9c0aa364f2c16aca06cdccb4868a2e442b7e6a56c6157a", }, }, }, @@ -1235,9 +1235,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/params/code/src", Type: artifact.TypeFilesystem, - ID: "sha256:267b572211115db6a2a4484a02317fbb6d4f050da0e95b1db4243d49889483de", + ID: "sha256:abaeb2f443a59940afd63e015d0bb737127ebde06306840f529dd40d65390703", BlobIDs: []string{ - "sha256:267b572211115db6a2a4484a02317fbb6d4f050da0e95b1db4243d49889483de", + "sha256:abaeb2f443a59940afd63e015d0bb737127ebde06306840f529dd40d65390703", }, }, }, @@ -1293,9 +1293,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/passed/src", Type: artifact.TypeFilesystem, - ID: "sha256:8ca92725ce2f47b7ffb1b0a9e0359d59ac2b3b3f517ba42f66a859436057e54a", + ID: "sha256:73bb13d7c722bfcdf6c00f51f368befda453e1acba31ba62aad31b5b04b235e8", BlobIDs: []string{ - "sha256:8ca92725ce2f47b7ffb1b0a9e0359d59ac2b3b3f517ba42f66a859436057e54a", + "sha256:73bb13d7c722bfcdf6c00f51f368befda453e1acba31ba62aad31b5b04b235e8", }, }, }, @@ -1381,9 +1381,9 @@ func TestDockerfileMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/dockerfile/single-failure/src", Type: artifact.TypeFilesystem, - ID: "sha256:627cbf451ec7929dfe5151dfe0e2305ed855906bf79136f198528a0cc3f6e4f9", + ID: "sha256:059f081288e7ea418286bddcee78167e8ebf33b47e365c20754b6cfa180d997d", BlobIDs: []string{ - "sha256:627cbf451ec7929dfe5151dfe0e2305ed855906bf79136f198528a0cc3f6e4f9", + "sha256:059f081288e7ea418286bddcee78167e8ebf33b47e365c20754b6cfa180d997d", }, }, }, @@ -1439,9 +1439,9 @@ func TestDockerfileMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/dockerfile/multiple-failures/src", Type: artifact.TypeFilesystem, - ID: "sha256:627cbf451ec7929dfe5151dfe0e2305ed855906bf79136f198528a0cc3f6e4f9", + ID: "sha256:059f081288e7ea418286bddcee78167e8ebf33b47e365c20754b6cfa180d997d", BlobIDs: []string{ - "sha256:627cbf451ec7929dfe5151dfe0e2305ed855906bf79136f198528a0cc3f6e4f9", + "sha256:059f081288e7ea418286bddcee78167e8ebf33b47e365c20754b6cfa180d997d", }, }, }, @@ -1469,9 +1469,9 @@ func TestDockerfileMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/dockerfile/no-results/src", Type: artifact.TypeFilesystem, - ID: "sha256:26c76a2cb55cb0ef2c3a2dd79e237bddb508ca2c4cefdb103698a1972c8a9c2d", + ID: "sha256:a3d40c3290bf45542f9c0aa364f2c16aca06cdccb4868a2e442b7e6a56c6157a", BlobIDs: []string{ - "sha256:26c76a2cb55cb0ef2c3a2dd79e237bddb508ca2c4cefdb103698a1972c8a9c2d", + "sha256:a3d40c3290bf45542f9c0aa364f2c16aca06cdccb4868a2e442b7e6a56c6157a", }, }, }, @@ -1529,9 +1529,9 @@ func TestDockerfileMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/dockerfile/passed/src", Type: artifact.TypeFilesystem, - ID: "sha256:4cc7f6bba417cc65c5391bc9c07fd1e205e21bdec87b271889433af18be1e454", + ID: "sha256:5f6504b01b68ef1418d59210c0c7b3604fe63f8575a72721d1172d9d2fdd2e23", BlobIDs: []string{ - "sha256:4cc7f6bba417cc65c5391bc9c07fd1e205e21bdec87b271889433af18be1e454", + "sha256:5f6504b01b68ef1418d59210c0c7b3604fe63f8575a72721d1172d9d2fdd2e23", }, }, }, @@ -1621,9 +1621,9 @@ func TestKubernetesMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/kubernetes/single-failure/src", Type: artifact.TypeFilesystem, - ID: "sha256:d5ca0b4e96aaaeafa424a2250db6297a5182cb6ca5db49bc1ff11790f6cdbee9", + ID: "sha256:14768f3c82388c43e0af6579632535b5ecd6bd4ac802d063d74b636725191fe9", BlobIDs: []string{ - "sha256:d5ca0b4e96aaaeafa424a2250db6297a5182cb6ca5db49bc1ff11790f6cdbee9", + "sha256:14768f3c82388c43e0af6579632535b5ecd6bd4ac802d063d74b636725191fe9", }, }, }, @@ -1707,9 +1707,9 @@ func TestKubernetesMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/kubernetes/multiple-failures/src", Type: artifact.TypeFilesystem, - ID: "sha256:eef9fff2fe8f5c4a123c018b4f91db25d9676e7d171a3a683c2fbfbbbe82fa54", + ID: "sha256:d7b5fe7fb0e8d51cb6a06b5f2027b1e976e25a2462ff40241f86e13e69ff210c", BlobIDs: []string{ - "sha256:eef9fff2fe8f5c4a123c018b4f91db25d9676e7d171a3a683c2fbfbbbe82fa54", + "sha256:d7b5fe7fb0e8d51cb6a06b5f2027b1e976e25a2462ff40241f86e13e69ff210c", }, }, }, @@ -1737,9 +1737,9 @@ func TestKubernetesMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/kubernetes/no-results/src", Type: artifact.TypeFilesystem, - ID: "sha256:2b54cf33feaa1fe1f5bf223f873ca6c3f7c3693b0bb3b0ce9e2e7fd79cd37b5a", + ID: "sha256:ddd0e363b0ebab71250f9d36de65d485ca2faa9510ab6ddea940de7af8267b67", BlobIDs: []string{ - "sha256:2b54cf33feaa1fe1f5bf223f873ca6c3f7c3693b0bb3b0ce9e2e7fd79cd37b5a", + "sha256:ddd0e363b0ebab71250f9d36de65d485ca2faa9510ab6ddea940de7af8267b67", }, }, }, @@ -1797,9 +1797,9 @@ func TestKubernetesMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/kubernetes/passed/src", Type: artifact.TypeFilesystem, - ID: "sha256:dc7a0fd3ea2f13b0ea05f4e517a16e602b0fc17fbd72aa5e34107ef12a91a30b", + ID: "sha256:af77d733739a4366b6e03e95605f5282262d45905efed501d83ec7ecc97a7574", BlobIDs: []string{ - "sha256:dc7a0fd3ea2f13b0ea05f4e517a16e602b0fc17fbd72aa5e34107ef12a91a30b", + "sha256:af77d733739a4366b6e03e95605f5282262d45905efed501d83ec7ecc97a7574", }, }, }, @@ -1886,9 +1886,9 @@ func TestAzureARMMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/azurearm/single-failure/src", Type: artifact.TypeFilesystem, - ID: "sha256:c1a8bfd544b9041ad194382cc42b54289f70966d061ef501b267aec8fd07c5df", + ID: "sha256:ff25ae7d346f724faf4d2653868bc249bf221460bcf6d364cf50372c303342d2", BlobIDs: []string{ - "sha256:c1a8bfd544b9041ad194382cc42b54289f70966d061ef501b267aec8fd07c5df", + "sha256:ff25ae7d346f724faf4d2653868bc249bf221460bcf6d364cf50372c303342d2", }, }, }, @@ -1968,9 +1968,9 @@ func TestAzureARMMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/azurearm/multiple-failures/src", Type: artifact.TypeFilesystem, - ID: "sha256:75bf0e88f8d2857be90fb8d10a350c04c1532ba7f510e1eb404a8bae30ce97d8", + ID: "sha256:e5c5ad752c6ff8d26f1b6764a9fb6dec345157df7c816b86d491b0af9e9f4ae6", BlobIDs: []string{ - "sha256:75bf0e88f8d2857be90fb8d10a350c04c1532ba7f510e1eb404a8bae30ce97d8", + "sha256:e5c5ad752c6ff8d26f1b6764a9fb6dec345157df7c816b86d491b0af9e9f4ae6", }, }, }, @@ -1998,9 +1998,9 @@ func TestAzureARMMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/azurearm/no-results/src", Type: artifact.TypeFilesystem, - ID: "sha256:26c76a2cb55cb0ef2c3a2dd79e237bddb508ca2c4cefdb103698a1972c8a9c2d", + ID: "sha256:a3d40c3290bf45542f9c0aa364f2c16aca06cdccb4868a2e442b7e6a56c6157a", BlobIDs: []string{ - "sha256:26c76a2cb55cb0ef2c3a2dd79e237bddb508ca2c4cefdb103698a1972c8a9c2d", + "sha256:a3d40c3290bf45542f9c0aa364f2c16aca06cdccb4868a2e442b7e6a56c6157a", }, }, }, @@ -2054,9 +2054,9 @@ func TestAzureARMMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/azurearm/passed/src", Type: artifact.TypeFilesystem, - ID: "sha256:b9ba7c4eafec405c8b6998dbb98ee1c7f7830caf8487fd1461433ff82d8779e9", + ID: "sha256:0702eea6f699984f98c788c737f3adf74ae00db2b24b7d44577c7eedc31b1eb1", BlobIDs: []string{ - "sha256:b9ba7c4eafec405c8b6998dbb98ee1c7f7830caf8487fd1461433ff82d8779e9", + "sha256:0702eea6f699984f98c788c737f3adf74ae00db2b24b7d44577c7eedc31b1eb1", }, }, }, From 1e61514cf22888ab34fb8f2ac5d4a87e923cbb2d Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 16:29:51 +0600 Subject: [PATCH 09/24] refactor --- pkg/fanal/analyzer/language/python/pip/pip.go | 63 +++++++++++-------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 4f747d91fcb6..389bd07d111e 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -2,6 +2,7 @@ package pip import ( "context" + xio "github.com/aquasecurity/trivy/pkg/x/io" "io" "io/fs" "os" @@ -19,7 +20,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/utils/fsutils" - xio "github.com/aquasecurity/trivy/pkg/x/io" ) func init() { @@ -44,33 +44,10 @@ func newPipLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, e func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { var apps []types.Application - var licenses = make(map[string][]string) - - if libDir, err := getPythonSitePackagesDir(); err != nil || libDir == "" { - a.logger.Warn("Unable to find python `lib` directory. License detection is skipped.", log.Err(err)) - } else { - requiredMetadata := func(filePath string, _ fs.DirEntry) bool { - return strings.HasSuffix(filepath.Dir(filePath), ".dist-info") && filepath.Base(filePath) == "METADATA" - } - - // Detect licenses from python lib directory - if err = fsutils.WalkDir(os.DirFS(libDir), ".", requiredMetadata, func(path string, d fs.DirEntry, r io.Reader) error { - rs, err := xio.NewReadSeekerAt(r) - if err != nil { - return xerrors.Errorf("Unable to convert reader: %w", err) - } - metadataPkg, _, err := a.metadataParser.Parse(rs) - if err != nil { - return xerrors.Errorf("metadata parse error: %w", err) - } - - // METADATA file contains info about only 1 package - licenses[packageID(metadataPkg[0].Name, metadataPkg[0].Version)] = metadataPkg[0].Licenses - return nil - }); err != nil { - return nil, xerrors.Errorf("walk python lib dir error: %w", err) - } + licenses, err := a.getLicenses() + if err != nil { + a.logger.Warn("Unable to find python `site-packages` directory. License detection is skipped.", log.Err(err)) } // We only saved the `requirement.txt` files @@ -124,6 +101,38 @@ func packageID(name, ver string) string { return dependency.ID(types.Pip, name, ver) } +func (a pipLibraryAnalyzer) getLicenses() (map[string][]string, error) { + var licenses = make(map[string][]string) + + spDir, err := getPythonSitePackagesDir() + if err != nil { + return nil, xerrors.Errorf("Unable to find python `site-packages` directory: %w", err) + } + requiredMetadata := func(filePath string, _ fs.DirEntry) bool { + return strings.HasSuffix(filepath.Dir(filePath), ".dist-info") && filepath.Base(filePath) == "METADATA" + } + + // Detect licenses from `site-packages` directory + if err = fsutils.WalkDir(os.DirFS(spDir), ".", requiredMetadata, func(path string, d fs.DirEntry, r io.Reader) error { + rs, err := xio.NewReadSeekerAt(r) + if err != nil { + return xerrors.Errorf("Unable to convert reader: %w", err) + } + + metadataPkg, _, err := a.metadataParser.Parse(rs) + if err != nil { + return xerrors.Errorf("metadata parse error: %w", err) + } + + // METADATA file contains info about only 1 package + licenses[packageID(metadataPkg[0].Name, metadataPkg[0].Version)] = metadataPkg[0].Licenses + return nil + }); err != nil { + return nil, xerrors.Errorf("walk site-packages dir error: %w", err) + } + return licenses, nil +} + func getPythonSitePackagesDir() (string, error) { // check VIRTUAL_ENV first if venv := os.Getenv("VIRTUAL_ENV"); venv != "" { From 7fe5333db157cb103c4f5653873d9b69cce701f4 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 16:51:43 +0600 Subject: [PATCH 10/24] refactor --- pkg/fanal/analyzer/language/python/pip/pip.go | 39 +++++++++++-------- .../analyzer/language/python/pip/pip_test.go | 4 +- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 389bd07d111e..9161ef062e9c 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -45,7 +45,7 @@ func newPipLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, e func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { var apps []types.Application - licenses, err := a.getLicenses() + licenses, err := a.licenseList() if err != nil { a.logger.Warn("Unable to find python `site-packages` directory. License detection is skipped.", log.Err(err)) } @@ -97,14 +97,11 @@ func (a pipLibraryAnalyzer) Version() int { return version } -func packageID(name, ver string) string { - return dependency.ID(types.Pip, name, ver) -} - -func (a pipLibraryAnalyzer) getLicenses() (map[string][]string, error) { +// licenseList returns list of licenses found in METADATA files in the `site-packages` directory. +func (a pipLibraryAnalyzer) licenseList() (map[string][]string, error) { var licenses = make(map[string][]string) - spDir, err := getPythonSitePackagesDir() + spDir, err := pythonSitePackagesDir() if err != nil { return nil, xerrors.Errorf("Unable to find python `site-packages` directory: %w", err) } @@ -133,7 +130,8 @@ func (a pipLibraryAnalyzer) getLicenses() (map[string][]string, error) { return licenses, nil } -func getPythonSitePackagesDir() (string, error) { +// pythonSitePackagesDir returns path to site-packages dir +func pythonSitePackagesDir() (string, error) { // check VIRTUAL_ENV first if venv := os.Getenv("VIRTUAL_ENV"); venv != "" { libDir := filepath.Join(venv, "lib") @@ -141,7 +139,7 @@ func getPythonSitePackagesDir() (string, error) { return "", xerrors.Errorf("Unable to detect `lib` dir for %q venv: %w", venv, err) } - spDir, err := sitePackagesDir(libDir) + spDir, err := findSitePackagesDir(libDir) if err != nil { return "", xerrors.Errorf("Unable to detect `site-packages` dir for %q venv: %w", spDir, err) } @@ -152,7 +150,7 @@ func getPythonSitePackagesDir() (string, error) { } // Find path to Python executable - pythonExecPath, err := getPythonExecutablePath() + pythonExecPath, err := pythonExecutablePath() if err != nil { return "", err } @@ -160,7 +158,7 @@ func getPythonSitePackagesDir() (string, error) { // Search for a directory starting with "python" in the lib directory libDir := filepath.Join(pythonExecDir, "..", "lib") - spDir, err := sitePackagesDir(libDir) + spDir, err := findSitePackagesDir(libDir) if err != nil { return "", xerrors.Errorf("Unable to detect `site-packages` dir for %q: %w", pythonExecPath, err) } @@ -170,15 +168,15 @@ func getPythonSitePackagesDir() (string, error) { // Try another common pattern if the Python library directory is not found spDir = filepath.Join(pythonExecDir, "..", "..", "lib", "site-packages") - _, err = os.Stat(spDir) - if os.IsNotExist(err) { + if _, err = os.Stat(spDir); os.IsNotExist(err) { return "", xerrors.Errorf("site-packages directory not found") } return spDir, nil } -func getPythonExecutablePath() (string, error) { +// pythonExecutablePath returns path to Python executable +func pythonExecutablePath() (string, error) { for _, execName := range pythonExecNames { // Get the absolute path of the python command pythonPath, err := exec.LookPath(execName) @@ -190,7 +188,8 @@ func getPythonExecutablePath() (string, error) { return "", xerrors.Errorf("Unable to find path to Python executable") } -func sitePackagesDir(libDir string) (string, error) { +// findSitePackagesDir finds `site-packages` dir in `lib` dir +func findSitePackagesDir(libDir string) (string, error) { entries, err := os.ReadDir(libDir) if err != nil { if !os.IsNotExist(err) { @@ -204,8 +203,16 @@ func sitePackagesDir(libDir string) (string, error) { for _, entry := range entries { if entry.IsDir() && strings.HasPrefix(entry.Name(), "python") { // Found a directory starting with "python", assume it's the Python library directory - spDir = filepath.Join(libDir, entry.Name(), "site-packages") + dir := filepath.Join(libDir, entry.Name(), "site-packages") + if _, err = os.Stat(dir); !os.IsNotExist(err) { + spDir = filepath.Join(libDir, entry.Name(), "site-packages") + } + } } return spDir, nil } + +func packageID(name, ver string) string { + return dependency.ID(types.Pip, name, ver) +} diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index 985802686023..fc5f7b7b0c25 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -203,7 +203,7 @@ func Test_pipAnalyzer_Required(t *testing.T) { } } -func Test_getPythonExecutablePath(t *testing.T) { +func Test_pythonExecutablePath(t *testing.T) { tests := []struct { name string execName string @@ -243,7 +243,7 @@ func Test_getPythonExecutablePath(t *testing.T) { t.Setenv("PATH", binDir) - path, err := getPythonExecutablePath() + path, err := pythonExecutablePath() if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return From a1e471c36cfc719c7ba45db6e57f7482fb547aae Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 16:53:39 +0600 Subject: [PATCH 11/24] fix linter error --- pkg/fanal/analyzer/language/python/pip/pip.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 9161ef062e9c..990fc384a47a 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -2,7 +2,6 @@ package pip import ( "context" - xio "github.com/aquasecurity/trivy/pkg/x/io" "io" "io/fs" "os" @@ -20,6 +19,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/utils/fsutils" + xio "github.com/aquasecurity/trivy/pkg/x/io" ) func init() { From 544adcf916a48c10b1f7de5f14277c8479ffe455 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 17:29:01 +0600 Subject: [PATCH 12/24] fix windows tests --- .../analyzer/language/python/pip/pip_test.go | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index fc5f7b7b0c25..d360d0d3d0b0 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "runtime" "strings" "testing" @@ -144,6 +145,8 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { if tt.venv != "" { t.Setenv("VIRTUAL_ENV", tt.venv) } + + var newPATH string if tt.pythonExecDir != "" { err := os.MkdirAll(tt.pythonExecDir, os.ModePerm) require.NoError(t, err) @@ -155,14 +158,19 @@ func Test_pipAnalyzer_Analyze(t *testing.T) { require.NoError(t, err) }() + pythonExecFileName := "python" + if runtime.GOOS == "windows" { + pythonExecFileName = "python.exe" + } // create temp python3 Executable - err = os.WriteFile(filepath.Join(tt.pythonExecDir, "python3"), nil, 0755) + err = os.WriteFile(filepath.Join(tt.pythonExecDir, pythonExecFileName), nil, 0755) require.NoError(t, err) - absPath, err := filepath.Abs(tt.pythonExecDir) + newPATH, err = filepath.Abs(tt.pythonExecDir) require.NoError(t, err) - t.Setenv("PATH", absPath) + } + t.Setenv("PATH", newPATH) a, err := newPipLibraryAnalyzer(analyzer.AnalyzerOptions{}) require.NoError(t, err) @@ -221,10 +229,6 @@ func Test_pythonExecutablePath(t *testing.T) { name: "happy path with `python2` filename", execName: "python2", }, - { - name: "happy path with `python.exe` filename", - execName: "python.exe", - }, { name: "sad path. Python executable not found", execName: "python-wrong", @@ -238,6 +242,9 @@ func Test_pythonExecutablePath(t *testing.T) { err := os.MkdirAll(binDir, os.ModePerm) require.NoError(t, err) + if runtime.GOOS == "windows" { + tt.execName = tt.execName + ".exe" + } err = os.WriteFile(filepath.Join(binDir, tt.execName), nil, 0755) require.NoError(t, err) From a96e5f84b977a27aeb632344ebe2acbf9843f81f Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 27 May 2024 17:37:07 +0600 Subject: [PATCH 13/24] fix linter error --- pkg/fanal/analyzer/language/python/pip/pip_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index d360d0d3d0b0..86ece4132b31 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -243,7 +243,7 @@ func Test_pythonExecutablePath(t *testing.T) { require.NoError(t, err) if runtime.GOOS == "windows" { - tt.execName = tt.execName + ".exe" + tt.execName += ".exe" } err = os.WriteFile(filepath.Join(binDir, tt.execName), nil, 0755) require.NoError(t, err) From beaeb6e2d525d1e4b106a90d161b0004e0975330 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Tue, 28 May 2024 15:16:30 +0600 Subject: [PATCH 14/24] refactor: parse only required METADATA files --- pkg/fanal/analyzer/language/python/pip/pip.go | 60 +++++++++---------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 990fc384a47a..0b6f6312c1b1 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -2,6 +2,7 @@ package pip import ( "context" + "fmt" "io" "io/fs" "os" @@ -19,7 +20,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/utils/fsutils" - xio "github.com/aquasecurity/trivy/pkg/x/io" ) func init() { @@ -45,7 +45,7 @@ func newPipLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, e func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { var apps []types.Application - licenses, err := a.licenseList() + sitePackagesDir, err := pythonSitePackagesDir() if err != nil { a.logger.Warn("Unable to find python `site-packages` directory. License detection is skipped.", log.Err(err)) } @@ -55,7 +55,7 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn return true } - if err := fsutils.WalkDir(input.FS, ".", required, func(pathPath string, d fs.DirEntry, r io.Reader) error { + if err = fsutils.WalkDir(input.FS, ".", required, func(pathPath string, d fs.DirEntry, r io.Reader) error { app, err := language.Parse(types.Pip, pathPath, r, pip.NewParser()) if err != nil { return xerrors.Errorf("unable to parse requirements.txt: %w", err) @@ -66,10 +66,9 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn } // Fill licenses - for i, pkg := range app.Packages { - pkgID := packageID(pkg.Name, pkg.Version) - if lics, ok := licenses[pkgID]; ok { - app.Packages[i].Licenses = lics + if sitePackagesDir != "" { + for i := range app.Packages { + app.Packages[i].Licenses = a.pkgLicense(app.Packages[i].Name, app.Packages[i].Version, sitePackagesDir) } } @@ -97,37 +96,32 @@ func (a pipLibraryAnalyzer) Version() int { return version } -// licenseList returns list of licenses found in METADATA files in the `site-packages` directory. -func (a pipLibraryAnalyzer) licenseList() (map[string][]string, error) { - var licenses = make(map[string][]string) - - spDir, err := pythonSitePackagesDir() - if err != nil { - return nil, xerrors.Errorf("Unable to find python `site-packages` directory: %w", err) - } - requiredMetadata := func(filePath string, _ fs.DirEntry) bool { - return strings.HasSuffix(filepath.Dir(filePath), ".dist-info") && filepath.Base(filePath) == "METADATA" +// pkgLicense parses `METADATA` pkg file to look for licenses +func (a pipLibraryAnalyzer) pkgLicense(pkgName, pkgVer, spDir string) []string { + // Don't look for licenses if `site-packages` directory is not found + if spDir == "" { + return nil } - // Detect licenses from `site-packages` directory - if err = fsutils.WalkDir(os.DirFS(spDir), ".", requiredMetadata, func(path string, d fs.DirEntry, r io.Reader) error { - rs, err := xio.NewReadSeekerAt(r) - if err != nil { - return xerrors.Errorf("Unable to convert reader: %w", err) - } - - metadataPkg, _, err := a.metadataParser.Parse(rs) - if err != nil { - return xerrors.Errorf("metadata parse error: %w", err) - } + // METADATA path is `**/site-packages/-.dist-info/METADATA` + pkgDir := fmt.Sprintf("%s-%s.dist-info", pkgName, pkgVer) + metadataPath := filepath.Join(spDir, pkgDir, "METADATA") + metadataFile, err := os.Open(metadataPath) + if os.IsNotExist(err) { + a.logger.Debug("site-packages directory doesn't contain package", log.String("site-packages dir", pkgDir), + log.String("name", pkgName), log.String("version", pkgVer)) + return nil + } - // METADATA file contains info about only 1 package - licenses[packageID(metadataPkg[0].Name, metadataPkg[0].Version)] = metadataPkg[0].Licenses + metadataPkg, _, err := a.metadataParser.Parse(metadataFile) + if err != nil { + a.logger.Warn("Unable to parse METADATA file", log.String("path", metadataPath), log.Err(err)) return nil - }); err != nil { - return nil, xerrors.Errorf("walk site-packages dir error: %w", err) } - return licenses, nil + + // METADATA file contains info about only 1 package + // cf. https://github.com/aquasecurity/trivy/blob/e66dbb935764908f0b2b9a55cbfe6c107f101a31/pkg/dependency/parser/python/packaging/parse.go#L86-L92 + return metadataPkg[0].Licenses } // pythonSitePackagesDir returns path to site-packages dir From 9ee8066ed035b0be8ef622276666cbbad994acaf Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Tue, 28 May 2024 15:28:40 +0600 Subject: [PATCH 15/24] refactor: remove unused `packageID()` --- pkg/fanal/analyzer/language/python/pip/pip.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 0b6f6312c1b1..0cdb3b5d5081 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -12,7 +12,6 @@ import ( "golang.org/x/xerrors" - "github.com/aquasecurity/trivy/pkg/dependency" "github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging" "github.com/aquasecurity/trivy/pkg/dependency/parser/python/pip" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" @@ -206,7 +205,3 @@ func findSitePackagesDir(libDir string) (string, error) { } return spDir, nil } - -func packageID(name, ver string) string { - return dependency.ID(types.Pip, name, ver) -} From 48617b2433a7b729af188625c88d9aca1bfe3a3e Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Tue, 28 May 2024 18:53:38 +0600 Subject: [PATCH 16/24] refactor: sort python dirs according to major and minor versions --- pkg/fanal/analyzer/language/python/pip/pip.go | 54 ++++++++++++++++--- .../analyzer/language/python/pip/pip_test.go | 33 ++++++++++++ 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 0cdb3b5d5081..29a82acc4add 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -8,6 +8,8 @@ import ( "os" "os/exec" "path/filepath" + "sort" + "strconv" "strings" "golang.org/x/xerrors" @@ -193,15 +195,53 @@ func findSitePackagesDir(libDir string) (string, error) { // Use latest python dir var spDir string + for _, pythonDir := range sortPythonDirs(entries) { + // Found a directory starting with "python", assume it's the Python library directory + dir := filepath.Join(libDir, pythonDir, "site-packages") + if _, err = os.Stat(dir); !os.IsNotExist(err) { + spDir = filepath.Join(libDir, pythonDir, "site-packages") + } + + } + return spDir, nil +} + +// sortPythonDirs finds `python` dirs and sorts them according to major and minor versions. +// e.g. python2.7 => python3.9 => python3.11 +func sortPythonDirs(entries []os.DirEntry) []string { + var pythonDirs []string for _, entry := range entries { if entry.IsDir() && strings.HasPrefix(entry.Name(), "python") { - // Found a directory starting with "python", assume it's the Python library directory - dir := filepath.Join(libDir, entry.Name(), "site-packages") - if _, err = os.Stat(dir); !os.IsNotExist(err) { - spDir = filepath.Join(libDir, entry.Name(), "site-packages") - } - + pythonDirs = append(pythonDirs, entry.Name()) } } - return spDir, nil + + sort.Slice(pythonDirs, func(i, j int) bool { + majorVer1, minorVer1, found1 := strings.Cut(pythonDirs[i], ".") + majorVer2, minorVer2, found2 := strings.Cut(pythonDirs[j], ".") + + // if one of dir doesn't contain minor version => sort as strings + if !found1 || !found2 { + return pythonDirs[i] < pythonDirs[j] + } + + // Sort major versions as strings + // e.g. `python2.7` and `python3.10` + if majorVer1 != majorVer2 { + return majorVer1 < majorVer2 + } + + // Convert minor versions + ver1, err1 := strconv.Atoi(minorVer1) + ver2, err2 := strconv.Atoi(minorVer2) + + // If we can't convert minor versions => sort as strings + if err1 != nil || err2 != nil { + return pythonDirs[i] < pythonDirs[j] + } + + return ver1 < ver2 + }) + + return pythonDirs } diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index 86ece4132b31..66419cd2c70d 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -260,3 +260,36 @@ func Test_pythonExecutablePath(t *testing.T) { }) } } + +func Test_sortPythonDirs(t *testing.T) { + dirs := []string{ + "wrong", + "wrong2.7", + "python3.11", + "python3.10", + "python2.7", + "python3.9", + "python3", + "python2", + } + wantDirs := []string{ + "python2", + "python2.7", + "python3", + "python3.9", + "python3.10", + "python3.11", + } + + tmp := t.TempDir() + for _, dir := range dirs { + err := os.Mkdir(filepath.Join(tmp, dir), os.ModePerm) + require.NoError(t, err) + } + + tmpDir, err := os.ReadDir(tmp) + require.NoError(t, err) + + got := sortPythonDirs(tmpDir) + require.Equal(t, wantDirs, got) +} From 226b3dac9e64dec6acf557166dd6c6d06d1009e4 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 30 May 2024 10:59:13 +0600 Subject: [PATCH 17/24] refactor: use `go-version` to sort python dirs --- pkg/fanal/analyzer/language/python/pip/pip.go | 73 ++++++++----------- .../analyzer/language/python/pip/pip_test.go | 7 +- 2 files changed, 35 insertions(+), 45 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 29a82acc4add..57a756d87719 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -9,11 +9,12 @@ import ( "os/exec" "path/filepath" "sort" - "strconv" "strings" + "github.com/samber/lo" "golang.org/x/xerrors" + goversion "github.com/aquasecurity/go-version/pkg/version" "github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging" "github.com/aquasecurity/trivy/pkg/dependency/parser/python/pip" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" @@ -46,7 +47,7 @@ func newPipLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, e func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { var apps []types.Application - sitePackagesDir, err := pythonSitePackagesDir() + sitePackagesDir, err := a.pythonSitePackagesDir() if err != nil { a.logger.Warn("Unable to find python `site-packages` directory. License detection is skipped.", log.Err(err)) } @@ -126,7 +127,7 @@ func (a pipLibraryAnalyzer) pkgLicense(pkgName, pkgVer, spDir string) []string { } // pythonSitePackagesDir returns path to site-packages dir -func pythonSitePackagesDir() (string, error) { +func (a pipLibraryAnalyzer) pythonSitePackagesDir() (string, error) { // check VIRTUAL_ENV first if venv := os.Getenv("VIRTUAL_ENV"); venv != "" { libDir := filepath.Join(venv, "lib") @@ -134,7 +135,7 @@ func pythonSitePackagesDir() (string, error) { return "", xerrors.Errorf("Unable to detect `lib` dir for %q venv: %w", venv, err) } - spDir, err := findSitePackagesDir(libDir) + spDir, err := a.findSitePackagesDir(libDir) if err != nil { return "", xerrors.Errorf("Unable to detect `site-packages` dir for %q venv: %w", spDir, err) } @@ -153,7 +154,7 @@ func pythonSitePackagesDir() (string, error) { // Search for a directory starting with "python" in the lib directory libDir := filepath.Join(pythonExecDir, "..", "lib") - spDir, err := findSitePackagesDir(libDir) + spDir, err := a.findSitePackagesDir(libDir) if err != nil { return "", xerrors.Errorf("Unable to detect `site-packages` dir for %q: %w", pythonExecPath, err) } @@ -184,7 +185,7 @@ func pythonExecutablePath() (string, error) { } // findSitePackagesDir finds `site-packages` dir in `lib` dir -func findSitePackagesDir(libDir string) (string, error) { +func (a pipLibraryAnalyzer) findSitePackagesDir(libDir string) (string, error) { entries, err := os.ReadDir(libDir) if err != nil { if !os.IsNotExist(err) { @@ -193,55 +194,39 @@ func findSitePackagesDir(libDir string) (string, error) { return "", nil } - // Use latest python dir - var spDir string - for _, pythonDir := range sortPythonDirs(entries) { - // Found a directory starting with "python", assume it's the Python library directory - dir := filepath.Join(libDir, pythonDir, "site-packages") + // Find python dir which contains `site-packages` dir + // First check for newer versions + pythonDirs := a.sortPythonDirs(entries) + for i := len(pythonDirs) - 1; i >= 0; i-- { + dir := filepath.Join(libDir, pythonDirs[i], "site-packages") if _, err = os.Stat(dir); !os.IsNotExist(err) { - spDir = filepath.Join(libDir, pythonDir, "site-packages") + return filepath.Join(libDir, pythonDirs[i], "site-packages"), nil } - } - return spDir, nil + return "", nil } -// sortPythonDirs finds `python` dirs and sorts them according to major and minor versions. +// sortPythonDirs finds dirs starting with `python` and sorts them // e.g. python2.7 => python3.9 => python3.11 -func sortPythonDirs(entries []os.DirEntry) []string { - var pythonDirs []string +func (a pipLibraryAnalyzer) sortPythonDirs(entries []os.DirEntry) []string { + var pythonVers []goversion.Version for _, entry := range entries { + // Found a directory starting with "python", assume it's the Python library directory if entry.IsDir() && strings.HasPrefix(entry.Name(), "python") { - pythonDirs = append(pythonDirs, entry.Name()) + ver := strings.TrimPrefix(entry.Name(), "python") + v, err := goversion.Parse(ver) + if err != nil { + a.logger.Debug("Unable to parse version from Python dir name", log.String("dir", entry.Name()), log.Err(err)) + continue + } + pythonVers = append(pythonVers, v) } } - sort.Slice(pythonDirs, func(i, j int) bool { - majorVer1, minorVer1, found1 := strings.Cut(pythonDirs[i], ".") - majorVer2, minorVer2, found2 := strings.Cut(pythonDirs[j], ".") - - // if one of dir doesn't contain minor version => sort as strings - if !found1 || !found2 { - return pythonDirs[i] < pythonDirs[j] - } + // Sort Python version + sort.Sort(goversion.Collection(pythonVers)) - // Sort major versions as strings - // e.g. `python2.7` and `python3.10` - if majorVer1 != majorVer2 { - return majorVer1 < majorVer2 - } - - // Convert minor versions - ver1, err1 := strconv.Atoi(minorVer1) - ver2, err2 := strconv.Atoi(minorVer2) - - // If we can't convert minor versions => sort as strings - if err1 != nil || err2 != nil { - return pythonDirs[i] < pythonDirs[j] - } - - return ver1 < ver2 + return lo.Map(pythonVers, func(v goversion.Version, _ int) string { + return "python" + v.String() }) - - return pythonDirs } diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index 66419cd2c70d..569c54885f8d 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -13,6 +13,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/log" ) func Test_pipAnalyzer_Analyze(t *testing.T) { @@ -271,6 +272,7 @@ func Test_sortPythonDirs(t *testing.T) { "python3.9", "python3", "python2", + "pythonBadVer", } wantDirs := []string{ "python2", @@ -290,6 +292,9 @@ func Test_sortPythonDirs(t *testing.T) { tmpDir, err := os.ReadDir(tmp) require.NoError(t, err) - got := sortPythonDirs(tmpDir) + a := pipLibraryAnalyzer{ + logger: log.WithPrefix("pip"), + } + got := a.sortPythonDirs(tmpDir) require.Equal(t, wantDirs, got) } From d33781962a76360326b51d125cc1927460542ee4 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 30 May 2024 11:10:51 +0600 Subject: [PATCH 18/24] refactor: fix typo in comment --- pkg/fanal/analyzer/language/python/pip/pip.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 57a756d87719..09be49f45c88 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -52,7 +52,7 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn a.logger.Warn("Unable to find python `site-packages` directory. License detection is skipped.", log.Err(err)) } - // We only saved the `requirement.txt` files + // We only saved the `requirements.txt` files required := func(_ string, _ fs.DirEntry) bool { return true } From ec15629aec947c453c045809e9ba021a16390a8b Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 30 May 2024 11:20:01 +0400 Subject: [PATCH 19/24] chore: lower-case error messages Signed-off-by: knqyf263 --- pkg/fanal/analyzer/language/python/pip/pip.go | 15 ++++++++++----- .../analyzer/language/python/pip/pip_test.go | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 09be49f45c88..aa506632b8cc 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -30,7 +30,12 @@ func init() { const version = 1 -var pythonExecNames = []string{"python3", "python", "python2", "python.exe"} +var pythonExecNames = []string{ + "python3", + "python", + "python2", + "python.exe", +} type pipLibraryAnalyzer struct { logger *log.Logger @@ -132,12 +137,12 @@ func (a pipLibraryAnalyzer) pythonSitePackagesDir() (string, error) { if venv := os.Getenv("VIRTUAL_ENV"); venv != "" { libDir := filepath.Join(venv, "lib") if _, err := os.Stat(libDir); os.IsNotExist(err) { - return "", xerrors.Errorf("Unable to detect `lib` dir for %q venv: %w", venv, err) + return "", xerrors.Errorf("unable to detect `lib` dir for %q venv: %w", venv, err) } spDir, err := a.findSitePackagesDir(libDir) if err != nil { - return "", xerrors.Errorf("Unable to detect `site-packages` dir for %q venv: %w", spDir, err) + return "", xerrors.Errorf("unable to detect `site-packages` dir for %q venv: %w", spDir, err) } if spDir != "" { @@ -156,7 +161,7 @@ func (a pipLibraryAnalyzer) pythonSitePackagesDir() (string, error) { libDir := filepath.Join(pythonExecDir, "..", "lib") spDir, err := a.findSitePackagesDir(libDir) if err != nil { - return "", xerrors.Errorf("Unable to detect `site-packages` dir for %q: %w", pythonExecPath, err) + return "", xerrors.Errorf("unable to detect `site-packages` dir for %q: %w", pythonExecPath, err) } if spDir != "" { return spDir, nil @@ -181,7 +186,7 @@ func pythonExecutablePath() (string, error) { } return pythonPath, nil } - return "", xerrors.Errorf("Unable to find path to Python executable") + return "", xerrors.Errorf("unable to find path to Python executable") } // findSitePackagesDir finds `site-packages` dir in `lib` dir diff --git a/pkg/fanal/analyzer/language/python/pip/pip_test.go b/pkg/fanal/analyzer/language/python/pip/pip_test.go index 569c54885f8d..e3041d0079c1 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip_test.go +++ b/pkg/fanal/analyzer/language/python/pip/pip_test.go @@ -233,7 +233,7 @@ func Test_pythonExecutablePath(t *testing.T) { { name: "sad path. Python executable not found", execName: "python-wrong", - wantErr: "Unable to find path to Python executable", + wantErr: "unable to find path to Python executable", }, } for _, tt := range tests { From bd5493893f8e0840be78279ad9a76fb3d74d863e Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 30 May 2024 11:22:58 +0400 Subject: [PATCH 20/24] refactor: use fsutils Signed-off-by: knqyf263 --- pkg/fanal/analyzer/language/python/pip/pip.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index aa506632b8cc..16266118ba5b 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -169,7 +169,7 @@ func (a pipLibraryAnalyzer) pythonSitePackagesDir() (string, error) { // Try another common pattern if the Python library directory is not found spDir = filepath.Join(pythonExecDir, "..", "..", "lib", "site-packages") - if _, err = os.Stat(spDir); os.IsNotExist(err) { + if !fsutils.DirExists(spDir) { return "", xerrors.Errorf("site-packages directory not found") } @@ -204,7 +204,7 @@ func (a pipLibraryAnalyzer) findSitePackagesDir(libDir string) (string, error) { pythonDirs := a.sortPythonDirs(entries) for i := len(pythonDirs) - 1; i >= 0; i-- { dir := filepath.Join(libDir, pythonDirs[i], "site-packages") - if _, err = os.Stat(dir); !os.IsNotExist(err) { + if fsutils.DirExists(dir) { return filepath.Join(libDir, pythonDirs[i], "site-packages"), nil } } From 0298bb456a515a4b1d53757375cc530069f460f0 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 30 May 2024 11:23:19 +0400 Subject: [PATCH 21/24] refactor: return the existing variable Signed-off-by: knqyf263 --- pkg/fanal/analyzer/language/python/pip/pip.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 16266118ba5b..766cf2077d55 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -205,7 +205,7 @@ func (a pipLibraryAnalyzer) findSitePackagesDir(libDir string) (string, error) { for i := len(pythonDirs) - 1; i >= 0; i-- { dir := filepath.Join(libDir, pythonDirs[i], "site-packages") if fsutils.DirExists(dir) { - return filepath.Join(libDir, pythonDirs[i], "site-packages"), nil + return dir, nil } } return "", nil From a58205abd14f9758f4754126c0a993f7c3ea101f Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 30 May 2024 11:27:46 +0400 Subject: [PATCH 22/24] refactor: make return statements consistent Signed-off-by: knqyf263 --- pkg/fanal/analyzer/language/python/pip/pip.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 766cf2077d55..c9f8cfa5d550 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -143,9 +143,7 @@ func (a pipLibraryAnalyzer) pythonSitePackagesDir() (string, error) { spDir, err := a.findSitePackagesDir(libDir) if err != nil { return "", xerrors.Errorf("unable to detect `site-packages` dir for %q venv: %w", spDir, err) - } - - if spDir != "" { + } else if spDir != "" { return spDir, nil } } @@ -162,18 +160,17 @@ func (a pipLibraryAnalyzer) pythonSitePackagesDir() (string, error) { spDir, err := a.findSitePackagesDir(libDir) if err != nil { return "", xerrors.Errorf("unable to detect `site-packages` dir for %q: %w", pythonExecPath, err) - } - if spDir != "" { + } else if spDir != "" { return spDir, nil } // Try another common pattern if the Python library directory is not found spDir = filepath.Join(pythonExecDir, "..", "..", "lib", "site-packages") - if !fsutils.DirExists(spDir) { - return "", xerrors.Errorf("site-packages directory not found") + if fsutils.DirExists(spDir) { + return spDir, nil } - return spDir, nil + return "", xerrors.Errorf("site-packages directory not found") } // pythonExecutablePath returns path to Python executable From 96cd594179d43ecdb8d04fccc69c5040431e3831 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 30 May 2024 11:30:55 +0400 Subject: [PATCH 23/24] refactor: some tweaks Signed-off-by: knqyf263 --- pkg/fanal/analyzer/language/python/pip/pip.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index c9f8cfa5d550..85b20a79465c 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -105,17 +105,12 @@ func (a pipLibraryAnalyzer) Version() int { // pkgLicense parses `METADATA` pkg file to look for licenses func (a pipLibraryAnalyzer) pkgLicense(pkgName, pkgVer, spDir string) []string { - // Don't look for licenses if `site-packages` directory is not found - if spDir == "" { - return nil - } - // METADATA path is `**/site-packages/-.dist-info/METADATA` pkgDir := fmt.Sprintf("%s-%s.dist-info", pkgName, pkgVer) metadataPath := filepath.Join(spDir, pkgDir, "METADATA") metadataFile, err := os.Open(metadataPath) if os.IsNotExist(err) { - a.logger.Debug("site-packages directory doesn't contain package", log.String("site-packages dir", pkgDir), + a.logger.Debug("No package metadata found", log.String("site-packages", pkgDir), log.String("name", pkgName), log.String("version", pkgVer)) return nil } From 71d0d102d2568ca6b14ac8cc66b4a5cee108768a Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 30 May 2024 11:38:27 +0400 Subject: [PATCH 24/24] docs: capitalize Signed-off-by: knqyf263 --- docs/docs/coverage/language/python.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/docs/coverage/language/python.md b/docs/docs/coverage/language/python.md index 2266cb9b484b..c4f6b6d83e86 100644 --- a/docs/docs/coverage/language/python.md +++ b/docs/docs/coverage/language/python.md @@ -41,7 +41,7 @@ Trivy parses your files generated by package managers in filesystem/repository s ### pip -#### dependency detection +#### Dependency detection Trivy only parses [version specifiers](https://packaging.python.org/en/latest/specifications/version-specifiers/#id4) with `==` comparison operator and without `.*`. To convert unsupported version specifiers - use the `pip freeze` command. @@ -93,16 +93,16 @@ urllib3==1.26.15 `requirements.txt` files don't contain information about dependencies used for development. Trivy could detect vulnerabilities on the development packages, which not affect your production environment. -#### license detection +#### License detection `requirements.txt` files don't contain information about licenses. Therefore, Trivy checks `METADATA` files from `lib/site-packages` directory. Trivy uses 3 ways to detect `site-packages` directory: -- Checks `VIRTUAL_ENV` enveroment. -- detects path to `python`[^1] binary and checks `../lib/pythonX.Y/site-packages` directory. -- detects path to `python`[^1] binary and checks `../../lib/site-packages` directory. +- Checks `VIRTUAL_ENV` environment variable. +- Detects path to `python`[^1] binary and checks `../lib/pythonX.Y/site-packages` directory. +- Detects path to `python`[^1] binary and checks `../../lib/site-packages` directory. ### Pipenv Trivy parses `Pipfile.lock`.