From 654217a65485ca0a07771ea61071977894eb4920 Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:21:38 +0600 Subject: [PATCH] feat(conda): add licenses support for `environment.yml` files (#6953) Co-authored-by: Teppei Fukuda --- docs/docs/coverage/os/conda.md | 35 +- .../parser/conda/environment/parse.go | 15 +- .../parser/conda/environment/parse_test.go | 333 +++++++++--------- .../language/conda/environment/environment.go | 82 ++++- .../conda/environment/environment_test.go | 63 ++++ .../conda-meta/_libgcc_mutex-0.1-main.json | 5 + .../conda-meta/_openmp_mutex-5.1-1_gnu.json | 5 + .../conda-meta/blas-1.0-openblas.json | 5 + .../conda-meta/bzip2-1.0.8-h5eee18b_6.json | 5 + .../conda/environment/testdata/empty.yaml | 5 + .../testdata/environment-with-licenses.yaml | 9 + 11 files changed, 378 insertions(+), 184 deletions(-) create mode 100644 pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/_libgcc_mutex-0.1-main.json create mode 100644 pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/_openmp_mutex-5.1-1_gnu.json create mode 100644 pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/blas-1.0-openblas.json create mode 100644 pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/bzip2-1.0.8-h5eee18b_6.json create mode 100644 pkg/fanal/analyzer/language/conda/environment/testdata/empty.yaml create mode 100644 pkg/fanal/analyzer/language/conda/environment/testdata/environment-with-licenses.yaml diff --git a/docs/docs/coverage/os/conda.md b/docs/docs/coverage/os/conda.md index 79a49194fd66..10c1e93c1b8c 100644 --- a/docs/docs/coverage/os/conda.md +++ b/docs/docs/coverage/os/conda.md @@ -6,31 +6,38 @@ Trivy supports the following scanners for Conda packages. |:-------------:|:---------:| | SBOM | ✓ | | Vulnerability | - | -| License | ✓[^1] | +| License | ✓ | -## SBOM -Trivy detects packages that have been installed with `Conda`. +## `.json` +### SBOM +Trivy parses `/envs//conda-meta/.json` files to find the dependencies installed in your env. -### `.json` -Trivy parses `/envs//conda-meta/.json` files to find the version and license for the dependencies installed in your env. +### License +The `.json` files contain package license information. +Trivy includes licenses for the packages it finds without having to parse additional files. -### `environment.yml`[^2] -Trivy supports parsing [environment.yml][environment.yml][^2] files to find dependency list. +## `environment.yml`[^1] +### SBOM +Trivy supports parsing [environment.yml][environment.yml][^1] files to find dependency list. -!!! note - License detection is currently not supported. - -`environment.yml`[^2] files supports [version range][env-version-range]. We can't be sure about versions for these dependencies. -Therefore, you need to use `conda env export` command to get dependency list in `Conda` default format before scanning `environment.yml`[^2] file. +`environment.yml`[^1] files supports [version range][env-version-range]. We can't be sure about versions for these dependencies. +Therefore, you need to use `conda env export` command to get dependency list in `Conda` default format before scanning `environment.yml`[^1] file. !!! note For dependencies in a non-Conda format, Trivy doesn't include a version of them. +### License +Trivy parses `conda-meta/.json` files at the [prefix] path. + +To correctly define licenses, make sure your `environment.yml`[^1] contains `prefix` field and `prefix` directory contains `package.json` files. + +!!! note + To get correct `environment.yml`[^1] file and fill `prefix` directory - use `conda env export` command. -[^1]: License detection is only supported for `.json` files -[^2]: Trivy supports both `yaml` and `yml` extensions. +[^1]: Trivy supports both `yaml` and `yml` extensions. [environment.yml]: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#sharing-an-environment [env-version-range]: https://docs.conda.io/projects/conda-build/en/latest/resources/package-spec.html#examples-of-package-specs +[prefix]: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#specifying-a-location-for-an-environment diff --git a/pkg/dependency/parser/conda/environment/parse.go b/pkg/dependency/parser/conda/environment/parse.go index be04d828b40a..3aaccbc1d34f 100644 --- a/pkg/dependency/parser/conda/environment/parse.go +++ b/pkg/dependency/parser/conda/environment/parse.go @@ -16,6 +16,7 @@ import ( type environment struct { Entries []Entry `yaml:"dependencies"` + Prefix string `yaml:"prefix"` } type Entry struct { @@ -27,6 +28,11 @@ type Dependency struct { Line int } +type Packages struct { + Packages ftypes.Packages + Prefix string +} + type Parser struct { logger *log.Logger once sync.Once @@ -39,10 +45,10 @@ func NewParser() *Parser { } } -func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) { +func (p *Parser) Parse(r xio.ReadSeekerAt) (Packages, error) { var env environment if err := yaml.NewDecoder(r).Decode(&env); err != nil { - return nil, nil, xerrors.Errorf("unable to decode conda environment.yml file: %w", err) + return Packages{}, xerrors.Errorf("unable to decode conda environment.yml file: %w", err) } var pkgs ftypes.Packages @@ -58,7 +64,10 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc } sort.Sort(pkgs) - return pkgs, nil, nil + return Packages{ + Packages: pkgs, + Prefix: env.Prefix, + }, nil } func (p *Parser) toPackage(dep Dependency) ftypes.Package { diff --git a/pkg/dependency/parser/conda/environment/parse_test.go b/pkg/dependency/parser/conda/environment/parse_test.go index 6283e8e135d6..9323f65c6ee4 100644 --- a/pkg/dependency/parser/conda/environment/parse_test.go +++ b/pkg/dependency/parser/conda/environment/parse_test.go @@ -15,175 +15,178 @@ func TestParse(t *testing.T) { tests := []struct { name string input string - want []ftypes.Package + want environment.Packages wantErr string }{ { name: "happy path", input: "testdata/happy.yaml", - want: []ftypes.Package{ - { - Name: "_openmp_mutex", - Locations: ftypes.Locations{ - { - StartLine: 6, - EndLine: 6, - }, - }, - }, - { - Name: "asgiref", - Version: "3.8.1", - Locations: ftypes.Locations{ - { - StartLine: 21, - EndLine: 21, - }, - }, - }, - { - Name: "blas", - Version: "1.0", - Locations: ftypes.Locations{ - { - StartLine: 5, - EndLine: 5, - }, - }, - }, - { - Name: "bzip2", - Version: "1.0.8", - Locations: ftypes.Locations{ - { - StartLine: 19, - EndLine: 19, - }, - }, - }, - { - Name: "ca-certificates", - Version: "2024.2", - Locations: ftypes.Locations{ - { - StartLine: 7, - EndLine: 7, - }, - }, - }, - { - Name: "django", - Version: "5.0.6", - Locations: ftypes.Locations{ - { - StartLine: 22, - EndLine: 22, - }, - }, - }, - { - Name: "ld_impl_linux-aarch64", - Locations: ftypes.Locations{ - { - StartLine: 8, - EndLine: 8, - }, - }, - }, - { - Name: "libblas", - Locations: ftypes.Locations{ - { - StartLine: 9, - EndLine: 9, - }, - }, - }, - { - Name: "libcblas", - Locations: ftypes.Locations{ - { - StartLine: 10, - EndLine: 10, - }, - }, - }, - { - Name: "libexpat", - Version: "2.6.2", - Locations: ftypes.Locations{ - { - StartLine: 11, - EndLine: 11, - }, - }, - }, - { - Name: "libffi", - Version: "3.4.2", - Locations: ftypes.Locations{ - { - StartLine: 12, - EndLine: 12, - }, - }, - }, - { - Name: "libgcc-ng", - Locations: ftypes.Locations{ - { - StartLine: 13, - EndLine: 13, - }, - }, - }, - { - Name: "libgfortran-ng", - Locations: ftypes.Locations{ - { - StartLine: 14, - EndLine: 14, - }, - }, - }, - { - Name: "libgfortran5", - Locations: ftypes.Locations{ - { - StartLine: 15, - EndLine: 15, - }, - }, - }, - { - Name: "libgomp", - Version: "13.2.0", - Locations: ftypes.Locations{ - { - StartLine: 16, - EndLine: 16, - }, - }, - }, - { - Name: "liblapack", - Locations: ftypes.Locations{ - { - StartLine: 17, - EndLine: 17, - }, - }, - }, - { - Name: "libnsl", - Version: "2.0.1", - Locations: ftypes.Locations{ - { - StartLine: 18, - EndLine: 18, - }, - }, - }, + want: environment.Packages{ + Packages: []ftypes.Package{ + { + Name: "_openmp_mutex", + Locations: ftypes.Locations{ + { + StartLine: 6, + EndLine: 6, + }, + }, + }, + { + Name: "asgiref", + Version: "3.8.1", + Locations: ftypes.Locations{ + { + StartLine: 21, + EndLine: 21, + }, + }, + }, + { + Name: "blas", + Version: "1.0", + Locations: ftypes.Locations{ + { + StartLine: 5, + EndLine: 5, + }, + }, + }, + { + Name: "bzip2", + Version: "1.0.8", + Locations: ftypes.Locations{ + { + StartLine: 19, + EndLine: 19, + }, + }, + }, + { + Name: "ca-certificates", + Version: "2024.2", + Locations: ftypes.Locations{ + { + StartLine: 7, + EndLine: 7, + }, + }, + }, + { + Name: "django", + Version: "5.0.6", + Locations: ftypes.Locations{ + { + StartLine: 22, + EndLine: 22, + }, + }, + }, + { + Name: "ld_impl_linux-aarch64", + Locations: ftypes.Locations{ + { + StartLine: 8, + EndLine: 8, + }, + }, + }, + { + Name: "libblas", + Locations: ftypes.Locations{ + { + StartLine: 9, + EndLine: 9, + }, + }, + }, + { + Name: "libcblas", + Locations: ftypes.Locations{ + { + StartLine: 10, + EndLine: 10, + }, + }, + }, + { + Name: "libexpat", + Version: "2.6.2", + Locations: ftypes.Locations{ + { + StartLine: 11, + EndLine: 11, + }, + }, + }, + { + Name: "libffi", + Version: "3.4.2", + Locations: ftypes.Locations{ + { + StartLine: 12, + EndLine: 12, + }, + }, + }, + { + Name: "libgcc-ng", + Locations: ftypes.Locations{ + { + StartLine: 13, + EndLine: 13, + }, + }, + }, + { + Name: "libgfortran-ng", + Locations: ftypes.Locations{ + { + StartLine: 14, + EndLine: 14, + }, + }, + }, + { + Name: "libgfortran5", + Locations: ftypes.Locations{ + { + StartLine: 15, + EndLine: 15, + }, + }, + }, + { + Name: "libgomp", + Version: "13.2.0", + Locations: ftypes.Locations{ + { + StartLine: 16, + EndLine: 16, + }, + }, + }, + { + Name: "liblapack", + Locations: ftypes.Locations{ + { + StartLine: 17, + EndLine: 17, + }, + }, + }, + { + Name: "libnsl", + Version: "2.0.1", + Locations: ftypes.Locations{ + { + StartLine: 18, + EndLine: 18, + }, + }, + }, + }, + Prefix: "/opt/conda/envs/test-env", }, }, { @@ -213,7 +216,7 @@ func TestParse(t *testing.T) { require.NoError(t, err) defer f.Close() - got, _, err := environment.NewParser().Parse(f) + got, err := environment.NewParser().Parse(f) if tt.wantErr != "" { assert.ErrorContains(t, err, tt.wantErr) diff --git a/pkg/fanal/analyzer/language/conda/environment/environment.go b/pkg/fanal/analyzer/language/conda/environment/environment.go index ee4dfbd7de88..305ac4821855 100644 --- a/pkg/fanal/analyzer/language/conda/environment/environment.go +++ b/pkg/fanal/analyzer/language/conda/environment/environment.go @@ -2,32 +2,110 @@ package environment import ( "context" + "fmt" "os" "path/filepath" + "sync" + "github.com/bmatcuk/doublestar/v4" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/dependency/parser/conda/environment" + "github.com/aquasecurity/trivy/pkg/dependency/parser/conda/meta" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/version/doc" + xio "github.com/aquasecurity/trivy/pkg/x/io" ) func init() { analyzer.RegisterAnalyzer(&environmentAnalyzer{}) } -const version = 1 +const version = 2 + +type parser struct{} + +func (*parser) Parse(r xio.ReadSeekerAt) ([]types.Package, []types.Dependency, error) { + p := environment.NewParser() + pkgs, err := p.Parse(r) + if err != nil { + return nil, nil, err + } + + once := sync.Once{} + for i, pkg := range pkgs.Packages { + // Skip packages without a version, because in this case we will not be able to get the correct file name. + if pkg.Version != "" { + licenses, err := findLicenseFromEnvDir(pkg, pkgs.Prefix) + if err != nil { + // Show log once per file + once.Do(func() { + // e.g. https://aquasecurity.github.io/trivy/latest/docs/coverage/os/conda/#license_1 + log.WithPrefix("conda").Debug(fmt.Sprintf("License not found. See %s for details.", doc.URL("docs/coverage/os/conda/", "license_1")), + log.String("pkg", pkg.Name), log.Err(err)) + }) + } + pkg.Licenses = licenses + } + pkgs.Packages[i] = pkg + } + + return pkgs.Packages, nil, nil +} type environmentAnalyzer struct{} func (a environmentAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { - res, err := language.Analyze(types.CondaEnv, input.FilePath, input.Content, environment.NewParser()) + res, err := language.Analyze(types.CondaEnv, input.FilePath, input.Content, &parser{}) if err != nil { return nil, xerrors.Errorf("unable to parse environment.yaml: %w", err) } + + if res == nil { + return nil, nil + } return res, nil } + +func findLicenseFromEnvDir(pkg types.Package, prefix string) ([]string, error) { + if prefix == "" { + return nil, xerrors.Errorf("`prefix` field doesn't exist") + } + condaMetaDir := filepath.Join(prefix, "conda-meta") + entries, err := os.ReadDir(condaMetaDir) + if err != nil { + return nil, xerrors.Errorf("unable to read conda-meta dir: %w", err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + + pattern := fmt.Sprintf("%s-%s-*.json", pkg.Name, pkg.Version) + matched, err := doublestar.Match(pattern, entry.Name()) + if err != nil { + return nil, xerrors.Errorf("incorrect packageJSON file pattern: %w", err) + } + if matched { + file, err := os.Open(filepath.Join(condaMetaDir, entry.Name())) + if err != nil { + return nil, xerrors.Errorf("unable to open packageJSON file: %w", err) + } + packageJson, _, err := meta.NewParser().Parse(file) + if err != nil { + return nil, xerrors.Errorf("unable to parse packageJSON file: %w", err) + } + // packageJson always contain only 1 element + // cf. https://github.com/aquasecurity/trivy/blob/c3192f061d7e84eaf38df8df7c879dc00b4ca137/pkg/dependency/parser/conda/meta/parse.go#L39-L45 + return packageJson[0].Licenses, nil + } + } + return nil, xerrors.Errorf("meta file didn't find") +} + func (a environmentAnalyzer) Required(filePath string, _ os.FileInfo) bool { return filepath.Base(filePath) == types.CondaEnvYml || filepath.Base(filePath) == types.CondaEnvYaml } diff --git a/pkg/fanal/analyzer/language/conda/environment/environment_test.go b/pkg/fanal/analyzer/language/conda/environment/environment_test.go index 044585f14683..02a188a87f2a 100644 --- a/pkg/fanal/analyzer/language/conda/environment/environment_test.go +++ b/pkg/fanal/analyzer/language/conda/environment/environment_test.go @@ -72,6 +72,69 @@ func Test_environmentAnalyzer_Analyze(t *testing.T) { }, }, }, + { + name: "happy path with licenses", + inputFile: "testdata/environment-with-licenses.yaml", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.CondaEnv, + FilePath: "testdata/environment-with-licenses.yaml", + Packages: types.Packages{ + { + Name: "_libgcc_mutex", + Locations: []types.Location{ + { + StartLine: 5, + EndLine: 5, + }, + }, + }, + { + Name: "_openmp_mutex", + Version: "5.1", + Locations: []types.Location{ + { + StartLine: 6, + EndLine: 6, + }, + }, + Licenses: []string{ + "BSD-3-Clause", + }, + }, + { + Name: "blas", + Version: "1.0", + Locations: []types.Location{ + { + StartLine: 7, + EndLine: 7, + }, + }, + }, + { + Name: "bzip2", + Version: "1.0.8", + Locations: []types.Location{ + { + StartLine: 8, + EndLine: 8, + }, + }, + Licenses: []string{ + "bzip2-1.0.8", + }, + }, + }, + }, + }, + }, + }, + { + name: "empty", + inputFile: "testdata/empty.yaml", + }, { name: "invalid", inputFile: "testdata/invalid.yaml", diff --git a/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/_libgcc_mutex-0.1-main.json b/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/_libgcc_mutex-0.1-main.json new file mode 100644 index 000000000000..bfc2daa841d5 --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/_libgcc_mutex-0.1-main.json @@ -0,0 +1,5 @@ +{ + "license": "MIT", + "name": "_libgcc_mutex", + "version": "0.1" +} \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/_openmp_mutex-5.1-1_gnu.json b/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/_openmp_mutex-5.1-1_gnu.json new file mode 100644 index 000000000000..a61f193ee669 --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/_openmp_mutex-5.1-1_gnu.json @@ -0,0 +1,5 @@ +{ + "license": "BSD-3-Clause", + "name": "_openmp_mutex", + "version": "5.1" +} \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/blas-1.0-openblas.json b/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/blas-1.0-openblas.json new file mode 100644 index 000000000000..9a14025cd5b2 --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/blas-1.0-openblas.json @@ -0,0 +1,5 @@ +{ + "license": "", + "name": "blas", + "version": "1.0" +} \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/bzip2-1.0.8-h5eee18b_6.json b/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/bzip2-1.0.8-h5eee18b_6.json new file mode 100644 index 000000000000..413243843b73 --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/testdata/conda-meta/bzip2-1.0.8-h5eee18b_6.json @@ -0,0 +1,5 @@ +{ + "license": "bzip2-1.0.8", + "name": "bzip2", + "version": "1.0.8" +} \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/conda/environment/testdata/empty.yaml b/pkg/fanal/analyzer/language/conda/environment/testdata/empty.yaml new file mode 100644 index 000000000000..f9622b8a1030 --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/testdata/empty.yaml @@ -0,0 +1,5 @@ +name: test-env +channels: + - defaults +dependencies: +prefix: /opt/conda/envs/test-env diff --git a/pkg/fanal/analyzer/language/conda/environment/testdata/environment-with-licenses.yaml b/pkg/fanal/analyzer/language/conda/environment/testdata/environment-with-licenses.yaml new file mode 100644 index 000000000000..a6db3a8a9e7c --- /dev/null +++ b/pkg/fanal/analyzer/language/conda/environment/testdata/environment-with-licenses.yaml @@ -0,0 +1,9 @@ +name: test-env +channels: + - defaults +dependencies: + - _libgcc_mutex + - _openmp_mutex=5.1 + - blas=1.0=openblas + - bzip2=1.0.8=h998d150_5 +prefix: testdata