Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(nodejs): add license parser to pnpm analyser #7036

Merged
merged 19 commits into from
Jul 3, 2024
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/docs/coverage/language/nodejs.md
DmitriyLewen marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ The following scanners are supported.
|----------|:----:|:-------------:|:-------:|
| npm | ✓ | ✓ | ✓ |
| Yarn | ✓ | ✓ | ✓ |
| pnpm | ✓ | ✓ | - |
| pnpm | ✓ | ✓ | |
| Bun | ✓ | ✓ | ✓ |

The following table provides an outline of the features Trivy offers.
Expand Down Expand Up @@ -54,6 +54,7 @@ By default, Trivy doesn't report development dependencies. Use the `--include-de

### pnpm
Trivy parses `pnpm-lock.yaml`, then finds production dependencies and builds a [tree][dependency-graph] of dependencies with vulnerabilities.
To identify licenses, you need to download dependencies to `node_modules` beforehand. Trivy analyzes `node_modules` for licenses.

#### lock file v9 version
Trivy supports `Dev` field for `pnpm-lock.yaml` v9 or later. Use the `--include-dev-deps` flag to include the developer's dependencies in the result.
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 12 additions & 8 deletions integration/testdata/fixtures/repo/pnpm/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

52 changes: 52 additions & 0 deletions integration/testdata/pnpm-licenses.json.golden
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You don't need to create a new gold file.

You need to update testdata/pnpm.json.golden file.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if I'm doing this right; the testdata/pnpm.json.golden file is not updating. 🤔
I'm running mage test:updateGolden. @DmitriyLewen Can you help me figure out what might be wrong?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, i missed that list-all-pkgs flag is not used for pnpm testcase.
I added flag and updated golden file in 602b4d9

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you!

Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
{
"SchemaVersion": 2,
"CreatedAt": "2021-08-25T12:20:30.000000005Z",
"ArtifactName": "testdata/fixtures/repo/pnpm",
"ArtifactType": "repository",
"Metadata": {
"ImageConfig": {
"architecture": "",
"created": "0001-01-01T00:00:00Z",
"os": "",
"rootfs": {
"type": "",
"diff_ids": null
},
"config": {}
}
},
"Results": [
{
"Target": "OS Packages",
"Class": "license"
},
{
"Target": "pnpm-lock.yaml",
"Class": "license",
"Licenses": [
{
"Severity": "LOW",
"Category": "notice",
"PkgName": "jquery",
"FilePath": "pnpm-lock.yaml",
"Name": "MIT",
"Confidence": 1,
"Link": ""
},
{
"Severity": "LOW",
"Category": "notice",
"PkgName": "lodash",
"FilePath": "pnpm-lock.yaml",
"Name": "MIT",
"Confidence": 1,
"Link": ""
}
]
},
{
"Target": "Loose File License(s)",
"Class": "license-file"
}
]
}
122 changes: 109 additions & 13 deletions pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,45 +2,141 @@ package pnpm

import (
"context"
"errors"
"io"
"io/fs"
"os"
"path"
"path/filepath"

"golang.org/x/xerrors"

"github.com/aquasecurity/trivy/pkg/dependency/parser/nodejs/packagejson"
"github.com/aquasecurity/trivy/pkg/dependency/parser/nodejs/pnpm"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer/language"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/fanal/utils"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/utils/fsutils"
xpath "github.com/aquasecurity/trivy/pkg/x/path"
)

func init() {
analyzer.RegisterAnalyzer(&pnpmLibraryAnalyzer{})
analyzer.RegisterPostAnalyzer(analyzer.TypePnpm, newPnpmAnalyzer)
}

const version = 1
const version = 2

var requiredFiles = []string{types.PnpmLock}
type pnpmAnalyzer struct {
logger *log.Logger
packageJsonParser *packagejson.Parser
lockParser language.Parser
}

func newPnpmAnalyzer(opt analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
return &pnpmAnalyzer{
logger: log.WithPrefix("pnpm"),
packageJsonParser: packagejson.NewParser(),
lockParser: pnpm.NewParser(),
}, nil
}

func (a pnpmAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) {
var apps []types.Application

required := func(path string, d fs.DirEntry) bool {
return filepath.Base(path) == types.PnpmLock
}

type pnpmLibraryAnalyzer struct{}
err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {
// Find licenses
licenses, err := a.findLicenses(input.FS, filePath)
if err != nil {
a.logger.Error("Unable to collect licenses", log.Err(err))
licenses = make(map[string][]string)
}

func (a pnpmLibraryAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) {
res, err := language.Analyze(types.Pnpm, input.FilePath, input.Content, pnpm.NewParser())
// Parse pnpm-lock.yaml
app, err := language.Parse(types.Pnpm, filePath, r, a.lockParser)
if err != nil {
return xerrors.Errorf("parse error: %w", err)
} else if app == nil {
return nil
}

// Fill licenses
for i, lib := range app.Packages {
if l, ok := licenses[lib.ID]; ok {
app.Packages[i].Licenses = l
}
}

apps = append(apps, *app)

return nil
})
if err != nil {
return nil, xerrors.Errorf("unable to parse %s: %w", input.FilePath, err)
return nil, xerrors.Errorf("pnpm walk error: %w", err)
}
return res, nil

return &analyzer.AnalysisResult{
Applications: apps,
}, nil
}

func (a pnpmLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool {
func (a pnpmAnalyzer) Required(filePath string, _ os.FileInfo) bool {
fileName := filepath.Base(filePath)
return utils.StringInSlice(fileName, requiredFiles)
// Don't save pnpm-lock.yaml from the `node_modules` directory to avoid duplication and mistakes.
if fileName == types.PnpmLock && !xpath.Contains(filePath, "node_modules") {
return true
}

// Save package.json files only from the `node_modules` directory.
// Required to search for licenses.
if fileName == types.NpmPkg && xpath.Contains(filePath, "node_modules") {
return true
}

return false
}

func (a pnpmLibraryAnalyzer) Type() analyzer.Type {
func (a pnpmAnalyzer) Type() analyzer.Type {
return analyzer.TypePnpm
}

func (a pnpmLibraryAnalyzer) Version() int {
func (a pnpmAnalyzer) Version() int {
return version
}

func (a pnpmAnalyzer) findLicenses(fsys fs.FS, lockPath string) (map[string][]string, error) {
dir := path.Dir(lockPath)
root := path.Join(dir, "node_modules")
if _, err := fs.Stat(fsys, root); errors.Is(err, fs.ErrNotExist) {
a.logger.Info(`To collect the license information of packages, "pnpm install" needs to be performed beforehand`,
log.String("dir", root))
return nil, nil
}

// Parse package.json
required := func(path string, _ fs.DirEntry) bool {
return filepath.Base(path) == types.NpmPkg
}

// Traverse node_modules dir and find licenses
// Note that fs.FS is always slashed regardless of the platform,
// and path.Join should be used rather than filepath.Join.
licenses := make(map[string][]string)
err := fsutils.WalkDir(fsys, root, required, func(filePath string, d fs.DirEntry, r io.Reader) error {
pkg, err := a.packageJsonParser.Parse(r)
if err != nil {
return xerrors.Errorf("unable to parse %q: %w", filePath, err)
}

licenses[pkg.ID] = pkg.Licenses
return nil
})
if err != nil {
return nil, xerrors.Errorf("walk error: %w", err)
}
return licenses, nil
}
Loading
Loading