From 31255ea9b34210ec06ea8909269278c31572257e Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 28 Mar 2024 17:40:01 +0600 Subject: [PATCH 1/6] refactor(pom): update logic for snapshots: - divide remote repositories into releases and snapshots - detect snapshots from the artifact version --- pkg/dependency/parser/java/pom/parse.go | 72 ++++++++++++++------ pkg/dependency/parser/java/pom/parse_test.go | 2 +- pkg/dependency/parser/java/pom/pom.go | 20 ++++-- 3 files changed, 65 insertions(+), 29 deletions(-) diff --git a/pkg/dependency/parser/java/pom/parse.go b/pkg/dependency/parser/java/pom/parse.go index 955f8cfd9e33..2cd18c8e5c27 100644 --- a/pkg/dependency/parser/java/pom/parse.go +++ b/pkg/dependency/parser/java/pom/parse.go @@ -31,8 +31,9 @@ const ( ) type options struct { - offline bool - remoteRepos []string + offline bool + releaseRemoteRepos []string + snapshotRemoteRepos []string } type option func(*options) @@ -43,25 +44,37 @@ func WithOffline(offline bool) option { } } -func WithRemoteRepos(repos []string) option { +func WithReleaseRemoteRepos(repos []string) option { return func(opts *options) { - opts.remoteRepos = repos + opts.releaseRemoteRepos = repos + } +} + +func WithSnapshotRemoteRepos(repos []string) option { + return func(opts *options) { + opts.snapshotRemoteRepos = repos } } type parser struct { - rootPath string - cache pomCache - localRepository string - remoteRepositories []string - offline bool - servers []Server + rootPath string + cache pomCache + localRepository string + releaseRemoteRepos []string + snapshotRemoteRepos []string + offline bool + servers []Server +} + +type RemoteRepo struct { + url string + snapshot bool } func NewParser(filePath string, opts ...option) types.Parser { o := &options{ - offline: false, - remoteRepos: []string{centralURL}, + offline: false, + releaseRemoteRepos: []string{centralURL}, } for _, opt := range opts { @@ -76,12 +89,13 @@ func NewParser(filePath string, opts ...option) types.Parser { } return &parser{ - rootPath: filepath.Clean(filePath), - cache: newPOMCache(), - localRepository: localRepository, - remoteRepositories: o.remoteRepos, - offline: o.offline, - servers: s.Servers, + rootPath: filepath.Clean(filePath), + cache: newPOMCache(), + localRepository: localRepository, + releaseRemoteRepos: o.releaseRemoteRepos, + snapshotRemoteRepos: o.snapshotRemoteRepos, + offline: o.offline, + servers: s.Servers, } } @@ -321,7 +335,9 @@ func (p *parser) analyze(pom *pom, opts analysisOptions) (analysisResult, error) } // Update remoteRepositories - p.remoteRepositories = utils.UniqueStrings(append(pom.repositories(p.servers), p.remoteRepositories...)) + pomReleaseRemoteRepos, pomSnapshotRemoteRepos := pom.repositories(p.servers) + p.releaseRemoteRepos = lo.Uniq(append(pomReleaseRemoteRepos, p.releaseRemoteRepos...)) + p.snapshotRemoteRepos = lo.Uniq(append(pomSnapshotRemoteRepos, p.snapshotRemoteRepos...)) // Parent parent, err := p.parseParent(pom.filePath, pom.content.Parent) @@ -612,7 +628,8 @@ func (p *parser) tryRepository(groupID, artifactID, version string) (*pom, error } // Search remote remoteRepositories - loaded, err = p.fetchPOMFromRemoteRepositories(paths) + snapshot := isSnapshot(version) + loaded, err = p.fetchPOMFromRemoteRepositories(paths, snapshot) if err == nil { return loaded, nil } @@ -627,15 +644,21 @@ func (p *parser) loadPOMFromLocalRepository(paths []string) (*pom, error) { return p.openPom(localPath) } -func (p *parser) fetchPOMFromRemoteRepositories(paths []string) (*pom, error) { +func (p *parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) (*pom, error) { // Do not try fetching pom.xml from remote repositories in offline mode if p.offline { log.Logger.Debug("Fetching the remote pom.xml is skipped") return nil, xerrors.New("offline mode") } + remoteRepos := p.releaseRemoteRepos + // TODO do we need to check releases repos for snapshots if snapshot repos don't exist? + if snapshot && p.snapshotRemoteRepos != nil { + remoteRepos = p.snapshotRemoteRepos + } + // try all remoteRepositories - for _, repo := range p.remoteRepositories { + for _, repo := range remoteRepos { fetched, err := fetchPOMFromRemoteRepository(repo, paths) if err != nil { return nil, xerrors.Errorf("fetch repository error: %w", err) @@ -699,3 +722,8 @@ func parsePom(r io.Reader) (*pomXML, error) { func packageID(name, version string) string { return dependency.ID(ftypes.Pom, name, version) } + +// cf. https://github.com/apache/maven/blob/259404701402230299fe05ee889ecdf1c9dae816/maven-artifact/src/main/java/org/apache/maven/artifact/DefaultArtifact.java#L482-L486 +func isSnapshot(ver string) bool { + return strings.HasSuffix(ver, "SNAPSHOT") || ver == "LATEST" +} diff --git a/pkg/dependency/parser/java/pom/parse_test.go b/pkg/dependency/parser/java/pom/parse_test.go index b73e40511507..2ba546894b93 100644 --- a/pkg/dependency/parser/java/pom/parse_test.go +++ b/pkg/dependency/parser/java/pom/parse_test.go @@ -1295,7 +1295,7 @@ func TestPom_Parse(t *testing.T) { remoteRepos = []string{ts.URL} } - p := pom.NewParser(tt.inputFile, pom.WithRemoteRepos(remoteRepos), pom.WithOffline(tt.offline)) + p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithOffline(tt.offline)) gotLibs, gotDeps, err := p.Parse(f) if tt.wantErr != "" { diff --git a/pkg/dependency/parser/java/pom/pom.go b/pkg/dependency/parser/java/pom/pom.go index 8b610cc5925b..39c4a2a4de68 100644 --- a/pkg/dependency/parser/java/pom/pom.go +++ b/pkg/dependency/parser/java/pom/pom.go @@ -115,11 +115,13 @@ func (p pom) licenses() []string { }) } -func (p pom) repositories(servers []Server) []string { - var urls []string +func (p pom) repositories(servers []Server) ([]string, []string) { + var releaseRepos, snapshotRepos []string for _, rep := range p.content.Repositories.Repository { + snapshot := rep.Releases.Enabled == "true" + release := rep.Releases.Enabled == "true" // Add only enabled repositories - if rep.Releases.Enabled == "false" && rep.Snapshots.Enabled == "false" { + if !release && !snapshot { continue } @@ -138,10 +140,16 @@ func (p pom) repositories(servers []Server) []string { } } - log.Logger.Debugf("Adding repository %s: %s", rep.ID, rep.URL) - urls = append(urls, repoURL.String()) + log.Logger.Debugf("Adding %s repository %s: %s", lo.Ternary(snapshot, "snapshot", "release"), rep.ID, rep.URL) + if snapshot { + snapshotRepos = append(snapshotRepos, repoURL.String()) + } + if release { + releaseRepos = append(releaseRepos, repoURL.String()) + } } - return urls + + return releaseRepos, snapshotRepos } type pomXML struct { From 6e78b40431bef9d667ed8815b4a261ed0c06faab Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 28 Mar 2024 18:32:50 +0600 Subject: [PATCH 2/6] test: add test for snapshot artifact --- pkg/dependency/parser/java/pom/parse_test.go | 33 ++++++++++++++++++- .../example-dependency-1.2.3.pom | 23 +++++++++++++ .../parser/java/pom/testdata/snapshot/pom.xml | 20 +++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom create mode 100644 pkg/dependency/parser/java/pom/testdata/snapshot/pom.xml diff --git a/pkg/dependency/parser/java/pom/parse_test.go b/pkg/dependency/parser/java/pom/parse_test.go index 2ba546894b93..3db282b84e22 100644 --- a/pkg/dependency/parser/java/pom/parse_test.go +++ b/pkg/dependency/parser/java/pom/parse_test.go @@ -70,7 +70,7 @@ func TestPom_Parse(t *testing.T) { }, }, { - name: "remote repository", + name: "remote release repository", inputFile: filepath.Join("testdata", "happy", "pom.xml"), local: false, want: []types.Library{ @@ -114,6 +114,37 @@ func TestPom_Parse(t *testing.T) { }, }, }, + { + name: "snapshot dependency", + inputFile: filepath.Join("testdata", "snapshot", "pom.xml"), + local: false, + want: []types.Library{ + { + ID: "com.example:happy:1.0.0", + Name: "com.example:happy", + Version: "1.0.0", + }, + { + ID: "org.example:example-dependency:1.2.3-SNAPSHOT", + Name: "org.example:example-dependency", + Version: "1.2.3-SNAPSHOT", + Locations: types.Locations{ + { + StartLine: 14, + EndLine: 18, + }, + }, + }, + }, + wantDeps: []types.Dependency{ + { + ID: "com.example:happy:1.0.0", + DependsOn: []string{ + "org.example:example-dependency:1.2.3-SNAPSHOT", + }, + }, + }, + }, { name: "offline mode", inputFile: filepath.Join("testdata", "offline", "pom.xml"), diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom new file mode 100644 index 000000000000..2ad90646e158 --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom @@ -0,0 +1,23 @@ + + + + 4.0.0 + + org.example + example-dependency + 1.2.3-SNAPSHOT + + jar + Example API Dependency + The example API + + + + org.example + example-api + 2.0.0 + + + + \ No newline at end of file diff --git a/pkg/dependency/parser/java/pom/testdata/snapshot/pom.xml b/pkg/dependency/parser/java/pom/testdata/snapshot/pom.xml new file mode 100644 index 000000000000..7294ecf0162f --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/snapshot/pom.xml @@ -0,0 +1,20 @@ + + 4.0.0 + + com.example + happy + 1.0.0 + + happy + Example + + + + + org.example + example-dependency + 1.2.3-SNAPSHOT + + + From 3214cffd200865f43b0eced974a0b34c6f347fd6 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 28 Mar 2024 18:46:35 +0600 Subject: [PATCH 3/6] refactor --- pkg/dependency/parser/java/pom/parse.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/pkg/dependency/parser/java/pom/parse.go b/pkg/dependency/parser/java/pom/parse.go index 2cd18c8e5c27..7f0842061249 100644 --- a/pkg/dependency/parser/java/pom/parse.go +++ b/pkg/dependency/parser/java/pom/parse.go @@ -50,12 +50,6 @@ func WithReleaseRemoteRepos(repos []string) option { } } -func WithSnapshotRemoteRepos(repos []string) option { - return func(opts *options) { - opts.snapshotRemoteRepos = repos - } -} - type parser struct { rootPath string cache pomCache @@ -66,15 +60,10 @@ type parser struct { servers []Server } -type RemoteRepo struct { - url string - snapshot bool -} - func NewParser(filePath string, opts ...option) types.Parser { o := &options{ offline: false, - releaseRemoteRepos: []string{centralURL}, + releaseRemoteRepos: []string{centralURL}, // Maven doesn't use central repository for snapshot dependencies } for _, opt := range opts { @@ -652,8 +641,8 @@ func (p *parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) ( } remoteRepos := p.releaseRemoteRepos - // TODO do we need to check releases repos for snapshots if snapshot repos don't exist? - if snapshot && p.snapshotRemoteRepos != nil { + // Maven uses only snapshot repos for snapshot artifacts + if snapshot { remoteRepos = p.snapshotRemoteRepos } From d2bd515b7a5881d06e13d6930276b09f2e77dec1 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 28 Mar 2024 18:46:40 +0600 Subject: [PATCH 4/6] fix typo --- pkg/dependency/parser/java/pom/pom.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/dependency/parser/java/pom/pom.go b/pkg/dependency/parser/java/pom/pom.go index 39c4a2a4de68..0517a8191092 100644 --- a/pkg/dependency/parser/java/pom/pom.go +++ b/pkg/dependency/parser/java/pom/pom.go @@ -118,7 +118,7 @@ func (p pom) licenses() []string { func (p pom) repositories(servers []Server) ([]string, []string) { var releaseRepos, snapshotRepos []string for _, rep := range p.content.Repositories.Repository { - snapshot := rep.Releases.Enabled == "true" + snapshot := rep.Snapshots.Enabled == "true" release := rep.Releases.Enabled == "true" // Add only enabled repositories if !release && !snapshot { @@ -140,7 +140,7 @@ func (p pom) repositories(servers []Server) ([]string, []string) { } } - log.Logger.Debugf("Adding %s repository %s: %s", lo.Ternary(snapshot, "snapshot", "release"), rep.ID, rep.URL) + log.Logger.Debugf("Adding repository %s: %s", rep.ID, rep.URL) if snapshot { snapshotRepos = append(snapshotRepos, repoURL.String()) } From cbae196944c027cf13ac116f5e760140611d7bb5 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 28 Mar 2024 18:56:24 +0600 Subject: [PATCH 5/6] docs: update java coverage page --- docs/docs/coverage/language/java.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/docs/coverage/language/java.md b/docs/docs/coverage/language/java.md index e2e97b46c61f..87db939ea288 100644 --- a/docs/docs/coverage/language/java.md +++ b/docs/docs/coverage/language/java.md @@ -42,7 +42,19 @@ Trivy parses your `pom.xml` file and tries to find files with dependencies from - relativePath field[^5] - local repository directory[^6]. -If your machine doesn't have the necessary files - Trivy tries to find the information about these dependencies in the [maven repository](https://repo.maven.apache.org/maven2/). +### remote repositories +If your machine doesn't have the necessary files - Trivy tries to find the information about these dependencies in the remote repositories: + +- [repositories from pom files][maven-pom-repos] +- [maven central repository][maven-central] + +Trivy reproduces Maven's repository selection and priority: + +- for snapshot artifacts: + - check only snapshot repositories from pom files (if exists) +- for other artifacts: + - check release repositories from pom files (if exists) + - check [maven central][maven-central] !!! Note Trivy only takes information about packages. We don't take a list of vulnerabilities for packages from the `maven repository`. @@ -92,4 +104,6 @@ Make sure that you have cache[^8] directory to find licenses from `*.pom` depend [^8]: The supported directories are `$GRADLE_USER_HOME/caches` and `$HOME/.gradle/caches` (`%HOMEPATH%\.gradle\caches` for Windows). [dependency-graph]: ../../configuration/reporting.md#show-origins-of-vulnerable-dependencies -[maven-invoker-plugin]: https://maven.apache.org/plugins/maven-invoker-plugin/usage.html \ No newline at end of file +[maven-invoker-plugin]: https://maven.apache.org/plugins/maven-invoker-plugin/usage.html +[maven-central]: https://repo.maven.apache.org/maven2/ +[maven-pom-repos]: https://maven.apache.org/settings.html#repositories \ No newline at end of file From 6c19ae700e6d7e34c7b2a97c24734ad40e9f667e Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Tue, 16 Apr 2024 12:57:41 +0600 Subject: [PATCH 6/6] refactor: fix typo --- pkg/dependency/parser/java/pom/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dependency/parser/java/pom/parse.go b/pkg/dependency/parser/java/pom/parse.go index 25ce839e4e69..a1b42b712e28 100644 --- a/pkg/dependency/parser/java/pom/parse.go +++ b/pkg/dependency/parser/java/pom/parse.go @@ -650,7 +650,7 @@ func (p *parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) ( // try all remoteRepositories for _, repo := range remoteRepos { - fetched, err := fetchPOMFromRemoteRepository(repo, paths) + fetched, err := p.fetchPOMFromRemoteRepository(repo, paths) if err != nil { return nil, xerrors.Errorf("fetch repository error: %w", err) } else if fetched == nil {