From 56434737dfb0ceb1f54bb9812947bde311c15dd3 Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Tue, 16 Apr 2024 13:48:58 +0600 Subject: [PATCH] fix(java): update logic to detect `pom.xml` file snapshot artifacts from remote repositories (#6412) --- docs/docs/coverage/language/java.md | 18 +++++- pkg/dependency/parser/java/pom/parse.go | 64 ++++++++++++------- pkg/dependency/parser/java/pom/parse_test.go | 35 +++++++++- pkg/dependency/parser/java/pom/pom.go | 18 ++++-- .../example-dependency-1.2.3.pom | 23 +++++++ .../parser/java/pom/testdata/snapshot/pom.xml | 20 ++++++ 6 files changed, 145 insertions(+), 33 deletions(-) create mode 100644 pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom create mode 100644 pkg/dependency/parser/java/pom/testdata/snapshot/pom.xml diff --git a/docs/docs/coverage/language/java.md b/docs/docs/coverage/language/java.md index e2e97b46c61f..87db939ea288 100644 --- a/docs/docs/coverage/language/java.md +++ b/docs/docs/coverage/language/java.md @@ -42,7 +42,19 @@ Trivy parses your `pom.xml` file and tries to find files with dependencies from - relativePath field[^5] - local repository directory[^6]. -If your machine doesn't have the necessary files - Trivy tries to find the information about these dependencies in the [maven repository](https://repo.maven.apache.org/maven2/). +### remote repositories +If your machine doesn't have the necessary files - Trivy tries to find the information about these dependencies in the remote repositories: + +- [repositories from pom files][maven-pom-repos] +- [maven central repository][maven-central] + +Trivy reproduces Maven's repository selection and priority: + +- for snapshot artifacts: + - check only snapshot repositories from pom files (if exists) +- for other artifacts: + - check release repositories from pom files (if exists) + - check [maven central][maven-central] !!! Note Trivy only takes information about packages. We don't take a list of vulnerabilities for packages from the `maven repository`. @@ -92,4 +104,6 @@ Make sure that you have cache[^8] directory to find licenses from `*.pom` depend [^8]: The supported directories are `$GRADLE_USER_HOME/caches` and `$HOME/.gradle/caches` (`%HOMEPATH%\.gradle\caches` for Windows). [dependency-graph]: ../../configuration/reporting.md#show-origins-of-vulnerable-dependencies -[maven-invoker-plugin]: https://maven.apache.org/plugins/maven-invoker-plugin/usage.html \ No newline at end of file +[maven-invoker-plugin]: https://maven.apache.org/plugins/maven-invoker-plugin/usage.html +[maven-central]: https://repo.maven.apache.org/maven2/ +[maven-pom-repos]: https://maven.apache.org/settings.html#repositories \ No newline at end of file diff --git a/pkg/dependency/parser/java/pom/parse.go b/pkg/dependency/parser/java/pom/parse.go index 18a62bffbf94..a1b42b712e28 100644 --- a/pkg/dependency/parser/java/pom/parse.go +++ b/pkg/dependency/parser/java/pom/parse.go @@ -30,8 +30,9 @@ const ( ) type options struct { - offline bool - remoteRepos []string + offline bool + releaseRemoteRepos []string + snapshotRemoteRepos []string } type option func(*options) @@ -42,26 +43,27 @@ func WithOffline(offline bool) option { } } -func WithRemoteRepos(repos []string) option { +func WithReleaseRemoteRepos(repos []string) option { return func(opts *options) { - opts.remoteRepos = repos + opts.releaseRemoteRepos = repos } } type parser struct { - logger *log.Logger - rootPath string - cache pomCache - localRepository string - remoteRepositories []string - offline bool - servers []Server + logger *log.Logger + rootPath string + cache pomCache + localRepository string + releaseRemoteRepos []string + snapshotRemoteRepos []string + offline bool + servers []Server } func NewParser(filePath string, opts ...option) types.Parser { o := &options{ - offline: false, - remoteRepos: []string{centralURL}, + offline: false, + releaseRemoteRepos: []string{centralURL}, // Maven doesn't use central repository for snapshot dependencies } for _, opt := range opts { @@ -76,13 +78,14 @@ func NewParser(filePath string, opts ...option) types.Parser { } return &parser{ - logger: log.WithPrefix("pom"), - rootPath: filepath.Clean(filePath), - cache: newPOMCache(), - localRepository: localRepository, - remoteRepositories: o.remoteRepos, - offline: o.offline, - servers: s.Servers, + logger: log.WithPrefix("pom"), + rootPath: filepath.Clean(filePath), + cache: newPOMCache(), + localRepository: localRepository, + releaseRemoteRepos: o.releaseRemoteRepos, + snapshotRemoteRepos: o.snapshotRemoteRepos, + offline: o.offline, + servers: s.Servers, } } @@ -324,7 +327,9 @@ func (p *parser) analyze(pom *pom, opts analysisOptions) (analysisResult, error) } // Update remoteRepositories - p.remoteRepositories = utils.UniqueStrings(append(pom.repositories(p.servers), p.remoteRepositories...)) + pomReleaseRemoteRepos, pomSnapshotRemoteRepos := pom.repositories(p.servers) + p.releaseRemoteRepos = lo.Uniq(append(pomReleaseRemoteRepos, p.releaseRemoteRepos...)) + p.snapshotRemoteRepos = lo.Uniq(append(pomSnapshotRemoteRepos, p.snapshotRemoteRepos...)) // Parent parent, err := p.parseParent(pom.filePath, pom.content.Parent) @@ -615,7 +620,7 @@ func (p *parser) tryRepository(groupID, artifactID, version string) (*pom, error } // Search remote remoteRepositories - loaded, err = p.fetchPOMFromRemoteRepositories(paths) + loaded, err = p.fetchPOMFromRemoteRepositories(paths, isSnapshot(version)) if err == nil { return loaded, nil } @@ -630,15 +635,21 @@ func (p *parser) loadPOMFromLocalRepository(paths []string) (*pom, error) { return p.openPom(localPath) } -func (p *parser) fetchPOMFromRemoteRepositories(paths []string) (*pom, error) { +func (p *parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) (*pom, error) { // Do not try fetching pom.xml from remote repositories in offline mode if p.offline { p.logger.Debug("Fetching the remote pom.xml is skipped") return nil, xerrors.New("offline mode") } + remoteRepos := p.releaseRemoteRepos + // Maven uses only snapshot repos for snapshot artifacts + if snapshot { + remoteRepos = p.snapshotRemoteRepos + } + // try all remoteRepositories - for _, repo := range p.remoteRepositories { + for _, repo := range remoteRepos { fetched, err := p.fetchPOMFromRemoteRepository(repo, paths) if err != nil { return nil, xerrors.Errorf("fetch repository error: %w", err) @@ -703,3 +714,8 @@ func parsePom(r io.Reader) (*pomXML, error) { func packageID(name, version string) string { return dependency.ID(ftypes.Pom, name, version) } + +// cf. https://github.com/apache/maven/blob/259404701402230299fe05ee889ecdf1c9dae816/maven-artifact/src/main/java/org/apache/maven/artifact/DefaultArtifact.java#L482-L486 +func isSnapshot(ver string) bool { + return strings.HasSuffix(ver, "SNAPSHOT") || ver == "LATEST" +} diff --git a/pkg/dependency/parser/java/pom/parse_test.go b/pkg/dependency/parser/java/pom/parse_test.go index b73e40511507..3db282b84e22 100644 --- a/pkg/dependency/parser/java/pom/parse_test.go +++ b/pkg/dependency/parser/java/pom/parse_test.go @@ -70,7 +70,7 @@ func TestPom_Parse(t *testing.T) { }, }, { - name: "remote repository", + name: "remote release repository", inputFile: filepath.Join("testdata", "happy", "pom.xml"), local: false, want: []types.Library{ @@ -114,6 +114,37 @@ func TestPom_Parse(t *testing.T) { }, }, }, + { + name: "snapshot dependency", + inputFile: filepath.Join("testdata", "snapshot", "pom.xml"), + local: false, + want: []types.Library{ + { + ID: "com.example:happy:1.0.0", + Name: "com.example:happy", + Version: "1.0.0", + }, + { + ID: "org.example:example-dependency:1.2.3-SNAPSHOT", + Name: "org.example:example-dependency", + Version: "1.2.3-SNAPSHOT", + Locations: types.Locations{ + { + StartLine: 14, + EndLine: 18, + }, + }, + }, + }, + wantDeps: []types.Dependency{ + { + ID: "com.example:happy:1.0.0", + DependsOn: []string{ + "org.example:example-dependency:1.2.3-SNAPSHOT", + }, + }, + }, + }, { name: "offline mode", inputFile: filepath.Join("testdata", "offline", "pom.xml"), @@ -1295,7 +1326,7 @@ func TestPom_Parse(t *testing.T) { remoteRepos = []string{ts.URL} } - p := pom.NewParser(tt.inputFile, pom.WithRemoteRepos(remoteRepos), pom.WithOffline(tt.offline)) + p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithOffline(tt.offline)) gotLibs, gotDeps, err := p.Parse(f) if tt.wantErr != "" { diff --git a/pkg/dependency/parser/java/pom/pom.go b/pkg/dependency/parser/java/pom/pom.go index 89adabe0ff3e..a994301ea75f 100644 --- a/pkg/dependency/parser/java/pom/pom.go +++ b/pkg/dependency/parser/java/pom/pom.go @@ -115,12 +115,14 @@ func (p pom) licenses() []string { }) } -func (p pom) repositories(servers []Server) []string { +func (p pom) repositories(servers []Server) ([]string, []string) { logger := log.WithPrefix("pom") - var urls []string + var releaseRepos, snapshotRepos []string for _, rep := range p.content.Repositories.Repository { + snapshot := rep.Snapshots.Enabled == "true" + release := rep.Releases.Enabled == "true" // Add only enabled repositories - if rep.Releases.Enabled == "false" && rep.Snapshots.Enabled == "false" { + if !release && !snapshot { continue } @@ -140,9 +142,15 @@ func (p pom) repositories(servers []Server) []string { } logger.Debug("Adding repository", log.String("id", rep.ID), log.String("url", rep.URL)) - urls = append(urls, repoURL.String()) + if snapshot { + snapshotRepos = append(snapshotRepos, repoURL.String()) + } + if release { + releaseRepos = append(releaseRepos, repoURL.String()) + } } - return urls + + return releaseRepos, snapshotRepos } type pomXML struct { diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom new file mode 100644 index 000000000000..2ad90646e158 --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom @@ -0,0 +1,23 @@ + + + + 4.0.0 + + org.example + example-dependency + 1.2.3-SNAPSHOT + + jar + Example API Dependency + The example API + + + + org.example + example-api + 2.0.0 + + + + \ No newline at end of file diff --git a/pkg/dependency/parser/java/pom/testdata/snapshot/pom.xml b/pkg/dependency/parser/java/pom/testdata/snapshot/pom.xml new file mode 100644 index 000000000000..7294ecf0162f --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/snapshot/pom.xml @@ -0,0 +1,20 @@ + + 4.0.0 + + com.example + happy + 1.0.0 + + happy + Example + + + + + org.example + example-dependency + 1.2.3-SNAPSHOT + + +