feat(java): add support for maven-metadata.xml files for remote snapshot repositories.

pkg/dependency/parser/java/pom/metadata.go

@@ -0,0 +1,17 @@
+package pom
+
+type Metadata struct {
+	GroupId    string     `xml:"groupId"`
+	ArtifactId string     `xml:"artifactId"`
+	Versioning Versioning `xml:"versioning"`
+	Version    string     `xml:"version"`
+}
+
+type Versioning struct {
+	SnapshotVersions []SnapshotVersion `xml:"snapshotVersions>snapshotVersion"`
+}
+
+type SnapshotVersion struct {
+	Extension string `xml:"extension"`
+	Value     string `xml:"value"`
+}

pkg/dependency/parser/java/pom/parse.go

@@ -14,6 +14,7 @@ import (
 
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/samber/lo"
+	"golang.org/x/exp/slices"
 	"golang.org/x/net/html/charset"
 	"golang.org/x/xerrors"
 
@@ -48,6 +49,12 @@ func WithReleaseRemoteRepos(repos []string) option {
 	}
 }
 
+func WithSnapshotRemoteRepos(repos []string) option {
+	return func(opts *options) {
+		opts.snapshotRemoteRepos = repos
+	}
+}
+
 type Parser struct {
 	logger              *log.Logger
 	rootPath            string
@@ -648,7 +655,18 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) (
 
 	// try all remoteRepositories
 	for _, repo := range remoteRepos {
-		fetched, err := p.fetchPOMFromRemoteRepository(repo, paths)
+		repoPaths := slices.Clone(paths) // Clone slice to avoid overwriting last element of `paths`
+		if snapshot {
+			pomFileName, err := p.fetchPomFileNameFromMavenMetadata(repo, repoPaths)
+			if err != nil {
+				return nil, xerrors.Errorf("fetch maven-metadata.xml error: %w", err)
+			}
+			// Use file name from `maven-metadata.xml` if it exists
+			if pomFileName != "" {
+				repoPaths[len(repoPaths)-1] = pomFileName
+			}
+		}
+		fetched, err := p.fetchPOMFromRemoteRepository(repo, repoPaths)
 		if err != nil {
 			return nil, xerrors.Errorf("fetch repository error: %w", err)
 		} else if fetched == nil {
@@ -659,7 +677,7 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) (
 	return nil, xerrors.Errorf("the POM was not found in remote remoteRepositories")
 }
 
-func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) {
+func (p *Parser) remoteRepoRequest(repo string, paths []string) (*http.Request, error) {
 	repoURL, err := url.Parse(repo)
 	if err != nil {
 		p.logger.Error("URL parse error", log.String("repo", repo))
@@ -670,7 +688,6 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom
 	repoURL.Path = path.Join(paths...)
 
 	logger := p.logger.With(log.String("host", repoURL.Host), log.String("path", repoURL.Path))
-	client := &http.Client{}
 	req, err := http.NewRequest("GET", repoURL.String(), http.NoBody)
 	if err != nil {
 		logger.Debug("HTTP request failed")
@@ -681,9 +698,54 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom
 		req.SetBasicAuth(repoURL.User.Username(), password)
 	}
 
+	return req, nil
+}
+
+// fetchPomFileNameFromMavenMetadata fetches `maven-metadata.xml` file to detect file name of pom file.
+func (p *Parser) fetchPomFileNameFromMavenMetadata(repo string, paths []string) (string, error) {
+	// Overwrite pom file name to `maven-metadata.xml`
+	mavenMetadataPaths := slices.Clone(paths[:len(paths)-1]) // Clone slice to avoid shadow overwriting last element of `paths`
+	mavenMetadataPaths = append(mavenMetadataPaths, "maven-metadata.xml")
+
+	req, err := p.remoteRepoRequest(repo, mavenMetadataPaths)
+	if err != nil {
+		return "", xerrors.Errorf("unable to create request for maven-metadata.xml file")
+	}
+
+	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil || resp.StatusCode != http.StatusOK {
-		logger.Debug("Failed to fetch")
+		p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()))
+		return "", nil
+	}
+	defer resp.Body.Close()
+
+	mavenMetadata, err := parseMavenMetadata(resp.Body)
+	if err != nil {
+		return "", xerrors.Errorf("failed to parse maven-metadata.xml file: %w", err)
+	}
+
+	var pomFileName string
+	for _, sv := range mavenMetadata.Versioning.SnapshotVersions {
+		if sv.Extension == "pom" {
+			// mavenMetadataPaths[len(mavenMetadataPaths)-3] is always artifactID
+			pomFileName = fmt.Sprintf("%s-%s.pom", mavenMetadataPaths[len(mavenMetadataPaths)-3], sv.Value)
+		}
+	}
+
+	return pomFileName, nil
+}
+
+func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) {
+	req, err := p.remoteRepoRequest(repo, paths)
+	if err != nil {
+		return nil, xerrors.Errorf("unable to create request for pom file")
+	}
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()))
 		return nil, nil
 	}
 	defer resp.Body.Close()
@@ -709,6 +771,16 @@ func parsePom(r io.Reader) (*pomXML, error) {
 	return parsed, nil
 }
 
+func parseMavenMetadata(r io.Reader) (*Metadata, error) {
+	parsed := &Metadata{}
+	decoder := xml.NewDecoder(r)
+	decoder.CharsetReader = charset.NewReaderLabel
+	if err := decoder.Decode(parsed); err != nil {
+		return nil, xerrors.Errorf("xml decode error: %w", err)
+	}
+	return parsed, nil
+}
+
 func packageID(name, version string) string {
 	return dependency.ID(ftypes.Pom, name, version)
 }

pkg/dependency/parser/java/pom/parse_test.go

@@ -143,6 +143,13 @@ func TestPom_Parse(t *testing.T) {
 						},
 					},
 				},
+				{
+					ID:           "org.example:example-api:2.0.0",
+					Name:         "org.example:example-api",
+					Version:      "2.0.0",
+					Licenses:     []string{"The Apache Software License, Version 2.0"},
+					Relationship: ftypes.RelationshipIndirect,
+				},
 			},
 			wantDeps: []ftypes.Dependency{
 				{
@@ -151,6 +158,58 @@ func TestPom_Parse(t *testing.T) {
 						"org.example:example-dependency:1.2.3-SNAPSHOT",
 					},
 				},
+				{
+					ID: "org.example:example-dependency:1.2.3-SNAPSHOT",
+					DependsOn: []string{
+						"org.example:example-api:2.0.0",
+					},
+				},
+			},
+		},
+		{
+			name:      "snapshot repository with maven-metadata.xml",
+			inputFile: filepath.Join("testdata", "snapshot", "with-maven-metadata", "pom.xml"),
+			local:     false,
+			want: []ftypes.Package{
+				{
+					ID:           "com.example:happy:1.0.0",
+					Name:         "com.example:happy",
+					Version:      "1.0.0",
+					Relationship: ftypes.RelationshipRoot,
+				},
+				{
+					ID:           "org.example:example-dependency:2.17.0-SNAPSHOT",
+					Name:         "org.example:example-dependency",
+					Version:      "2.17.0-SNAPSHOT",
+					Relationship: ftypes.RelationshipDirect,
+					Locations: ftypes.Locations{
+						{
+							StartLine: 14,
+							EndLine:   18,
+						},
+					},
+				},
+				{
+					ID:           "org.example:example-api:2.0.0",
+					Name:         "org.example:example-api",
+					Version:      "2.0.0",
+					Licenses:     []string{"The Apache Software License, Version 2.0"},
+					Relationship: ftypes.RelationshipIndirect,
+				},
+			},
+			wantDeps: []ftypes.Dependency{
+				{
+					ID: "com.example:happy:1.0.0",
+					DependsOn: []string{
+						"org.example:example-dependency:2.17.0-SNAPSHOT",
+					},
+				},
+				{
+					ID: "org.example:example-dependency:2.17.0-SNAPSHOT",
+					DependsOn: []string{
+						"org.example:example-api:2.0.0",
+					},
+				},
 			},
 		},
 		{
@@ -1404,7 +1463,7 @@ func TestPom_Parse(t *testing.T) {
 				remoteRepos = []string{ts.URL}
 			}
 
-			p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithOffline(tt.offline))
+			p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithSnapshotRemoteRepos(remoteRepos), pom.WithOffline(tt.offline))
 
 			gotPkgs, gotDeps, err := p.Parse(f)
 			if tt.wantErr != "" {

pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.example</groupId>
+    <artifactId>example-dependency</artifactId>
+    <version>2.17.0-SNAPSHOT</version>
+
+    <packaging>jar</packaging>
+    <name>Example API Dependency</name>
+    <description>The example API</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.example</groupId>
+            <artifactId>example-api</artifactId>
+            <version>2.0.0</version>
+        </dependency>
+    </dependencies>
+
+</project>

pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml

@@ -0,0 +1,35 @@
+<metadata modelVersion="1.1.0">
+    <groupId>org.example</groupId>
+    <artifactId>example-dependency</artifactId>
+    <versioning>
+        <lastUpdated>20240312035235</lastUpdated>
+        <snapshot>
+            <timestamp>20240312.035235</timestamp>
+            <buildNumber>10</buildNumber>
+        </snapshot>
+        <snapshotVersions>
+            <snapshotVersion>
+                <classifier>sources</classifier>
+                <extension>jar</extension>
+                <value>2.17.0-20240312.035235-10</value>
+                <updated>20240312035235</updated>
+            </snapshotVersion>
+            <snapshotVersion>
+                <extension>module</extension>
+                <value>2.17.0-20240312.035235-10</value>
+                <updated>20240312035235</updated>
+            </snapshotVersion>
+            <snapshotVersion>
+                <extension>jar</extension>
+                <value>2.17.0-20240312.035235-10</value>
+                <updated>20240312035235</updated>
+            </snapshotVersion>
+            <snapshotVersion>
+                <extension>pom</extension>
+                <value>2.17.0-20240312.035235-10</value>
+                <updated>20240312035235</updated>
+            </snapshotVersion>
+        </snapshotVersions>
+    </versioning>
+    <version>2.17.0-SNAPSHOT</version>
+</metadata>

pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml

@@ -0,0 +1,20 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.example</groupId>
+    <artifactId>happy</artifactId>
+    <version>1.0.0</version>
+
+    <name>happy</name>
+    <description>Example</description>
+
+
+    <dependencies>
+        <dependency>
+            <groupId>org.example</groupId>
+            <artifactId>example-dependency</artifactId>
+            <version>2.17.0-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+</project>