diff --git a/internal/manifest/fixtures/maven/empty.xml b/internal/manifest/fixtures/maven/empty.xml new file mode 100644 index 00000000000..8cfeebaaa4d --- /dev/null +++ b/internal/manifest/fixtures/maven/empty.xml @@ -0,0 +1,7 @@ + + 4.0.0 + + com.mycompany.app + my-app + 1 + diff --git a/internal/manifest/fixtures/maven/interpolation.xml b/internal/manifest/fixtures/maven/interpolation.xml new file mode 100644 index 00000000000..eb783c8e710 --- /dev/null +++ b/internal/manifest/fixtures/maven/interpolation.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + io.library + my-library + 1.0-SNAPSHOT + jar + + + 1.0.0 + 2.3.4 + [9.4.35.v20201120,9.5) + + + + + org.mine + mypackage + ${mypackageVersion} + + + + org.mine + my.package + ${my.package.version} + + + + + + + org.mine + ranged-package + ${version-range} + + + + diff --git a/internal/manifest/fixtures/maven/invalid-syntax.xml b/internal/manifest/fixtures/maven/invalid-syntax.xml new file mode 100644 index 00000000000..761a32c1abb --- /dev/null +++ b/internal/manifest/fixtures/maven/invalid-syntax.xml @@ -0,0 +1,13 @@ + + + <${Id}.version>${project.version} + + + + + io.netty + netty-all + 4.1.42.Final + + + diff --git a/internal/manifest/fixtures/maven/not-pom.txt b/internal/manifest/fixtures/maven/not-pom.txt new file mode 100644 index 00000000000..f9df712bcb2 --- /dev/null +++ b/internal/manifest/fixtures/maven/not-pom.txt @@ -0,0 +1 @@ +this is not a pom.xml file! diff --git a/internal/manifest/fixtures/maven/one-package.xml b/internal/manifest/fixtures/maven/one-package.xml new file mode 100644 index 00000000000..4d9618b3e9c --- /dev/null +++ b/internal/manifest/fixtures/maven/one-package.xml @@ -0,0 +1,13 @@ + + + 3.0 + + + + + org.apache.maven + maven-artifact + 1.0.0 + + + diff --git a/internal/manifest/fixtures/maven/two-packages.xml b/internal/manifest/fixtures/maven/two-packages.xml new file mode 100644 index 00000000000..f5f0a98bfef --- /dev/null +++ b/internal/manifest/fixtures/maven/two-packages.xml @@ -0,0 +1,18 @@ + + + 3.0 + + + + + io.netty + netty-all + 4.1.42.Final + + + org.slf4j + slf4j-log4j12 + 1.7.25 + + + diff --git a/internal/manifest/fixtures/maven/with-dependency-management.xml b/internal/manifest/fixtures/maven/with-dependency-management.xml new file mode 100644 index 00000000000..76f419c16d5 --- /dev/null +++ b/internal/manifest/fixtures/maven/with-dependency-management.xml @@ -0,0 +1,33 @@ + + + 3.0 + + + + + io.netty + netty-all + 4.1.9 + + + org.slf4j + slf4j-log4j12 + 1.7.25 + + + + + + + io.netty + netty-all + 4.1.42.Final + + + com.google.code.findbugs + jsr305 + 3.0.2 + + + + diff --git a/internal/manifest/fixtures/maven/with-scope.xml b/internal/manifest/fixtures/maven/with-scope.xml new file mode 100644 index 00000000000..179b05a3175 --- /dev/null +++ b/internal/manifest/fixtures/maven/with-scope.xml @@ -0,0 +1,10 @@ + + + + junit + junit + 4.12 + test + + + diff --git a/internal/manifest/helpers_test.go b/internal/manifest/helpers_test.go new file mode 100644 index 00000000000..ffcab3d06ea --- /dev/null +++ b/internal/manifest/helpers_test.go @@ -0,0 +1,105 @@ +package manifest_test + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/google/osv-scanner/internal/output" + "github.com/google/osv-scanner/pkg/lockfile" +) + +func expectErrContaining(t *testing.T, err error, str string) { + t.Helper() + + if err == nil { + t.Errorf("Expected to get error, but did not") + } + + if !strings.Contains(err.Error(), str) { + t.Errorf("Expected to get \"%s\" error, but got \"%v\"", str, err) + } +} + +func expectErrIs(t *testing.T, err error, expected error) { + t.Helper() + + if err == nil { + t.Errorf("Expected to get error, but did not") + } + + if !errors.Is(err, expected) { + t.Errorf("Expected to get \"%v\" error but got \"%v\" instead", expected, err) + } +} + +func packageToString(pkg lockfile.PackageDetails) string { + commit := pkg.Commit + + if commit == "" { + commit = "" + } + + groups := strings.Join(pkg.DepGroups, ", ") + + if groups == "" { + groups = "" + } + + return fmt.Sprintf("%s@%s (%s, %s, %s)", pkg.Name, pkg.Version, pkg.Ecosystem, commit, groups) +} + +func hasPackage(t *testing.T, packages []lockfile.PackageDetails, pkg lockfile.PackageDetails) bool { + t.Helper() + + for _, details := range packages { + if reflect.DeepEqual(details, pkg) { + return true + } + } + + return false +} + +func findMissingPackages(t *testing.T, actualPackages []lockfile.PackageDetails, expectedPackages []lockfile.PackageDetails) []lockfile.PackageDetails { + t.Helper() + var missingPackages []lockfile.PackageDetails + + for _, pkg := range actualPackages { + if !hasPackage(t, expectedPackages, pkg) { + missingPackages = append(missingPackages, pkg) + } + } + + return missingPackages +} + +func expectPackages(t *testing.T, actualPackages []lockfile.PackageDetails, expectedPackages []lockfile.PackageDetails) { + t.Helper() + + if len(expectedPackages) != len(actualPackages) { + t.Errorf( + "Expected to get %d %s, but got %d", + len(expectedPackages), + output.Form(len(expectedPackages), "package", "packages"), + len(actualPackages), + ) + } + + missingActualPackages := findMissingPackages(t, actualPackages, expectedPackages) + missingExpectedPackages := findMissingPackages(t, expectedPackages, actualPackages) + + if len(missingActualPackages) != 0 { + for _, unexpectedPackage := range missingActualPackages { + t.Errorf("Did not expect %s", packageToString(unexpectedPackage)) + } + } + + if len(missingExpectedPackages) != 0 { + for _, unexpectedPackage := range missingExpectedPackages { + t.Errorf("Did not find %s", packageToString(unexpectedPackage)) + } + } +} diff --git a/internal/manifest/maven.go b/internal/manifest/maven.go new file mode 100644 index 00000000000..8c6a191e38f --- /dev/null +++ b/internal/manifest/maven.go @@ -0,0 +1,122 @@ +package manifest + +import ( + "context" + "encoding/xml" + "fmt" + "path/filepath" + + depsdevpb "deps.dev/api/v3" + "deps.dev/util/maven" + "deps.dev/util/semver" + "github.com/google/osv-scanner/pkg/lockfile" + "golang.org/x/exp/maps" +) + +type MavenResolverExtractor struct { + Client depsdevpb.InsightsClient +} + +func (e MavenResolverExtractor) ShouldExtract(path string) bool { + return filepath.Base(path) == "pom.xml" +} + +func (e MavenResolverExtractor) Extract(f lockfile.DepFile) ([]lockfile.PackageDetails, error) { + ctx := context.Background() + + var project maven.Project + if err := xml.NewDecoder(f).Decode(&project); err != nil { + return []lockfile.PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) + } + if err := project.Interpolate(); err != nil { + return []lockfile.PackageDetails{}, fmt.Errorf("could not interpolate Maven project %s: %w", project.ProjectKey.Name(), err) + } + + details := map[string]lockfile.PackageDetails{} + + for _, dep := range project.Dependencies { + name := dep.Name() + v, err := e.resolveVersion(ctx, dep) + if err != nil { + return []lockfile.PackageDetails{}, err + } + pkgDetails := lockfile.PackageDetails{ + Name: name, + Version: v, + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + } + if dep.Scope != "" { + pkgDetails.DepGroups = append(pkgDetails.DepGroups, string(dep.Scope)) + } + // A dependency may be declared more than one times, we keep the details + // from the last declared one as what `mvn` does. + details[name] = pkgDetails + } + + // managed dependencies take precedent over standard dependencies + for _, dep := range project.DependencyManagement.Dependencies { + name := dep.Name() + v, err := e.resolveVersion(ctx, dep) + if err != nil { + return []lockfile.PackageDetails{}, err + } + pkgDetails := lockfile.PackageDetails{ + Name: name, + Version: v, + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + } + if dep.Scope != "" { + pkgDetails.DepGroups = append(pkgDetails.DepGroups, string(dep.Scope)) + } + // A dependency may be declared more than one times, we keep the details + // from the last declared one as what `mvn` does. + details[name] = pkgDetails + } + + return maps.Values(details), nil +} + +func (e MavenResolverExtractor) resolveVersion(ctx context.Context, dep maven.Dependency) (string, error) { + constraint, err := semver.Maven.ParseConstraint(string(dep.Version)) + if err != nil { + return "", fmt.Errorf("failed parsing Maven constraint %s: %w", dep.Version, err) + } + if constraint.IsSimple() { + // Return the constraint if it is a simple version string. + return constraint.String(), nil + } + + // Otherwise return the greatest version matching the constraint. + // TODO: invoke Maven resolver to decide the exact version. + resp, err := e.Client.GetPackage(ctx, &depsdevpb.GetPackageRequest{ + PackageKey: &depsdevpb.PackageKey{ + System: depsdevpb.System_MAVEN, + Name: dep.Name(), + }, + }) + if err != nil { + return "", fmt.Errorf("requesting versions of Maven package %s: %w", dep.Name(), err) + } + + var result *semver.Version + for _, ver := range resp.GetVersions() { + v, _ := semver.Maven.Parse(ver.GetVersionKey().GetVersion()) + if constraint.MatchVersion(v) && result.Compare(v) < 0 { + result = v + } + } + + return result.String(), nil +} + +func ParseMavenWithResolver(depsdev depsdevpb.InsightsClient, pathToLockfile string) ([]lockfile.PackageDetails, error) { + f, err := lockfile.OpenLocalDepFile(pathToLockfile) + if err != nil { + return []lockfile.PackageDetails{}, err + } + defer f.Close() + + return MavenResolverExtractor{Client: depsdev}.Extract(f) +} diff --git a/internal/manifest/maven_test.go b/internal/manifest/maven_test.go new file mode 100644 index 00000000000..a75a34658c1 --- /dev/null +++ b/internal/manifest/maven_test.go @@ -0,0 +1,293 @@ +package manifest_test + +import ( + "context" + "errors" + "io/fs" + "testing" + + depsdevpb "deps.dev/api/v3" + "github.com/google/osv-scanner/internal/manifest" + "github.com/google/osv-scanner/pkg/lockfile" + "google.golang.org/grpc" +) + +type fakeDepsDevClient struct { + depsdevpb.InsightsClient +} + +func (c *fakeDepsDevClient) GetPackage(ctx context.Context, in *depsdevpb.GetPackageRequest, opts ...grpc.CallOption) (*depsdevpb.Package, error) { + if in.GetPackageKey().GetName() == "org.mine:ranged-package" { + return &depsdevpb.Package{ + Versions: []*depsdevpb.Package_Version{ + { + VersionKey: &depsdevpb.VersionKey{ + Version: "9.4.35", + }, + }, + { + VersionKey: &depsdevpb.VersionKey{ + Version: "9.4.36", + }, + }, + { + VersionKey: &depsdevpb.VersionKey{ + Version: "9.4.37", + }, + }, + { + VersionKey: &depsdevpb.VersionKey{ + Version: "9.5", + }, + }, + }, + }, nil + } + + return nil, errors.New("package not found") +} + +func (c *fakeDepsDevClient) GetVersion(ctx context.Context, in *depsdevpb.GetVersionRequest, opts ...grpc.CallOption) (*depsdevpb.Version, error) { + return nil, errors.New("not implemented") +} + +func (c *fakeDepsDevClient) GetRequirements(ctx context.Context, in *depsdevpb.GetRequirementsRequest, opts ...grpc.CallOption) (*depsdevpb.Requirements, error) { + return nil, errors.New("not implemented") +} + +func (c *fakeDepsDevClient) GetDependencies(ctx context.Context, in *depsdevpb.GetDependenciesRequest, opts ...grpc.CallOption) (*depsdevpb.Dependencies, error) { + return nil, errors.New("not implemented") +} + +func (c *fakeDepsDevClient) GetProject(ctx context.Context, in *depsdevpb.GetProjectRequest, opts ...grpc.CallOption) (*depsdevpb.Project, error) { + return nil, errors.New("not implemented") +} + +func (c *fakeDepsDevClient) GetProjectPackageVersions(ctx context.Context, in *depsdevpb.GetProjectPackageVersionsRequest, opts ...grpc.CallOption) (*depsdevpb.ProjectPackageVersions, error) { + return nil, errors.New("not implemented") +} + +func (c *fakeDepsDevClient) GetAdvisory(ctx context.Context, in *depsdevpb.GetAdvisoryRequest, opts ...grpc.CallOption) (*depsdevpb.Advisory, error) { + return nil, errors.New("not implemented") +} + +func (c *fakeDepsDevClient) Query(ctx context.Context, in *depsdevpb.QueryRequest, opts ...grpc.CallOption) (*depsdevpb.QueryResult, error) { + return nil, errors.New("not implemented") +} + +func TestMavenResolverExtractor_ShouldExtract(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + want bool + }{ + { + name: "", + path: "", + want: false, + }, + { + name: "", + path: "pom.xml", + want: true, + }, + { + name: "", + path: "path/to/my/pom.xml", + want: true, + }, + { + name: "", + path: "path/to/my/pom.xml/file", + want: false, + }, + { + name: "", + path: "path/to/my/pom.xml.file", + want: false, + }, + { + name: "", + path: "path.to.my.pom.xml", + want: false, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + e := manifest.MavenResolverExtractor{} + got := e.ShouldExtract(tt.path) + if got != tt.want { + t.Errorf("Extract() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseMavenWithResolver_FileDoesNotExist(t *testing.T) { + t.Parallel() + + packages, err := manifest.ParseMavenWithResolver(nil, "fixtures/maven/does-not-exist") + + expectErrIs(t, err, fs.ErrNotExist) + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseMavenWithResolver_Invalid(t *testing.T) { + t.Parallel() + + packages, err := manifest.ParseMavenWithResolver(nil, "fixtures/maven/not-pom.txt") + + expectErrContaining(t, err, "could not extract from") + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseMavenWithResolver_InvalidSyntax(t *testing.T) { + t.Parallel() + + packages, err := manifest.ParseMavenWithResolver(nil, "fixtures/maven/invalid-syntax.xml") + + expectErrContaining(t, err, "XML syntax error") + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseMavenWithResolver_NoPackages(t *testing.T) { + t.Parallel() + + packages, err := manifest.ParseMavenWithResolver(nil, "fixtures/maven/empty.xml") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseMavenWithResolver_OnePackage(t *testing.T) { + t.Parallel() + + packages, err := manifest.ParseMavenWithResolver(nil, "fixtures/maven/one-package.xml") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "org.apache.maven:maven-artifact", + Version: "1.0.0", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + }) +} + +func TestParseMavenWithResolver_TwoPackages(t *testing.T) { + t.Parallel() + + packages, err := manifest.ParseMavenWithResolver(nil, "fixtures/maven/two-packages.xml") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "io.netty:netty-all", + Version: "4.1.42.Final", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + { + Name: "org.slf4j:slf4j-log4j12", + Version: "1.7.25", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + }) +} + +func TestParseMavenWithResolver_WithDependencyManagement(t *testing.T) { + t.Parallel() + + packages, err := manifest.ParseMavenWithResolver(nil, "fixtures/maven/with-dependency-management.xml") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "io.netty:netty-all", + Version: "4.1.42.Final", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + { + Name: "org.slf4j:slf4j-log4j12", + Version: "1.7.25", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + { + Name: "com.google.code.findbugs:jsr305", + Version: "3.0.2", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + }) +} + +func TestParseMavenWithResolver_Interpolation(t *testing.T) { + t.Parallel() + + packages, err := manifest.ParseMavenWithResolver(&fakeDepsDevClient{}, "fixtures/maven/interpolation.xml") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "org.mine:mypackage", + Version: "1.0.0", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + { + Name: "org.mine:my.package", + Version: "2.3.4", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + { + Name: "org.mine:ranged-package", + Version: "9.4.37", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + }) +} + +func TestParseMavenWithResolver_WithScope(t *testing.T) { + t.Parallel() + + packages, err := manifest.ParseMavenWithResolver(nil, "fixtures/maven/with-scope.xml") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "junit:junit", + Version: "4.12", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + DepGroups: []string{"test"}, + }, + }) +}