From ccdc7a09a9637b6feb5e189f7816c4254688ab5b Mon Sep 17 00:00:00 2001 From: abhisek Date: Wed, 14 Dec 2022 19:46:12 +0530 Subject: [PATCH] Add gradle lockfile support Fix test case Add gradle lockfile support Fix test case Update pkg/lockfile/ecosystems_test.go Co-authored-by: Gareth Jones Update test case as per PR review comment Signed-off-by: abhisek Update README --- README.md | 2 + pkg/lockfile/ecosystems_test.go | 9 +- pkg/lockfile/fixtures/gradle/5-pkg | 10 ++ pkg/lockfile/fixtures/gradle/one-pkg | 5 + pkg/lockfile/fixtures/gradle/only-comments | 8 ++ pkg/lockfile/fixtures/gradle/only-empty | 1 + pkg/lockfile/fixtures/gradle/with-bad-pkg | 19 +++ pkg/lockfile/parse-gradle-lock.go | 70 +++++++++++ pkg/lockfile/parse-gradle-lock_test.go | 128 +++++++++++++++++++++ pkg/lockfile/parse.go | 26 +++-- pkg/lockfile/parse_test.go | 19 ++- 11 files changed, 276 insertions(+), 21 deletions(-) create mode 100644 pkg/lockfile/fixtures/gradle/5-pkg create mode 100644 pkg/lockfile/fixtures/gradle/one-pkg create mode 100644 pkg/lockfile/fixtures/gradle/only-comments create mode 100644 pkg/lockfile/fixtures/gradle/only-empty create mode 100644 pkg/lockfile/fixtures/gradle/with-bad-pkg create mode 100644 pkg/lockfile/parse-gradle-lock.go create mode 100644 pkg/lockfile/parse-gradle-lock_test.go diff --git a/README.md b/README.md index f701095710f..840d7874116 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ A wide range of lockfiles are supported by utilizing this [lockfile package](htt - `pubspec.lock` - `pom.xml`[\*](https://github.com/google/osv-scanner/issues/35) - `requirements.txt`[\*](https://github.com/google/osv-scanner/issues/34) +- `gradle.lockfile` +- `buildscript-gradle.lockfile` #### Example diff --git a/pkg/lockfile/ecosystems_test.go b/pkg/lockfile/ecosystems_test.go index 92818fe450b..7ce70f5bd74 100644 --- a/pkg/lockfile/ecosystems_test.go +++ b/pkg/lockfile/ecosystems_test.go @@ -1,10 +1,11 @@ package lockfile_test import ( - "github.com/google/osv-scanner/pkg/lockfile" "os" "strings" "testing" + + "github.com/google/osv-scanner/pkg/lockfile" ) func numberOfLockfileParsers(t *testing.T) int { @@ -33,9 +34,9 @@ func TestKnownEcosystems(t *testing.T) { expectedCount := numberOfLockfileParsers(t) - // npm, yarn, and pnpm, and pip and poetry, all use the same ecosystem - // so "ignore" those parsers in the count - expectedCount -= 3 + // npm, yarn, and pnpm, and pip and poetry, and maven and gradle, all + // use the same ecosystem so "ignore" those parsers in the count + expectedCount -= 4 ecosystems := lockfile.KnownEcosystems() diff --git a/pkg/lockfile/fixtures/gradle/5-pkg b/pkg/lockfile/fixtures/gradle/5-pkg new file mode 100644 index 00000000000..53af5d46720 --- /dev/null +++ b/pkg/lockfile/fixtures/gradle/5-pkg @@ -0,0 +1,10 @@ +## +## Comments +## + +org.springframework.boot:spring-boot-autoconfigure:2.7.4=compileClasspath,developmentOnly,productionRuntimeClasspath,runtimeClasspath +org.springframework.boot:spring-boot-configuration-processor:2.7.5=annotationProcessor,compileClasspath +org.springframework.boot:spring-boot-devtools:2.7.6=developmentOnly,runtimeClasspath +org.springframework.boot:spring-boot-starter-aop:2.7.7=compileClasspath,productionRuntimeClasspath,runtimeClasspath +org.springframework.boot:spring-boot-starter-data-jpa:2.7.8=compileClasspath,productionRuntimeClasspath,runtimeClasspath +empty= diff --git a/pkg/lockfile/fixtures/gradle/one-pkg b/pkg/lockfile/fixtures/gradle/one-pkg new file mode 100644 index 00000000000..fcb55543724 --- /dev/null +++ b/pkg/lockfile/fixtures/gradle/one-pkg @@ -0,0 +1,5 @@ +# Comment example +# + +org.springframework.security:spring-security-crypto:5.7.3=compileClasspath,productionRuntimeClasspath,runtimeClasspath +empty= diff --git a/pkg/lockfile/fixtures/gradle/only-comments b/pkg/lockfile/fixtures/gradle/only-comments new file mode 100644 index 00000000000..503dfb227e9 --- /dev/null +++ b/pkg/lockfile/fixtures/gradle/only-comments @@ -0,0 +1,8 @@ +# This +# is +# a +# comment + + + + diff --git a/pkg/lockfile/fixtures/gradle/only-empty b/pkg/lockfile/fixtures/gradle/only-empty new file mode 100644 index 00000000000..c0918d5384f --- /dev/null +++ b/pkg/lockfile/fixtures/gradle/only-empty @@ -0,0 +1 @@ +empty= diff --git a/pkg/lockfile/fixtures/gradle/with-bad-pkg b/pkg/lockfile/fixtures/gradle/with-bad-pkg new file mode 100644 index 00000000000..6f0dcd5e0d8 --- /dev/null +++ b/pkg/lockfile/fixtures/gradle/with-bad-pkg @@ -0,0 +1,19 @@ +# This sample gradle lockfile has bad lines +# +>>> +//// +empty======= + +org.springframework.boot:spring-boot-autoconfigure:2.7.4=compileClasspath,developmentOnly,productionRuntimeClasspath,runtimeClasspath + + a + b + + + +org.springframework.boot:spring-boot-configuration-processor:2.7.5=compileClasspath,developmentOnly,productionRuntimeClasspath,runtimeClasspath + + + + + diff --git a/pkg/lockfile/parse-gradle-lock.go b/pkg/lockfile/parse-gradle-lock.go new file mode 100644 index 00000000000..6cce6034c23 --- /dev/null +++ b/pkg/lockfile/parse-gradle-lock.go @@ -0,0 +1,70 @@ +package lockfile + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +const ( + gradleLockFileCommentPrefix = "#" + gradleLockFileEmptyPrefix = "empty=" +) + +func isGradleLockFileDepLine(line string) bool { + ret := strings.HasPrefix(line, gradleLockFileCommentPrefix) || + strings.HasPrefix(line, gradleLockFileEmptyPrefix) + + return !ret +} + +func parseToGradlePackageDetail(line string) (PackageDetails, error) { + parts := strings.SplitN(line, ":", 3) + if len(parts) < 3 { + return PackageDetails{}, fmt.Errorf("invalid line in gradle lockfile: %s", line) + } + + group, artifact, version := parts[0], parts[1], parts[2] + version = strings.SplitN(version, "=", 2)[0] + + return PackageDetails{ + Name: fmt.Sprintf("%s:%s", group, artifact), + Version: version, + Ecosystem: MavenEcosystem, + CompareAs: MavenEcosystem, + }, nil +} + +func ParseGradleLock(pathToLockfile string) ([]PackageDetails, error) { + lockFile, err := os.Open(pathToLockfile) + if err != nil { + return []PackageDetails{}, fmt.Errorf("could not open %s: %w", pathToLockfile, err) + } + + defer lockFile.Close() + + pkgs := make([]PackageDetails, 0, 0) + scanner := bufio.NewScanner(lockFile) + + for scanner.Scan() { + lockLine := strings.TrimSpace(scanner.Text()) + if !isGradleLockFileDepLine(lockLine) { + continue + } + + pkg, err := parseToGradlePackageDetail(lockLine) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to parse lockline: %s\n", err.Error()) + continue + } + + pkgs = append(pkgs, pkg) + } + + if err := scanner.Err(); err != nil { + return []PackageDetails{}, fmt.Errorf("failed to read %s: %w", pathToLockfile, err) + } + + return pkgs, nil +} diff --git a/pkg/lockfile/parse-gradle-lock_test.go b/pkg/lockfile/parse-gradle-lock_test.go new file mode 100644 index 00000000000..73b694b9abe --- /dev/null +++ b/pkg/lockfile/parse-gradle-lock_test.go @@ -0,0 +1,128 @@ +package lockfile_test + +import ( + "testing" + + "github.com/google/osv-scanner/pkg/lockfile" +) + +func TestParseGradleLock_FileDoesNotExist(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseGradleLock("fixtures/gradle/does-not-exist") + + expectErrContaining(t, err, "could not open") + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseGradleLock_OnlyComments(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseGradleLock("fixtures/gradle/only-comments") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseGradleLock_EmptyStatement(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseGradleLock("fixtures/gradle/only-empty") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseGradleLock_OnePackage(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseGradleLock("fixtures/gradle/one-pkg") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "org.springframework.security:spring-security-crypto", + Version: "5.7.3", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + }) +} + +func TestParseGradleLock_MultiplePackage(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseGradleLock("fixtures/gradle/5-pkg") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "org.springframework.boot:spring-boot-autoconfigure", + Version: "2.7.4", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + { + Name: "org.springframework.boot:spring-boot-configuration-processor", + Version: "2.7.5", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + { + Name: "org.springframework.boot:spring-boot-devtools", + Version: "2.7.6", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + + { + Name: "org.springframework.boot:spring-boot-starter-aop", + Version: "2.7.7", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + { + Name: "org.springframework.boot:spring-boot-starter-data-jpa", + Version: "2.7.8", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + }) +} + +func TestParseGradleLock_WithInvalidLines(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseGradleLock("fixtures/gradle/with-bad-pkg") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "org.springframework.boot:spring-boot-autoconfigure", + Version: "2.7.4", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + { + Name: "org.springframework.boot:spring-boot-configuration-processor", + Version: "2.7.5", + Ecosystem: lockfile.MavenEcosystem, + CompareAs: lockfile.MavenEcosystem, + }, + }) +} diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go index e8cd9ef8b6e..6c474098138 100644 --- a/pkg/lockfile/parse.go +++ b/pkg/lockfile/parse.go @@ -18,18 +18,20 @@ func FindParser(pathToLockfile string, parseAs string) (PackageDetailsParser, st //nolint:gochecknoglobals // this is an optimisation and read-only var parsers = map[string]PackageDetailsParser{ - "Cargo.lock": ParseCargoLock, - "composer.lock": ParseComposerLock, - "Gemfile.lock": ParseGemfileLock, - "go.mod": ParseGoLock, - "mix.lock": ParseMixLock, - "package-lock.json": ParseNpmLock, - "pnpm-lock.yaml": ParsePnpmLock, - "poetry.lock": ParsePoetryLock, - "pom.xml": ParseMavenLock, - "pubspec.lock": ParsePubspecLock, - "requirements.txt": ParseRequirementsTxt, - "yarn.lock": ParseYarnLock, + "Cargo.lock": ParseCargoLock, + "composer.lock": ParseComposerLock, + "Gemfile.lock": ParseGemfileLock, + "go.mod": ParseGoLock, + "mix.lock": ParseMixLock, + "package-lock.json": ParseNpmLock, + "pnpm-lock.yaml": ParsePnpmLock, + "poetry.lock": ParsePoetryLock, + "pom.xml": ParseMavenLock, + "pubspec.lock": ParsePubspecLock, + "requirements.txt": ParseRequirementsTxt, + "yarn.lock": ParseYarnLock, + "gradle.lockfile": ParseGradleLock, + "buildscript-gradle.lockfile": ParseGradleLock, } func ListParsers() []string { diff --git a/pkg/lockfile/parse_test.go b/pkg/lockfile/parse_test.go index 2c73fb2105b..237354a415f 100644 --- a/pkg/lockfile/parse_test.go +++ b/pkg/lockfile/parse_test.go @@ -2,11 +2,12 @@ package lockfile_test import ( "errors" - "github.com/google/osv-scanner/pkg/lockfile" "os" "reflect" "strings" "testing" + + "github.com/google/osv-scanner/pkg/lockfile" ) func expectNumberOfParsersCalled(t *testing.T, numberOfParsersCalled int) { @@ -95,6 +96,8 @@ func TestParse_FindsExpectedParsers(t *testing.T) { "poetry.lock", "pubspec.lock", "requirements.txt", + "gradle.lockfile", + "buildscript-gradle.lockfile", } count := 0 @@ -109,6 +112,9 @@ func TestParse_FindsExpectedParsers(t *testing.T) { count++ } + // gradle.lockfile and buildscript-gradle.lockfile use the same parser + count = count - 1 + expectNumberOfParsersCalled(t, count) } @@ -131,12 +137,15 @@ func TestListParsers(t *testing.T) { parsers := lockfile.ListParsers() - if first := parsers[0]; first != "Cargo.lock" { - t.Errorf("Expected first element to be Cargo.lock, but got %s", first) + firstExpected := "buildscript-gradle.lockfile" + lastExpected := "yarn.lock" + + if first := parsers[0]; first != firstExpected { + t.Errorf("Expected first element to be %s, but got %s", firstExpected, first) } - if last := parsers[len(parsers)-1]; last != "yarn.lock" { - t.Errorf("Expected last element to be requirements.txt, but got %s", last) + if last := parsers[len(parsers)-1]; last != lastExpected { + t.Errorf("Expected last element to be %s, but got %s", lastExpected, last) } }