From 55ee3b22af1497d1451f3e840aab7eb674fbc714 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Wed, 3 Jul 2024 15:11:54 +0600 Subject: [PATCH 1/7] fix(java): use `go-mvn-version` to remove duplicates --- pkg/dependency/parser/java/jar/parse.go | 27 ++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/pkg/dependency/parser/java/jar/parse.go b/pkg/dependency/parser/java/jar/parse.go index 8cb81d1fc48e..58aa4deaf774 100644 --- a/pkg/dependency/parser/java/jar/parse.go +++ b/pkg/dependency/parser/java/jar/parse.go @@ -6,15 +6,15 @@ import ( "crypto/sha1" // nolint:gosec "encoding/hex" "errors" - "fmt" + mavenversion "github.com/masahiro331/go-mvn-version" "io" "os" "path" "path/filepath" "regexp" + "slices" "strings" - "github.com/samber/lo" "golang.org/x/xerrors" ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" @@ -439,7 +439,24 @@ func (m manifest) determineVersion() (string, error) { } func removePackageDuplicates(pkgs []ftypes.Package) []ftypes.Package { - return lo.UniqBy(pkgs, func(pkg ftypes.Package) string { - return fmt.Sprintf("%s::%s::%s", pkg.Name, pkg.Version, pkg.FilePath) - }) + // name::filePath => versions + var uniq = make(map[string][]mavenversion.Version) + var uniqPkgs []ftypes.Package + for _, pkg := range pkgs { + uniqID := pkg.Name + "::" + pkg.FilePath + // err is always nil + // cf. https://github.com/masahiro331/go-mvn-version/blob/d3157d602a08806ad94464c443e0cef1370694a1/version.go#L20-L25 + pkgVer, _ := mavenversion.NewVersion(pkg.Version) + savedVers, ok := uniq[uniqID] + if !ok || !slices.ContainsFunc(savedVers, func(v mavenversion.Version) bool { + // There are times when patch `0` is omitted. + // So we can't compare versions just as strings + // for example `2.17.0` and `2.17` must be equal + return v.Equal(pkgVer) + }) { + uniq[uniqID] = []mavenversion.Version{pkgVer} + uniqPkgs = append(uniqPkgs, pkg) + } + } + return uniqPkgs } From dac3e2a5c7001572a495655da716eba173993d94 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Wed, 3 Jul 2024 15:12:03 +0600 Subject: [PATCH 2/7] test(java): update test --- pkg/dependency/parser/java/jar/parse_test.go | 14 +++++++------- ...nal.jar => io.quarkus.gizmo.gizmo-1.1.jar} | Bin 882367 -> 882459 bytes 2 files changed, 7 insertions(+), 7 deletions(-) rename pkg/dependency/parser/java/jar/testdata/{io.quarkus.gizmo.gizmo-1.1.1.Final.jar => io.quarkus.gizmo.gizmo-1.1.jar} (99%) diff --git a/pkg/dependency/parser/java/jar/parse_test.go b/pkg/dependency/parser/java/jar/parse_test.go index 20f9c682be3b..e9db923705d9 100644 --- a/pkg/dependency/parser/java/jar/parse_test.go +++ b/pkg/dependency/parser/java/jar/parse_test.go @@ -172,18 +172,18 @@ var ( wantDuplicatesJar = []ftypes.Package{ { Name: "io.quarkus.gizmo:gizmo", - Version: "1.1.1.Final", - FilePath: "testdata/io.quarkus.gizmo.gizmo-1.1.1.Final.jar", + Version: "1.1", + FilePath: "testdata/io.quarkus.gizmo.gizmo-1.1.jar", }, { Name: "log4j:log4j", Version: "1.2.16", - FilePath: "testdata/io.quarkus.gizmo.gizmo-1.1.1.Final.jar/jars/log4j-1.2.16.jar", + FilePath: "testdata/io.quarkus.gizmo.gizmo-1.1.jar/jars/log4j-1.2.16.jar", }, { Name: "log4j:log4j", Version: "1.2.17", - FilePath: "testdata/io.quarkus.gizmo.gizmo-1.1.1.Final.jar/jars/log4j-1.2.17.jar", + FilePath: "testdata/io.quarkus.gizmo.gizmo-1.1.jar/jars/log4j-1.2.17.jar", }, } ) @@ -251,7 +251,7 @@ func TestParse(t *testing.T) { }, { name: "duplicate libraries", - file: "testdata/io.quarkus.gizmo.gizmo-1.1.1.Final.jar", + file: "testdata/io.quarkus.gizmo.gizmo-1.1.jar", want: wantDuplicatesJar, }, } @@ -277,13 +277,13 @@ func TestParse(t *testing.T) { } case strings.Contains(r.URL.Query().Get("q"), "Gizmo"): res.Response.NumFound = 0 - case strings.Contains(r.URL.Query().Get("q"), "85d30c06026afd9f5be26da3194d4698c447a904"): + case strings.Contains(r.URL.Query().Get("q"), "1c78bbc4d8c58b9af8eee82b84f2c26ec48e9a2b"): res.Response.Docs = []doc{ { ID: "io.quarkus.gizmo.gizmo", GroupID: "io.quarkus.gizmo", ArtifactID: "gizmo", - Version: "1.1.1.Final", + Version: "1.1.0", }, } case strings.Contains(r.URL.Query().Get("q"), "heuristic"): diff --git a/pkg/dependency/parser/java/jar/testdata/io.quarkus.gizmo.gizmo-1.1.1.Final.jar b/pkg/dependency/parser/java/jar/testdata/io.quarkus.gizmo.gizmo-1.1.jar similarity index 99% rename from pkg/dependency/parser/java/jar/testdata/io.quarkus.gizmo.gizmo-1.1.1.Final.jar rename to pkg/dependency/parser/java/jar/testdata/io.quarkus.gizmo.gizmo-1.1.jar index 080b15ba5e40ec9431d97d41fd2cd9afa29be811..84c2fe2908c7679239215546d6688b7e814fe476 100644 GIT binary patch delta 741 zcmdo0)^zqe(}ott7N!>F7M2#)7Pc1l7LFFq7OocV7M?A<<{9-KuoJ-S+y-uqLU451)7(nDefM>7oVO`O3+jtiLoI8ejWJ%`BGq zs2ek_DRS|9tGB+(CIu#^9~93lay&5ikH(j!-@Obrm#h|AZ1FYyGQ-brECw&W2AulETq4o7 zer@JSE{m&PoN0di*yj(KiX!#)Ang+sX(K0=$m{*^vEoS@7 zVqP;wMLA{>1_lle2Fw_j02<4{z&QOw39qa~Xb3ALqR~y`E#=i>JUQK>l-Gv&TP82a z5sK4i00o&%vcNp`>2E;13J6b6H^7^b31%s#FQAqJE#A_oRtz=(;wL?@3$XYnz?+o~ RF7M2#)7Pc1l7LFFq7OocV7M?A<<{9;20Y0IZ3thH{FfuTt zFflNQFfcIqx`sIFdiuHP`#So0y1532==r+!PB@)+$UwmL`@~QD{~9g$eOTf|SN!r= zxYj8#+H#w4Op>*)jNIQ}Qj3CIB167;81LFwe9Tzc>&YI&dy}sBUf^mLU|VOr($a8F ziJlB(ZTGsBZJi5}fZC&vR7d;KbZPUN2`J6P| zAs!zTpOwDHexGW%&vglv^@pNnb6i#0V-L)oViu)D*Gd;0d_n9|fdUZOl^j>57zK7fY%*Oi zi&uGiT^TRe^t@sqF{PMSpF`orw3*Xq9A@2qw3ye7QBedGV;md|*y0gr1_Q%%nNnWa z>4BxZT8x{g*Oc@+J3(V7;ZdAsr!JGu)shbq!=jv7D<^*^% tG9g=m;y$qX5MoQCKrz@H_38J^c=^Nvyjj^mGTcDO&&+S#m From a5c6d065ae419825406fea38a5dda37a34d58983 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Wed, 3 Jul 2024 15:29:47 +0600 Subject: [PATCH 3/7] refactor import --- pkg/dependency/parser/java/jar/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dependency/parser/java/jar/parse.go b/pkg/dependency/parser/java/jar/parse.go index 58aa4deaf774..00058d007d37 100644 --- a/pkg/dependency/parser/java/jar/parse.go +++ b/pkg/dependency/parser/java/jar/parse.go @@ -6,7 +6,6 @@ import ( "crypto/sha1" // nolint:gosec "encoding/hex" "errors" - mavenversion "github.com/masahiro331/go-mvn-version" "io" "os" "path" @@ -15,6 +14,7 @@ import ( "slices" "strings" + mavenversion "github.com/masahiro331/go-mvn-version" "golang.org/x/xerrors" ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" From 9eeed00fdd835ce1996763721f137e72684bc881 Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Thu, 4 Jul 2024 10:25:17 +0600 Subject: [PATCH 4/7] refactor Co-authored-by: Teppei Fukuda --- pkg/dependency/parser/java/jar/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dependency/parser/java/jar/parse.go b/pkg/dependency/parser/java/jar/parse.go index 00058d007d37..be767e0cf248 100644 --- a/pkg/dependency/parser/java/jar/parse.go +++ b/pkg/dependency/parser/java/jar/parse.go @@ -454,7 +454,7 @@ func removePackageDuplicates(pkgs []ftypes.Package) []ftypes.Package { // for example `2.17.0` and `2.17` must be equal return v.Equal(pkgVer) }) { - uniq[uniqID] = []mavenversion.Version{pkgVer} + uniq[uniqID] = append(uniq[uniqID], pkgVer) uniqPkgs = append(uniqPkgs, pkg) } } From e36b4b18de38f4dd9fc0aac1ee9a6a666b0f2ff1 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 4 Jul 2024 13:00:00 +0600 Subject: [PATCH 5/7] test(unit): update `io.quarkus.gizmo.gizmo-1.1.jar` to check regression --- pkg/dependency/parser/java/jar/parse_test.go | 32 ++++++++++++++++++ .../testdata/io.quarkus.gizmo.gizmo-1.1.jar | Bin 882459 -> 884288 bytes 2 files changed, 32 insertions(+) diff --git a/pkg/dependency/parser/java/jar/parse_test.go b/pkg/dependency/parser/java/jar/parse_test.go index e9db923705d9..725dc3454682 100644 --- a/pkg/dependency/parser/java/jar/parse_test.go +++ b/pkg/dependency/parser/java/jar/parse_test.go @@ -169,6 +169,28 @@ var ( } // manually created + // . + //├── bar + //│ ├── bar + //│ │ └── pom.properties (jackson-databind:2.13.4) + //│ └── foo + //│ └── pom.properties (jackson-databind:2.12.3) + //├── foo + //│ ├── bar + //│ │ └── pom.properties (jackson-databind:2.12.3 + //│ └── foo + //│ └── pom.properties (jackson-databind:2.13.4) + //├── jars + //│ ├── log4j-1.2.16.jar (log4j:1.2.16) + //│ └── log4j-1.2.17.jar (log4j:1.2.17) + //└── META-INF + // ├── INDEX.LIST + // ├── MANIFEST.MF + // └── maven + // └── io.quarkus.gizmo + // └── gizmo + // ├── pom.properties (gizmo:1.1) + // └── pom.xml wantDuplicatesJar = []ftypes.Package{ { Name: "io.quarkus.gizmo:gizmo", @@ -185,6 +207,16 @@ var ( Version: "1.2.17", FilePath: "testdata/io.quarkus.gizmo.gizmo-1.1.jar/jars/log4j-1.2.17.jar", }, + { + Name: "com.fasterxml.jackson.core:jackson-databind", + Version: "2.12.3", + FilePath: "testdata/io.quarkus.gizmo.gizmo-1.1.jar", + }, + { + Name: "com.fasterxml.jackson.core:jackson-databind", + Version: "2.13.4", + FilePath: "testdata/io.quarkus.gizmo.gizmo-1.1.jar", + }, } ) diff --git a/pkg/dependency/parser/java/jar/testdata/io.quarkus.gizmo.gizmo-1.1.jar b/pkg/dependency/parser/java/jar/testdata/io.quarkus.gizmo.gizmo-1.1.jar index 84c2fe2908c7679239215546d6688b7e814fe476..3b82fae55ec99e93c949375061ebf7af82aa96bd 100644 GIT binary patch delta 1788 zcmbRJ&h)?^(}ott7N!>F7M2#)7Pc1l7LFFq7OocV7M?A4L@^E2 z_SV%pcjo+N(@VxCmyDlz>7UZq{ou3F_uTob-rhQTp1P|}oV|G3%jcZlx$~F3&*|5D zJ@wMjI_cx>qsO?asq6JJm1!z!sn3=&Fd#hl6D??9o`VO^R-orNG2H?(0pd4Fpx-#& ztl&rYTT)_ukv^xwWN%Fq(n7rwsWT6nGd8%-w&(Wlm+o`4y}_p(`?1 zfBb2+BkrAEmX58fgtAX!=+1MN>XX<+&vx6rZQ#u`e_8LaNOgf#xXS-)@15O_w^sA% zBjO!fq6WDP9z38#&5AGH&4GUNRb#gCLh~EQ1c=}2Q4%#5&|@Hs<~ig-~i6#)?aj6mDBG`b<_M^6|)(|`p6s%tRQCbDbp zLrsIERG@xfCPsA)W~N1UjbbUpHz563Tm#I(VAITCVS|~^kX^G2Xc|0RfVqwopZtL8 z#qAT!EQah8t1^h)ID7)kY^bimAIq1aronv!i{(66EMsO0WY>t6LtKN%B8VIUj3q3& WBfy)L4U|4bfv_6LS`RdVfdK%bUc|fr delta 62 zcmX@`$8`2P(}ott7N!>F7M2#)7Pc1l7LFFq7OocV7M?A Date: Thu, 4 Jul 2024 13:02:13 +0600 Subject: [PATCH 6/7] refactor(comment): add `1.1.0` version for `io.quarkus.gizmo.gizmo-1.1.jar` --- pkg/dependency/parser/java/jar/parse_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/dependency/parser/java/jar/parse_test.go b/pkg/dependency/parser/java/jar/parse_test.go index 725dc3454682..c9eca9e900b6 100644 --- a/pkg/dependency/parser/java/jar/parse_test.go +++ b/pkg/dependency/parser/java/jar/parse_test.go @@ -169,7 +169,7 @@ var ( } // manually created - // . + // io.quarkus.gizmo.gizmo-1.1.jar (gizmo:1.1.0 (from sha1)) //├── bar //│ ├── bar //│ │ └── pom.properties (jackson-databind:2.13.4) @@ -177,7 +177,7 @@ var ( //│ └── pom.properties (jackson-databind:2.12.3) //├── foo //│ ├── bar - //│ │ └── pom.properties (jackson-databind:2.12.3 + //│ │ └── pom.properties (jackson-databind:2.12.3) //│ └── foo //│ └── pom.properties (jackson-databind:2.13.4) //├── jars From 7a5018e1d943839b85ea9a7df6404f82f1aaad89 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 4 Jul 2024 13:03:11 +0600 Subject: [PATCH 7/7] refactor(comment) --- pkg/dependency/parser/java/jar/parse_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/dependency/parser/java/jar/parse_test.go b/pkg/dependency/parser/java/jar/parse_test.go index c9eca9e900b6..958b2d0667a5 100644 --- a/pkg/dependency/parser/java/jar/parse_test.go +++ b/pkg/dependency/parser/java/jar/parse_test.go @@ -168,8 +168,8 @@ var ( }, } - // manually created - // io.quarkus.gizmo.gizmo-1.1.jar (gizmo:1.1.0 (from sha1)) + // Manually created. + // Files of `io.quarkus.gizmo.gizmo-1.1.jar` (gizmo:1.1.0 (from sha1)): //├── bar //│ ├── bar //│ │ └── pom.properties (jackson-databind:2.13.4)