From e7ffc3161df3445592cf9e1ea4149d8c72f41f01 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Fri, 30 Dec 2022 18:38:55 +0100 Subject: [PATCH 01/13] Added Alpine APK filelock support --- pkg/lockfile/ecosystems.go | 1 + pkg/lockfile/parse-apk-installed.go | 58 +++++++++++++++++++++++++++++ pkg/lockfile/parse.go | 1 + 3 files changed, 60 insertions(+) create mode 100644 pkg/lockfile/parse-apk-installed.go diff --git a/pkg/lockfile/ecosystems.go b/pkg/lockfile/ecosystems.go index adc5dc78e1f..1efe559cfdb 100644 --- a/pkg/lockfile/ecosystems.go +++ b/pkg/lockfile/ecosystems.go @@ -11,5 +11,6 @@ func KnownEcosystems() []Ecosystem { MavenEcosystem, PipEcosystem, PubEcosystem, + AlpineEcosystem, } } diff --git a/pkg/lockfile/parse-apk-installed.go b/pkg/lockfile/parse-apk-installed.go new file mode 100644 index 00000000000..6ca8ca7f6d3 --- /dev/null +++ b/pkg/lockfile/parse-apk-installed.go @@ -0,0 +1,58 @@ +package lockfile + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +const AlpineEcosystem Ecosystem = "Alpine" + +func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { + var packages []PackageDetails + + file, err := os.Open(pathToLockfile) + if err != nil { + return packages, fmt.Errorf("could not open %s: %w", pathToLockfile, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + var curPkg PackageDetails = PackageDetails{} + + for scanner.Scan() { + line := scanner.Text() + if line == ""{ + // First line is empty or multiple empty lines, no current package values + if (PackageDetails{}) == curPkg{ + continue + } + // Empty line follows a package info block. Append package + //TODO Check if PackageDetails is valid? + packages = append(packages, curPkg) + curPkg = PackageDetails{} + continue + } + // https://wiki.alpinelinux.org/wiki/Apk_spec + if strings.HasPrefix(line,"P:"){ + curPkg.Name = strings.Split(line,"P:")[1] + } else if strings.HasPrefix(line,"V:"){ + curPkg.Version = strings.Split(line,"V:")[1] + // Add only in one of the + curPkg.Ecosystem = AlpineEcosystem + curPkg.CompareAs = AlpineEcosystem + } + + } + if (PackageDetails{}) != curPkg{ + packages = append(packages, curPkg) + } + + if err := scanner.Err(); err != nil { + return packages, fmt.Errorf("error while scanning %s: %w", pathToLockfile, err) + } + + return packages, nil +} \ No newline at end of file diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go index 6c474098138..83caa0a9043 100644 --- a/pkg/lockfile/parse.go +++ b/pkg/lockfile/parse.go @@ -32,6 +32,7 @@ var parsers = map[string]PackageDetailsParser{ "yarn.lock": ParseYarnLock, "gradle.lockfile": ParseGradleLock, "buildscript-gradle.lockfile": ParseGradleLock, + "installed": ParseApkInstalled, } func ListParsers() []string { From 8bac0e74257635f9fb1563b6f78b8fcd5695fb05 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Fri, 30 Dec 2022 20:50:22 +0100 Subject: [PATCH 02/13] Alpine APK Installed Database V2 - Added some basic tests --- pkg/lockfile/fixtures/apk/installed_empty | 0 pkg/lockfile/fixtures/apk/installed_multiple | 123 +++++++++++++++++++ pkg/lockfile/fixtures/apk/installed_single | 32 +++++ pkg/lockfile/parse-apk-installed_test.go | 68 ++++++++++ 4 files changed, 223 insertions(+) create mode 100644 pkg/lockfile/fixtures/apk/installed_empty create mode 100644 pkg/lockfile/fixtures/apk/installed_multiple create mode 100644 pkg/lockfile/fixtures/apk/installed_single create mode 100644 pkg/lockfile/parse-apk-installed_test.go diff --git a/pkg/lockfile/fixtures/apk/installed_empty b/pkg/lockfile/fixtures/apk/installed_empty new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkg/lockfile/fixtures/apk/installed_multiple b/pkg/lockfile/fixtures/apk/installed_multiple new file mode 100644 index 00000000000..9d0fa07ce65 --- /dev/null +++ b/pkg/lockfile/fixtures/apk/installed_multiple @@ -0,0 +1,123 @@ +C:Q1/JgpM8J6DWI/541tUX+uHEzSjqo= +P:alpine-baselayout-data +V:3.4.0-r0 +A:x86_64 +S:11664 +I:77824 +T:Alpine base dir structure and init scripts +U:https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout +L:GPL-2.0-only +o:alpine-baselayout +m:redacted +t:1668104640 +c:f93af038c3de7146121c2ea8124ba5ce29b4b058 +p:so:libc.musl-x86_64.so.1=1 +F:lib +R:ld-musl-x86_64.so.1 +a:0:0:755 +Z:Q1tGxgx2FLrD+0Uk03NUBwbbEiRCU= +R:libc.musl-x86_64.so.1 +a:0:0:777 +Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI= + +C:Q1NN3sp0yr99btRysqty3nQUrWHaY= +P:busybox +V:1.35.0-r29 +A:x86_64 +S:509600 +I:962560 +T:Size optimized toolbox of many common UNIX utilities +U:https://busybox.net/ +L:GPL-2.0-only +o:busybox +m:redacted +t:1668852790 +c:1dbf7a793afae640ea643a055b6dd4f430ac116b +D:so:libc.musl-x86_64.so.1 +p:cmd:busybox=1.35.0-r29 +r:busybox-initscripts +F:bin +R:busybox +a:0:0:755 +Z:Q1yVNLeeB7VouhCO/kz+dbfL3dY4c= +F:etc +R:securetty +Z:Q1mB95Hq2NUTZ599RDiSsj9w5FrOU= +R:udhcpd.conf +Z:Q1EgLFjj67ou3eMqp4m3r2ZjnQ7QU= +F:etc/logrotate.d +R:acpid +Z:Q1TylyCINVmnS+A/Tead4vZhE7Bks= +F:etc/network +F:etc/network/if-down.d +F:etc/network/if-post-down.d +F:etc/network/if-post-up.d +F:etc/network/if-pre-down.d +F:etc/network/if-pre-up.d +F:etc/network/if-up.d +R:dad +a:0:0:775 +Z:Q1ORf+lPRKuYgdkBBcKoevR1t60Q4= +F:sbin +F:tmp +M:0:0:1777 +F:usr +F:usr/sbin +F:usr/share +F:usr/share/udhcpc +R:default.script +a:0:0:755 +Z:Q1HWpG3eQD8Uoi4mks2E3SSvOAUhY= +F:var +F:var/cache +F:var/cache/misc +F:var/lib +F:var/lib/udhcpd diff --git a/pkg/lockfile/fixtures/apk/installed_single b/pkg/lockfile/fixtures/apk/installed_single new file mode 100644 index 00000000000..181245d8265 --- /dev/null +++ b/pkg/lockfile/fixtures/apk/installed_single @@ -0,0 +1,32 @@ +C:Q1Ef3iwt+cMdGngEgaFr2URIJhKzQ= +P:apk-tools +V:2.12.10-r1 +A:x86_64 +S:120973 +I:307200 +T:Alpine Package Keeper - package manager for alpine +U:https://gitlab.alpinelinux.org/alpine/apk-tools +L:GPL-2.0-only +o:apk-tools +m:Natanael Copa +t:1666552494 +c:0188f510baadbae393472103427b9c1875117136 +D:musl>=1.2 ca-certificates-bundle so:libc.musl-x86_64.so.1 so:libcrypto.so.3 so:libssl.so.3 so:libz.so.1 +p:so:libapk.so.3.12.0=3.12.0 cmd:apk=2.12.10-r1 +F:etc +F:etc/apk +F:etc/apk/keys +F:etc/apk/protected_paths.d +F:lib +R:libapk.so.3.12.0 +a:0:0:755 +Z:Q1opjpYqXgzmOVo7EbNe8l5Xol08g= +F:lib/apk +F:lib/apk/exec +F:sbin +R:apk +a:0:0:755 +Z:Q1/4bmOPe/H1YhHRzlrj27oufThMw= +F:var +F:var/lib +F:var/lib/apk \ No newline at end of file diff --git a/pkg/lockfile/parse-apk-installed_test.go b/pkg/lockfile/parse-apk-installed_test.go new file mode 100644 index 00000000000..ff9d16d0167 --- /dev/null +++ b/pkg/lockfile/parse-apk-installed_test.go @@ -0,0 +1,68 @@ +package lockfile_test + +import ( + "github.com/google/osv-scanner/pkg/lockfile" + "testing" +) + +func TestParseApkInstalled_Empty(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseApkInstalled("fixtures/apk/installed_empty") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseApkInstalled_Single(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseApkInstalled("fixtures/apk/installed_single") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "apk-tools", + Version: "2.12.10-r1", + Ecosystem: lockfile.AlpineEcosystem, + CompareAs: lockfile.AlpineEcosystem, + }, + }) +} + +func TestParseApkInstalled_Multiple(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseApkInstalled("fixtures/apk/installed_multiple") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "alpine-baselayout-data", + Version: "3.4.0-r0", + Ecosystem: lockfile.AlpineEcosystem, + CompareAs: lockfile.AlpineEcosystem, + }, + { + Name: "musl", + Version: "1.2.3-r4", + Ecosystem: lockfile.AlpineEcosystem, + CompareAs: lockfile.AlpineEcosystem, + }, + { + Name: "busybox", + Version: "1.35.0-r29", + Ecosystem: lockfile.AlpineEcosystem, + CompareAs: lockfile.AlpineEcosystem, + }, + }) +} \ No newline at end of file From e61b2ac7629f39c7332f7ef4e5dda767dc172e45 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Fri, 30 Dec 2022 21:02:16 +0100 Subject: [PATCH 03/13] Updated README with /lib/apk/db/installed support --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fceaeb4a8eb..174108c0b16 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ A wide range of lockfiles are supported by utilizing this [lockfile package](htt - `requirements.txt`[\*](https://github.com/google/osv-scanner/issues/34) - `gradle.lockfile` - `buildscript-gradle.lockfile` +- `/lib/apk/db/installed` #### Example From 089186271a90fa6b83ea91e94e4e169e1865e5bf Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Fri, 30 Dec 2022 21:11:33 +0100 Subject: [PATCH 04/13] Alpine installed file - reformat and remove unwanted comments --- pkg/lockfile/parse-apk-installed.go | 30 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/pkg/lockfile/parse-apk-installed.go b/pkg/lockfile/parse-apk-installed.go index 6ca8ca7f6d3..f3229b783b9 100644 --- a/pkg/lockfile/parse-apk-installed.go +++ b/pkg/lockfile/parse-apk-installed.go @@ -17,42 +17,40 @@ func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { return packages, fmt.Errorf("could not open %s: %w", pathToLockfile, err) } defer file.Close() - + scanner := bufio.NewScanner(file) - + var curPkg PackageDetails = PackageDetails{} for scanner.Scan() { line := scanner.Text() - if line == ""{ + if line == "" { // First line is empty or multiple empty lines, no current package values - if (PackageDetails{}) == curPkg{ + if (PackageDetails{}) == curPkg { continue } - // Empty line follows a package info block. Append package - //TODO Check if PackageDetails is valid? + // Empty line follows a package info block. Append package before going to next one packages = append(packages, curPkg) curPkg = PackageDetails{} continue } - // https://wiki.alpinelinux.org/wiki/Apk_spec - if strings.HasPrefix(line,"P:"){ - curPkg.Name = strings.Split(line,"P:")[1] - } else if strings.HasPrefix(line,"V:"){ - curPkg.Version = strings.Split(line,"V:")[1] - // Add only in one of the + // File SPECS: https://wiki.alpinelinux.org/wiki/Apk_spec + if strings.HasPrefix(line, "P:") { + curPkg.Name = strings.Split(line, "P:")[1] + } else if strings.HasPrefix(line, "V:") { + curPkg.Version = strings.Split(line, "V:")[1] curPkg.Ecosystem = AlpineEcosystem curPkg.CompareAs = AlpineEcosystem } - + } - if (PackageDetails{}) != curPkg{ + if (PackageDetails{}) != curPkg { packages = append(packages, curPkg) } - + if err := scanner.Err(); err != nil { return packages, fmt.Errorf("error while scanning %s: %w", pathToLockfile, err) } return packages, nil -} \ No newline at end of file +} From 6d12e4d4fa68a33dcd3f28602335f21435b80a67 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Sun, 1 Jan 2023 10:33:43 +0100 Subject: [PATCH 05/13] TrimPrefix and missing formatting --- pkg/lockfile/parse-apk-installed.go | 4 ++-- pkg/lockfile/parse-apk-installed_test.go | 2 +- pkg/lockfile/parse.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/lockfile/parse-apk-installed.go b/pkg/lockfile/parse-apk-installed.go index f3229b783b9..bec62399b17 100644 --- a/pkg/lockfile/parse-apk-installed.go +++ b/pkg/lockfile/parse-apk-installed.go @@ -36,9 +36,9 @@ func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { } // File SPECS: https://wiki.alpinelinux.org/wiki/Apk_spec if strings.HasPrefix(line, "P:") { - curPkg.Name = strings.Split(line, "P:")[1] + curPkg.Name = strings.TrimPrefix(line, "P:") } else if strings.HasPrefix(line, "V:") { - curPkg.Version = strings.Split(line, "V:")[1] + curPkg.Version = strings.TrimPrefix(line, "V:") curPkg.Ecosystem = AlpineEcosystem curPkg.CompareAs = AlpineEcosystem } diff --git a/pkg/lockfile/parse-apk-installed_test.go b/pkg/lockfile/parse-apk-installed_test.go index ff9d16d0167..22a9c2ccdab 100644 --- a/pkg/lockfile/parse-apk-installed_test.go +++ b/pkg/lockfile/parse-apk-installed_test.go @@ -65,4 +65,4 @@ func TestParseApkInstalled_Multiple(t *testing.T) { CompareAs: lockfile.AlpineEcosystem, }, }) -} \ No newline at end of file +} diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go index 83caa0a9043..b6789afa2c8 100644 --- a/pkg/lockfile/parse.go +++ b/pkg/lockfile/parse.go @@ -32,7 +32,7 @@ var parsers = map[string]PackageDetailsParser{ "yarn.lock": ParseYarnLock, "gradle.lockfile": ParseGradleLock, "buildscript-gradle.lockfile": ParseGradleLock, - "installed": ParseApkInstalled, + "installed": ParseApkInstalled, } func ListParsers() []string { From 94403aa63f29051d622795a65ece40e931701aac Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Sun, 1 Jan 2023 10:49:11 +0100 Subject: [PATCH 06/13] Alpine APK support - Update parse_test.go to reflect the new parser and rename of fixtures to better match standard naming --- .../fixtures/apk/{installed_empty => empty_installed} | 0 .../fixtures/apk/{installed_multiple => multiple_installed} | 0 .../fixtures/apk/{installed_single => single_installed} | 0 pkg/lockfile/parse-apk-installed_test.go | 6 +++--- pkg/lockfile/parse_test.go | 1 + 5 files changed, 4 insertions(+), 3 deletions(-) rename pkg/lockfile/fixtures/apk/{installed_empty => empty_installed} (100%) rename pkg/lockfile/fixtures/apk/{installed_multiple => multiple_installed} (100%) rename pkg/lockfile/fixtures/apk/{installed_single => single_installed} (100%) diff --git a/pkg/lockfile/fixtures/apk/installed_empty b/pkg/lockfile/fixtures/apk/empty_installed similarity index 100% rename from pkg/lockfile/fixtures/apk/installed_empty rename to pkg/lockfile/fixtures/apk/empty_installed diff --git a/pkg/lockfile/fixtures/apk/installed_multiple b/pkg/lockfile/fixtures/apk/multiple_installed similarity index 100% rename from pkg/lockfile/fixtures/apk/installed_multiple rename to pkg/lockfile/fixtures/apk/multiple_installed diff --git a/pkg/lockfile/fixtures/apk/installed_single b/pkg/lockfile/fixtures/apk/single_installed similarity index 100% rename from pkg/lockfile/fixtures/apk/installed_single rename to pkg/lockfile/fixtures/apk/single_installed diff --git a/pkg/lockfile/parse-apk-installed_test.go b/pkg/lockfile/parse-apk-installed_test.go index 22a9c2ccdab..15bd5a34b0b 100644 --- a/pkg/lockfile/parse-apk-installed_test.go +++ b/pkg/lockfile/parse-apk-installed_test.go @@ -8,7 +8,7 @@ import ( func TestParseApkInstalled_Empty(t *testing.T) { t.Parallel() - packages, err := lockfile.ParseApkInstalled("fixtures/apk/installed_empty") + packages, err := lockfile.ParseApkInstalled("fixtures/apk/empty_installed") if err != nil { t.Errorf("Got unexpected error: %v", err) @@ -20,7 +20,7 @@ func TestParseApkInstalled_Empty(t *testing.T) { func TestParseApkInstalled_Single(t *testing.T) { t.Parallel() - packages, err := lockfile.ParseApkInstalled("fixtures/apk/installed_single") + packages, err := lockfile.ParseApkInstalled("fixtures/apk/single_installed") if err != nil { t.Errorf("Got unexpected error: %v", err) @@ -39,7 +39,7 @@ func TestParseApkInstalled_Single(t *testing.T) { func TestParseApkInstalled_Multiple(t *testing.T) { t.Parallel() - packages, err := lockfile.ParseApkInstalled("fixtures/apk/installed_multiple") + packages, err := lockfile.ParseApkInstalled("fixtures/apk/multiple_installed") if err != nil { t.Errorf("Got unexpected error: %v", err) diff --git a/pkg/lockfile/parse_test.go b/pkg/lockfile/parse_test.go index 237354a415f..c545536d730 100644 --- a/pkg/lockfile/parse_test.go +++ b/pkg/lockfile/parse_test.go @@ -98,6 +98,7 @@ func TestParse_FindsExpectedParsers(t *testing.T) { "requirements.txt", "gradle.lockfile", "buildscript-gradle.lockfile", + "installed", } count := 0 From 89b21696ac4a906a5970915221c86ba4056e8a46 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Sun, 1 Jan 2023 11:10:35 +0100 Subject: [PATCH 07/13] Alpine APK support - Update tests adding shuffled installed file and the case of missing file --- pkg/lockfile/fixtures/apk/shuffled_installed | 32 ++++++++++++++++++++ pkg/lockfile/parse-apk-installed_test.go | 28 +++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 pkg/lockfile/fixtures/apk/shuffled_installed diff --git a/pkg/lockfile/fixtures/apk/shuffled_installed b/pkg/lockfile/fixtures/apk/shuffled_installed new file mode 100644 index 00000000000..c1cbf66baa0 --- /dev/null +++ b/pkg/lockfile/fixtures/apk/shuffled_installed @@ -0,0 +1,32 @@ +F:lib/apk/exec +F:var/lib +F:sbin +L:GPL-2.0-only +D:musl>=1.2 ca-certificates-bundle so:libc.musl-x86_64.so.1 so:libcrypto.so.3 so:libssl.so.3 so:libz.so.1 +m:Natanael Copa +Z:Q1/4bmOPe/H1YhHRzlrj27oufThMw= +I:307200 +S:120973 +c:0188f510baadbae393472103427b9c1875117136 +F:etc +t:1666552494 +A:x86_64 +V:2.12.10-r1 +F:lib/apk +F:var/lib/apk +a:0:0:755 +C:Q1Ef3iwt+cMdGngEgaFr2URIJhKzQ= +T:Alpine Package Keeper - package manager for alpine +R:apk +P:apk-tools +F:etc/apk/protected_paths.d +F:etc/apk/keys +F:lib +Z:Q1opjpYqXgzmOVo7EbNe8l5Xol08g= +F:var +p:so:libapk.so.3.12.0=3.12.0 cmd:apk=2.12.10-r1 +R:libapk.so.3.12.0 +a:0:0:755 +F:etc/apk +o:apk-tools +U:https://gitlab.alpinelinux.org/alpine/apk-tools \ No newline at end of file diff --git a/pkg/lockfile/parse-apk-installed_test.go b/pkg/lockfile/parse-apk-installed_test.go index 15bd5a34b0b..e379e83e4cd 100644 --- a/pkg/lockfile/parse-apk-installed_test.go +++ b/pkg/lockfile/parse-apk-installed_test.go @@ -5,6 +5,15 @@ import ( "testing" ) +func TestParseApkInstalled_FileDoesNotExist(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRequirementsTxt("fixtures/apk/does-not-exist") + + expectErrContaining(t, err, "could not open") + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + func TestParseApkInstalled_Empty(t *testing.T) { t.Parallel() @@ -36,6 +45,25 @@ func TestParseApkInstalled_Single(t *testing.T) { }) } +func TestParseApkInstalled_Shuffled(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseApkInstalled("fixtures/apk/shuffled_installed") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "apk-tools", + Version: "2.12.10-r1", + Ecosystem: lockfile.AlpineEcosystem, + CompareAs: lockfile.AlpineEcosystem, + }, + }) +} + func TestParseApkInstalled_Multiple(t *testing.T) { t.Parallel() From 1cdaba31205c12fc3ec7cbc31c9027cea9e6b381 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Mon, 2 Jan 2023 09:10:39 +0100 Subject: [PATCH 08/13] fix: APK installed - support for missing pkg and version fields. Added commit parsing. --- pkg/lockfile/fixtures/apk/malformed_installed | 131 ++++++++++++++++++ pkg/lockfile/fixtures/apk/not_installed | 12 ++ pkg/lockfile/parse-apk-installed.go | 41 +++++- pkg/lockfile/parse-apk-installed_test.go | 37 +++++ 4 files changed, 218 insertions(+), 3 deletions(-) create mode 100644 pkg/lockfile/fixtures/apk/malformed_installed create mode 100644 pkg/lockfile/fixtures/apk/not_installed diff --git a/pkg/lockfile/fixtures/apk/malformed_installed b/pkg/lockfile/fixtures/apk/malformed_installed new file mode 100644 index 00000000000..ac23d9633b6 --- /dev/null +++ b/pkg/lockfile/fixtures/apk/malformed_installed @@ -0,0 +1,131 @@ + +This is a malformed APK installed file + + + + no package: + +Z:Q1olh8TpdAi2QnTl4FK3TjdUiSwTo= +R:loader_attic.so +R:afalg.so +Z:Q1mcqLbO6iQe8TmQCoRDozFWScisQ= +F:usr/lib +r:libcrypto1.1 +F:etc +F:lib +F:usr/lib/ossl-modules +R:libcrypto.so.3 +a:0:0:755 +Z:Q1AFDkJxxzzc5SCdOjdbK1BA/vbsY= +o:openssl +I:4206592 +R:legacy.so +Z:Q1fqYq/iJ6x71cTpr8fcO4/6IgyQg= +C:Q1TjmrAEa5PKd40PNz6OIpo+bBeW0= +a:0:0:755 +R:openssl.cnf +R:openssl.cnf.dist +F:etc/ssl/private +R:padlock.so +Z:Q1zr8y7mYzOdgG1uz+DmGLBLOZ/jM= +a:0:0:755 +R:capi.so +S:1707237 +F:etc/ssl +a:0:0:777 +p:so:libcrypto.so.3=3 +R:ct_log_list.cnf.dist +F:usr/lib/engines-3 +V:3.0.7-r0 +Z:Q1WjKZkr5xeMyOxhVrCQfW04JiiME= +F:etc/ssl/certs +c:37a47708fdd97644624ff4b7238bb3299e037eaf +a:0:0:777 +t:1667317778 +a:0:0:755 +Z:Q1op76VCo7av+GQqk9nT9kEezP1I8= +L:Apache-2.0 +R:tsget +a:0:0:755 +R:CA.pl +R:ct_log_list.cnf +D:so:libc.musl-x86_64.so.1 +Z:Q1XK8nt7AyX7GIGpMOLlkJk5dy81c= +R:libcrypto.so.3 +T:Crypto library from openssl +Z:Q1fqYq/iJ6x71cTpr8fcO4/6IgyQg= +Z:Q1olh8TpdAi2QnTl4FK3TjdUiSwTo= +F:usr +Z:Q13NVgfr7dQUuGYxur0tNalH6EIjU= +R:tsget.pl +Z:Q1yzxstq05Nm+4DAS0gR/XScMthRY= +a:0:0:755 +a:0:0:755 +a:0:0:755 +Z:Q1nwKfkE6NiHpVJ8wZRoQYglMEYwQ= +U:https://www.openssl.org/ +F:etc/ssl/misc +A:x86_64 +Z:Q1Uv35WBwtuePGrdxQuLDKbHVVTT4= + +nothing here: +Z:Q1yVNLeeB7VouhCO/kz+dbfL3dY4c= +F:sbin +Z:Q1EgLFjj67ou3eMqp4m3r2ZjnQ7QU= +Z:Q1ORf+lPRKuYgdkBBcKoevR1t60Q4= +M:0:0:1777 +Z:Q1mB95Hq2NUTZ599RDiSsj9w5FrOU= +F:etc +F:var +F:etc/network/if-pre-up.d +Z:Q1HWpG3eQD8Uoi4mks2E3SSvOAUhY= +F:usr/sbin + + + + + + + + +no version +F:var/lib +A:x86_64 +F:var/cache +F:etc/network/if-up.d +F:usr/share +t:1668852790 +F:etc/network/if-down.d +a:0:0:755 +C:Q1NN3sp0yr99btRysqty3nQUrWHaY= +S:509600 +a:0:0:755 +F:etc/network/if-post-up.d +o:busybox +R:udhcpd.conf +U:https://busybox.net/ +F:var/cache/misc +a:0:0:775 +F:usr/share/udhcpc +p:cmd:busybox=1.35.0-r29 +R:dad +R:securetty +F:etc/network/if-post-down.d +I:962560 +F:etc/logrotate.d +D:so:libc.musl-x86_64.so.1 +F:bin +c:1dbf7a793afae640ea643a055b6dd4f430ac116b +F:var/lib/udhcpd +F:tmp +Z:Q1TylyCINVmnS+A/Tead4vZhE7Bks= +T:Size optimized toolbox of many common UNIX utilities +F:etc/network/if-pre-down.d +R:default.script +R:acpid +r:busybox-initscripts +R:busybox +L:GPL-2.0-only +F:usr +F:etc/network +P:busybox \ No newline at end of file diff --git a/pkg/lockfile/fixtures/apk/not_installed b/pkg/lockfile/fixtures/apk/not_installed new file mode 100644 index 00000000000..0302b391438 --- /dev/null +++ b/pkg/lockfile/fixtures/apk/not_installed @@ -0,0 +1,12 @@ +Not an APK file! +[[package]] +name = "hello_world" +version = "0.1.0" +dependencies = [ + "regex 1.5.0 (git+https://github.com/rust-lang/regex.git#9f9f693768c584971a4d53bc3c586c33ed3a6831)", +] + +[[package]] +name = "regex" +version = "1.5.0" +source = "git+https://github.com/rust-lang/regex.git#9f9f693768c584971a4d53bc3c586c33ed3a6831" diff --git a/pkg/lockfile/parse-apk-installed.go b/pkg/lockfile/parse-apk-installed.go index bec62399b17..1c35a2238ff 100644 --- a/pkg/lockfile/parse-apk-installed.go +++ b/pkg/lockfile/parse-apk-installed.go @@ -30,6 +30,23 @@ func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { continue } // Empty line follows a package info block. Append package before going to next one + // If Package name is missing, record is invalid then skip + if curPkg.Name == "" { + _, _ = fmt.Fprintf( + os.Stderr, + "Warning: malformed APK installed file. Found no package name in record. File: %s\n", + pathToLockfile, + ) + curPkg = PackageDetails{} + continue + } + if curPkg.Version == "" { + _, _ = fmt.Fprintf( + os.Stderr, + "Warning: malformed APK installed file. Found no version number in record. File: %s\n", + pathToLockfile, + ) + } packages = append(packages, curPkg) curPkg = PackageDetails{} continue @@ -37,15 +54,33 @@ func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { // File SPECS: https://wiki.alpinelinux.org/wiki/Apk_spec if strings.HasPrefix(line, "P:") { curPkg.Name = strings.TrimPrefix(line, "P:") - } else if strings.HasPrefix(line, "V:") { - curPkg.Version = strings.TrimPrefix(line, "V:") curPkg.Ecosystem = AlpineEcosystem curPkg.CompareAs = AlpineEcosystem + } else if strings.HasPrefix(line, "V:") { + curPkg.Version = strings.TrimPrefix(line, "V:") + } else if strings.HasPrefix(line, "c:") { + curPkg.Commit = strings.TrimPrefix(line, "c:") } } + if (PackageDetails{}) != curPkg { - packages = append(packages, curPkg) + if curPkg.Name == "" { + _, _ = fmt.Fprintf( + os.Stderr, + "Warning: malformed APK installed file. Found no package name in record. File: %s\n", + pathToLockfile, + ) + } else { + if curPkg.Version == "" { + _, _ = fmt.Fprintf( + os.Stderr, + "Warning: malformed APK installed file. Found no version number in record. File: %s\n", + pathToLockfile, + ) + } + packages = append(packages, curPkg) + } } if err := scanner.Err(); err != nil { diff --git a/pkg/lockfile/parse-apk-installed_test.go b/pkg/lockfile/parse-apk-installed_test.go index e379e83e4cd..1b3877447c8 100644 --- a/pkg/lockfile/parse-apk-installed_test.go +++ b/pkg/lockfile/parse-apk-installed_test.go @@ -26,6 +26,38 @@ func TestParseApkInstalled_Empty(t *testing.T) { expectPackages(t, packages, []lockfile.PackageDetails{}) } +func TestParseApkInstalled_NotAnInstalled(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseApkInstalled("fixtures/apk/not_installed") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseApkInstalled_Malformed(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseApkInstalled("fixtures/apk/malformed_installed") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "busybox", + Version: "", + Commit: "1dbf7a793afae640ea643a055b6dd4f430ac116b", + Ecosystem: lockfile.AlpineEcosystem, + CompareAs: lockfile.AlpineEcosystem, + }, + }) +} + func TestParseApkInstalled_Single(t *testing.T) { t.Parallel() @@ -39,6 +71,7 @@ func TestParseApkInstalled_Single(t *testing.T) { { Name: "apk-tools", Version: "2.12.10-r1", + Commit: "0188f510baadbae393472103427b9c1875117136", Ecosystem: lockfile.AlpineEcosystem, CompareAs: lockfile.AlpineEcosystem, }, @@ -58,6 +91,7 @@ func TestParseApkInstalled_Shuffled(t *testing.T) { { Name: "apk-tools", Version: "2.12.10-r1", + Commit: "0188f510baadbae393472103427b9c1875117136", Ecosystem: lockfile.AlpineEcosystem, CompareAs: lockfile.AlpineEcosystem, }, @@ -77,18 +111,21 @@ func TestParseApkInstalled_Multiple(t *testing.T) { { Name: "alpine-baselayout-data", Version: "3.4.0-r0", + Commit: "bd965a7ebf7fd8f07d7a0cc0d7375bf3e4eb9b24", Ecosystem: lockfile.AlpineEcosystem, CompareAs: lockfile.AlpineEcosystem, }, { Name: "musl", Version: "1.2.3-r4", + Commit: "f93af038c3de7146121c2ea8124ba5ce29b4b058", Ecosystem: lockfile.AlpineEcosystem, CompareAs: lockfile.AlpineEcosystem, }, { Name: "busybox", Version: "1.35.0-r29", + Commit: "1dbf7a793afae640ea643a055b6dd4f430ac116b", Ecosystem: lockfile.AlpineEcosystem, CompareAs: lockfile.AlpineEcosystem, }, From e03ae3d309c40bf7a91534ddfb09f335ab6e1e70 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Tue, 3 Jan 2023 18:24:01 +0100 Subject: [PATCH 09/13] APK installed support: refactor to minimize duplicate loops --- pkg/lockfile/parse-apk-installed.go | 121 ++++++++++++++-------------- 1 file changed, 62 insertions(+), 59 deletions(-) diff --git a/pkg/lockfile/parse-apk-installed.go b/pkg/lockfile/parse-apk-installed.go index 1c35a2238ff..a1b885babaa 100644 --- a/pkg/lockfile/parse-apk-installed.go +++ b/pkg/lockfile/parse-apk-installed.go @@ -9,77 +9,80 @@ import ( const AlpineEcosystem Ecosystem = "Alpine" -func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { - var packages []PackageDetails - - file, err := os.Open(pathToLockfile) - if err != nil { - return packages, fmt.Errorf("could not open %s: %w", pathToLockfile, err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - - var curPkg PackageDetails = PackageDetails{} +func groupApkPackageLines(scanner *bufio.Scanner) [][]string { + var groups [][]string + var group []string for scanner.Scan() { line := scanner.Text() + if line == "" { - // First line is empty or multiple empty lines, no current package values - if (PackageDetails{}) == curPkg { - continue - } - // Empty line follows a package info block. Append package before going to next one - // If Package name is missing, record is invalid then skip - if curPkg.Name == "" { - _, _ = fmt.Fprintf( - os.Stderr, - "Warning: malformed APK installed file. Found no package name in record. File: %s\n", - pathToLockfile, - ) - curPkg = PackageDetails{} - continue - } - if curPkg.Version == "" { - _, _ = fmt.Fprintf( - os.Stderr, - "Warning: malformed APK installed file. Found no version number in record. File: %s\n", - pathToLockfile, - ) + if len(group) > 0 { + groups = append(groups, group) } - packages = append(packages, curPkg) - curPkg = PackageDetails{} + group = make([]string, 0) continue } - // File SPECS: https://wiki.alpinelinux.org/wiki/Apk_spec - if strings.HasPrefix(line, "P:") { - curPkg.Name = strings.TrimPrefix(line, "P:") - curPkg.Ecosystem = AlpineEcosystem - curPkg.CompareAs = AlpineEcosystem - } else if strings.HasPrefix(line, "V:") { - curPkg.Version = strings.TrimPrefix(line, "V:") - } else if strings.HasPrefix(line, "c:") { - curPkg.Commit = strings.TrimPrefix(line, "c:") + group = append(group, line) + } + + if len(group) > 0 { + groups = append(groups, group) + } + + return groups +} + +func parseApkPackageGroup(group []string, pathToLockfile string) (PackageDetails, error) { + var pkg PackageDetails = PackageDetails{} + + // File SPECS: https://wiki.alpinelinux.org/wiki/Apk_spec + for _, line := range group { + switch { + case strings.HasPrefix(line, "P:"): + pkg.Name = strings.TrimPrefix(line, "P:") + pkg.Ecosystem = AlpineEcosystem + pkg.CompareAs = AlpineEcosystem + case strings.HasPrefix(line, "V:"): + pkg.Version = strings.TrimPrefix(line, "V:") + case strings.HasPrefix(line, "c:"): + pkg.Commit = strings.TrimPrefix(line, "c:") } + } + if pkg.Name == "" { + return PackageDetails{}, fmt.Errorf("warning: malformed APK installed file. Found no version number in record. File: %s", pathToLockfile) + } + if pkg.Version == "" { + _, _ = fmt.Fprintf( + os.Stderr, + "warning: malformed APK installed file. Found no version number in record. File: %s\n", + pathToLockfile, + ) } - if (PackageDetails{}) != curPkg { - if curPkg.Name == "" { - _, _ = fmt.Fprintf( - os.Stderr, - "Warning: malformed APK installed file. Found no package name in record. File: %s\n", - pathToLockfile, - ) + return pkg, nil +} + +func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { + + file, err := os.Open(pathToLockfile) + if err != nil { + return []PackageDetails{}, fmt.Errorf("could not open %s: %w", pathToLockfile, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + packageGroups := groupApkPackageLines(scanner) + + packages := make([]PackageDetails, 0, len(packageGroups)) + + for _, group := range packageGroups { + if pkg, err := parseApkPackageGroup(group, pathToLockfile); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) } else { - if curPkg.Version == "" { - _, _ = fmt.Fprintf( - os.Stderr, - "Warning: malformed APK installed file. Found no version number in record. File: %s\n", - pathToLockfile, - ) - } - packages = append(packages, curPkg) + packages = append(packages, pkg) } } From 08d27e06d2ce404c6420d73a922e2ca031d5e19f Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Wed, 4 Jan 2023 09:51:25 +0100 Subject: [PATCH 10/13] Refactor: rename Yarn parse package functions to follow apk installed naming --- pkg/lockfile/parse-yarn-lock.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/lockfile/parse-yarn-lock.go b/pkg/lockfile/parse-yarn-lock.go index 3603269477a..b2271780919 100644 --- a/pkg/lockfile/parse-yarn-lock.go +++ b/pkg/lockfile/parse-yarn-lock.go @@ -15,7 +15,7 @@ func shouldSkipYarnLine(line string) bool { return line == "" || strings.HasPrefix(line, "#") } -func groupPackageLines(scanner *bufio.Scanner) [][]string { +func groupYarnPackageLines(scanner *bufio.Scanner) [][]string { var groups [][]string var group []string @@ -148,7 +148,7 @@ func tryExtractCommit(resolution string) string { return "" } -func parsePackageGroup(group []string) PackageDetails { +func parseYarnPackageGroup(group []string) PackageDetails { name := extractYarnPackageName(group[0]) version := determineYarnPackageVersion(group) resolution := determineYarnPackageResolution(group) @@ -179,7 +179,7 @@ func ParseYarnLock(pathToLockfile string) ([]PackageDetails, error) { scanner := bufio.NewScanner(file) - packageGroups := groupPackageLines(scanner) + packageGroups := groupYarnPackageLines(scanner) if err := scanner.Err(); err != nil { return []PackageDetails{}, fmt.Errorf("error while scanning %s: %w", pathToLockfile, err) @@ -192,7 +192,7 @@ func ParseYarnLock(pathToLockfile string) ([]PackageDetails, error) { continue } - packages = append(packages, parsePackageGroup(group)) + packages = append(packages, parseYarnPackageGroup(group)) } return packages, nil From 68fddd3e286dc4a86cb88a05a6febde39eb6e99c Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Wed, 4 Jan 2023 11:57:49 +0100 Subject: [PATCH 11/13] APK installed support: refactor to match with similar Yarn parser --- pkg/lockfile/parse-apk-installed.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pkg/lockfile/parse-apk-installed.go b/pkg/lockfile/parse-apk-installed.go index a1b885babaa..d8b3703e16d 100644 --- a/pkg/lockfile/parse-apk-installed.go +++ b/pkg/lockfile/parse-apk-installed.go @@ -33,16 +33,17 @@ func groupApkPackageLines(scanner *bufio.Scanner) [][]string { return groups } -func parseApkPackageGroup(group []string, pathToLockfile string) (PackageDetails, error) { - var pkg PackageDetails = PackageDetails{} +func parseApkPackageGroup(group []string, pathToLockfile string) PackageDetails { + var pkg = PackageDetails{} + + pkg.Ecosystem = AlpineEcosystem + pkg.CompareAs = AlpineEcosystem // File SPECS: https://wiki.alpinelinux.org/wiki/Apk_spec for _, line := range group { switch { case strings.HasPrefix(line, "P:"): pkg.Name = strings.TrimPrefix(line, "P:") - pkg.Ecosystem = AlpineEcosystem - pkg.CompareAs = AlpineEcosystem case strings.HasPrefix(line, "V:"): pkg.Version = strings.TrimPrefix(line, "V:") case strings.HasPrefix(line, "c:"): @@ -50,9 +51,6 @@ func parseApkPackageGroup(group []string, pathToLockfile string) (PackageDetails } } - if pkg.Name == "" { - return PackageDetails{}, fmt.Errorf("warning: malformed APK installed file. Found no version number in record. File: %s", pathToLockfile) - } if pkg.Version == "" { _, _ = fmt.Fprintf( os.Stderr, @@ -61,7 +59,7 @@ func parseApkPackageGroup(group []string, pathToLockfile string) (PackageDetails ) } - return pkg, nil + return pkg } func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { @@ -79,8 +77,15 @@ func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { packages := make([]PackageDetails, 0, len(packageGroups)) for _, group := range packageGroups { - if pkg, err := parseApkPackageGroup(group, pathToLockfile); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) + pkg := parseApkPackageGroup(group, pathToLockfile) + + if pkg.Name == "" { + _, _ = fmt.Fprintf( + os.Stderr, + "warning: malformed APK installed file. Found no package name in record. File: %s\n", + pathToLockfile, + ) + continue } else { packages = append(packages, pkg) } From bdd5cbefe7d4fe3a7dc67afd2c0437d759056499 Mon Sep 17 00:00:00 2001 From: Claudio Maritan Date: Fri, 6 Jan 2023 08:00:57 +0100 Subject: [PATCH 12/13] APK installed support: Add package name in warning message. Small additional refactoring --- pkg/lockfile/parse-apk-installed.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pkg/lockfile/parse-apk-installed.go b/pkg/lockfile/parse-apk-installed.go index d8b3703e16d..1ee431b0608 100644 --- a/pkg/lockfile/parse-apk-installed.go +++ b/pkg/lockfile/parse-apk-installed.go @@ -34,10 +34,10 @@ func groupApkPackageLines(scanner *bufio.Scanner) [][]string { } func parseApkPackageGroup(group []string, pathToLockfile string) PackageDetails { - var pkg = PackageDetails{} - - pkg.Ecosystem = AlpineEcosystem - pkg.CompareAs = AlpineEcosystem + var pkg = PackageDetails{ + Ecosystem: AlpineEcosystem, + CompareAs: AlpineEcosystem, + } // File SPECS: https://wiki.alpinelinux.org/wiki/Apk_spec for _, line := range group { @@ -52,9 +52,15 @@ func parseApkPackageGroup(group []string, pathToLockfile string) PackageDetails } if pkg.Version == "" { + var pkgPrintName string = pkg.Name + if pkgPrintName == "" { + pkgPrintName = "" + } + _, _ = fmt.Fprintf( os.Stderr, - "warning: malformed APK installed file. Found no version number in record. File: %s\n", + "warning: malformed APK installed file. Found no version number in record. Package %s. File: %s\n", + pkgPrintName, pathToLockfile, ) } @@ -63,7 +69,6 @@ func parseApkPackageGroup(group []string, pathToLockfile string) PackageDetails } func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { - file, err := os.Open(pathToLockfile) if err != nil { return []PackageDetails{}, fmt.Errorf("could not open %s: %w", pathToLockfile, err) @@ -86,9 +91,9 @@ func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) { pathToLockfile, ) continue - } else { - packages = append(packages, pkg) } + + packages = append(packages, pkg) } if err := scanner.Err(); err != nil { From 1427fce1d0bd27f6603c06db660b72b9c060a64e Mon Sep 17 00:00:00 2001 From: Rex P <106129829+another-rex@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:51:25 +1300 Subject: [PATCH 13/13] Update pkg/lockfile/parse.go Fix indentation I messed up in the merge --- pkg/lockfile/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go index 6e2cde8ec41..579b10cb6e0 100644 --- a/pkg/lockfile/parse.go +++ b/pkg/lockfile/parse.go @@ -24,7 +24,7 @@ var parsers = map[string]PackageDetailsParser{ "Gemfile.lock": ParseGemfileLock, "go.mod": ParseGoLock, "gradle.lockfile": ParseGradleLock, - "installed": ParseApkInstalled, + "installed": ParseApkInstalled, "mix.lock": ParseMixLock, "Pipfile.lock": ParsePipenvLock, "package-lock.json": ParseNpmLock,