From c8622d5f3472698e872b7b6a6ff817da42bbcf07 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 7 Feb 2017 11:48:42 -0800 Subject: [PATCH] vulnsrc/alpine: unify schema and parse v3.5 HEAD of Alpine SecDB now uses one consistent schema for all of their vulnerabilities, so the logic around parsing different versions can now be removed. This change also crawls the directory structure to parse all files due to the addition of community.yaml tracking community Alpine Linux packages. --- ext/vulnsrc/alpine/alpine.go | 127 ++++++++-------------- ext/vulnsrc/alpine/alpine_test.go | 22 +--- ext/vulnsrc/alpine/testdata/v33_main.yaml | 69 ------------ 3 files changed, 47 insertions(+), 171 deletions(-) delete mode 100644 ext/vulnsrc/alpine/testdata/v33_main.yaml diff --git a/ext/vulnsrc/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go index 5f3acb22dd..e99926ba7f 100644 --- a/ext/vulnsrc/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -17,11 +17,11 @@ package alpine import ( - "fmt" "io" "io/ioutil" "os" "os/exec" + "path/filepath" "strings" "gopkg.in/yaml.v2" @@ -79,8 +79,13 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er return } + // Get the list of namespaces from the repository. var namespaces []string - namespaces, err = detectNamespaces(u.repositoryLocalPath) + namespaces, err = ls(u.repositoryLocalPath, directoriesOnly) + if err != nil { + return + } + // Append any changed vulnerabilities to the response. for _, namespace := range namespaces { var vulns []database.Vulnerability @@ -104,58 +109,70 @@ func (u *updater) Clean() { } } -func detectNamespaces(path string) ([]string, error) { - // Open the root directory. +type lsFilter int + +const ( + filesOnly lsFilter = iota + directoriesOnly +) + +func ls(path string, filter lsFilter) ([]string, error) { dir, err := os.Open(path) if err != nil { return nil, err } defer dir.Close() - // Get a list of the namspaces from the directory names. finfos, err := dir.Readdir(0) if err != nil { return nil, err } - var namespaces []string + var files []string for _, info := range finfos { - if !info.IsDir() { + if filter == directoriesOnly && !info.IsDir() { + continue + } + + if filter == filesOnly && info.IsDir() { continue } - // Filter out hidden directories like `.git`. + if strings.HasPrefix(info.Name(), ".") { continue } - namespaces = append(namespaces, info.Name()) + files = append(files, info.Name()) } - return namespaces, nil -} - -type parserFunc func(io.Reader) ([]database.Vulnerability, error) - -var parsers = map[string]parserFunc{ - "v3.3": parse33YAML, - "v3.4": parse34YAML, + return files, nil } func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, note string, err error) { - var file io.ReadCloser - file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml") + nsDir := filepath.Join(repositoryPath, namespace) + var dbFilenames []string + dbFilenames, err = ls(nsDir, filesOnly) if err != nil { return } - defer file.Close() - parseFunc, exists := parsers[namespace] - if !exists { - note = fmt.Sprintf("The file %s is not mapped to any Alpine version number", namespace) - return + for _, filename := range dbFilenames { + var file io.ReadCloser + file, err = os.Open(filepath.Join(nsDir, filename)) + if err != nil { + return + } + + var fileVulns []database.Vulnerability + fileVulns, err = parseYAML(file) + if err != nil { + return + } + + vulns = append(vulns, fileVulns...) + file.Close() } - vulns, err = parseFunc(file) return } @@ -193,61 +210,7 @@ func (u *updater) pullRepository() (commit string, err error) { return } -type secdb33File struct { - Distro string `yaml:"distroversion"` - Packages []struct { - Pkg struct { - Name string `yaml:"name"` - Version string `yaml:"ver"` - Fixes []string `yaml:"fixes"` - } `yaml:"pkg"` - } `yaml:"packages"` -} - -func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) { - var rBytes []byte - rBytes, err = ioutil.ReadAll(r) - if err != nil { - return - } - - var file secdb33File - err = yaml.Unmarshal(rBytes, &file) - if err != nil { - return - } - for _, pack := range file.Packages { - pkg := pack.Pkg - for _, fix := range pkg.Fixes { - err = versionfmt.Valid(dpkg.ParserName, pkg.Version) - if err != nil { - log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error()) - continue - } - - vulns = append(vulns, database.Vulnerability{ - Name: fix, - Severity: database.UnknownSeverity, - Link: nvdURLPrefix + fix, - FixedIn: []database.FeatureVersion{ - { - Feature: database.Feature{ - Namespace: database.Namespace{ - Name: "alpine:" + file.Distro, - VersionFormat: dpkg.ParserName, - }, - Name: pkg.Name, - }, - Version: pkg.Version, - }, - }, - }) - } - } - return -} - -type secdb34File struct { +type secDBFile struct { Distro string `yaml:"distroversion"` Packages []struct { Pkg struct { @@ -257,14 +220,14 @@ type secdb34File struct { } `yaml:"packages"` } -func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) { +func parseYAML(r io.Reader) (vulns []database.Vulnerability, err error) { var rBytes []byte rBytes, err = ioutil.ReadAll(r) if err != nil { return } - var file secdb34File + var file secDBFile err = yaml.Unmarshal(rBytes, &file) if err != nil { return diff --git a/ext/vulnsrc/alpine/alpine_test.go b/ext/vulnsrc/alpine/alpine_test.go index 4588d97ec9..ac95f5c524 100644 --- a/ext/vulnsrc/alpine/alpine_test.go +++ b/ext/vulnsrc/alpine/alpine_test.go @@ -23,32 +23,14 @@ import ( "github.com/stretchr/testify/assert" ) -func TestAlpine33YAMLParsing(t *testing.T) { - _, filename, _, _ := runtime.Caller(0) - path := filepath.Join(filepath.Dir(filename)) - - testData, _ := os.Open(path + "/testdata/v33_main.yaml") - defer testData.Close() - - vulns, err := parse33YAML(testData) - if err != nil { - assert.Nil(t, err) - } - assert.Equal(t, 15, len(vulns)) - assert.Equal(t, "CVE-2016-2147", vulns[0].Name) - assert.Equal(t, "alpine:v3.3", vulns[0].FixedIn[0].Feature.Namespace.Name) - assert.Equal(t, "busybox", vulns[0].FixedIn[0].Feature.Name) - assert.Equal(t, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-2147", vulns[0].Link) -} - -func TestAlpine34YAMLParsing(t *testing.T) { +func TestYAMLParsing(t *testing.T) { _, filename, _, _ := runtime.Caller(0) path := filepath.Join(filepath.Dir(filename)) testData, _ := os.Open(path + "/testdata/v34_main.yaml") defer testData.Close() - vulns, err := parse34YAML(testData) + vulns, err := parseYAML(testData) if err != nil { assert.Nil(t, err) } diff --git a/ext/vulnsrc/alpine/testdata/v33_main.yaml b/ext/vulnsrc/alpine/testdata/v33_main.yaml deleted file mode 100644 index 21f246682f..0000000000 --- a/ext/vulnsrc/alpine/testdata/v33_main.yaml +++ /dev/null @@ -1,69 +0,0 @@ ---- -distroversion: v3.3 -reponame: main -archs: - - x86_64 - - x86 - - armhf -urlprefix: http://dl-cdn.alpinelinux.org/alpine -apkurl: "{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-${{pkg.ver}}.apk" -packages: - - pkg: - name: busybox - ver: 1.24.2-r0 - fixes: - - CVE-2016-2147 - - CVE-2016-2148 - - pkg: - name: expat - ver: 2.1.1-r1 - fixes: - - CVE-2016-0718 - - pkg: - name: gd - ver: 2.1.1-r1 - fixes: - - CVE-2016-3074 - - pkg: - name: giflib - ver: 5.1.1-r1 - fixes: - - CVE-2016-3977 - - pkg: - name: jq - ver: 1.5-r1 - fixes: - - CVE-2015-8863 - - pkg: - name: libarchive - ver: 3.1.2-r3 - fixes: - - CVE-2016-1541 - - pkg: - name: libssh2 - ver: 1.6.0-r1 - fixes: - - CVE-2016-0787 - - pkg: - name: mercurial - ver: 3.7.3-r1 - fixes: - - CVE-2016-3105 - - pkg: - name: openssl - ver: 1.0.2h-r1 - fixes: - - CVE-2016-2177 - - CVE-2016-2178 - - pkg: - name: pcre - ver: 8.38-r1 - fixes: - - CVE-2016-1283 - - CVE-2016-3191 - - pkg: - name: wpa_supplicant - ver: 2.5-r2 - fixes: - - CVE-2016-4476 - - CVE-2016-4477