diff --git a/detector/vuls2/db.go b/detector/vuls2/db.go index 02e4d86498..70729b1e60 100644 --- a/detector/vuls2/db.go +++ b/detector/vuls2/db.go @@ -1,35 +1,23 @@ package vuls2 import ( - "context" - "encoding/json" - "io" + "fmt" "os" "path/filepath" - "strconv" "time" + "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/logging" - "github.com/klauspost/compress/zstd" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - progressbar "github.com/schollz/progressbar/v3" "golang.org/x/xerrors" - oras "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content" - "oras.land/oras-go/v2/content/memory" - "oras.land/oras-go/v2/registry/remote" db "github.com/MaineK00n/vuls2/pkg/db/common" -) - -const ( - dbMediaType = "application/vnd.vulsio.vuls.db.layer.v1+zstd" + "github.com/MaineK00n/vuls2/pkg/db/fetch" ) var ( // DefaultGHCRRepository is GitHub Container Registry for vuls2 db - DefaultGHCRRepository = "ghcr.io/vulsio/vuls-nightly-db" + DefaultGHCRRepository = fmt.Sprintf("%s:%d", "ghcr.io/vulsio/vuls-nightly-db", db.SchemaVersion) // DefaultPath is the path for vuls2 db file DefaultPath = func() string { @@ -38,160 +26,68 @@ var ( }() ) -type Config struct { - Repository string - Path string - SkipUpdate bool - Quiet bool -} - -func (c Config) Refresh() error { - lastModified, fileExists, err := c.loadLastModified() +func newDBConnection(vuls2Cnf config.Vuls2DictConf, noProgress bool) (db.DB, error) { + willDownload, err := shouldDownload(vuls2Cnf, time.Now()) if err != nil { - return xerrors.Errorf("Failed to load vuls2 db metadata. err: %w", err) - } - - if fileExists && time.Now().Before(lastModified.Add(6*time.Hour)) { - return nil + return nil, xerrors.Errorf("Failed to check whether to download vuls2 db. err: %w", err) } - if c.SkipUpdate { - if !fileExists { - return xerrors.New("Vuls2 db not found, cannot skip update") + if willDownload { + logging.Log.Infof("Downloading vuls2 db. repository: %s", vuls2Cnf.Repository) + if err := fetch.Fetch(fetch.WithRepository(vuls2Cnf.Repository), fetch.WithDBPath(vuls2Cnf.Path), fetch.WithNoProgress(noProgress)); err != nil { + return nil, xerrors.Errorf("Failed to fetch vuls2 db. err: %w", err) } - return nil } - logging.Log.Infof("Downloading vuls2 db. repository: %s", c.Repository) - if err := c.fetch(c.Repository); err != nil { - return xerrors.Errorf("Failed to fetch vuls2 db. repository: %s, err: %w", c.Repository, err) - } - - return nil -} - -func (c Config) New() (db.DB, error) { - vuls2Config := db.Config{ + dbc, err := (&db.Config{ Type: "boltdb", - Path: c.Path, - } - - dbc, err := vuls2Config.New() + Path: vuls2Cnf.Path, + }).New() if err != nil { - return nil, xerrors.Errorf("Failed to new vuls2 db. err: %w", err) + return nil, xerrors.Errorf("Failed to new vuls2 db connection. err: %w", err) } return dbc, nil } -func (c Config) fetch(repoPath string) error { - logging.Log.Infof("Fetch vuls.db from %s", repoPath) - - ctx := context.TODO() - - ms := memory.New() - - repo, err := remote.NewRepository(repoPath) - if err != nil { - return xerrors.Errorf("Failed to create repository client. repository: %s, err: %w", repoPath, err) - } - - manifestDescriptor, err := oras.Copy(ctx, repo, strconv.Itoa(db.SchemaVersion), ms, "", oras.DefaultCopyOptions) - if err != nil { - return xerrors.Errorf("Failed to copy. repository: %s, err: %w", repoPath, err) - } - - r, err := ms.Fetch(ctx, manifestDescriptor) - if err != nil { - return xerrors.Errorf("Failed to fetch manifest. err: %w", err) - } - defer r.Close() - - var manifest ocispec.Manifest - if err := json.NewDecoder(content.NewVerifyReader(r, manifestDescriptor)).Decode(&manifest); err != nil { - return xerrors.Errorf("Failed to decode manifest. err: %w", err) - } - - l := func() *ocispec.Descriptor { - for _, l := range manifest.Layers { - if l.MediaType == dbMediaType { - return &l +func shouldDownload(vuls2Cnf config.Vuls2DictConf, now time.Time) (bool, error) { + if _, err := os.Stat(vuls2Cnf.Path); err != nil { + if errors.Is(err, os.ErrNotExist) { + if vuls2Cnf.SkipUpdate { + return false, xerrors.Errorf("%s not found, cannot skip update", vuls2Cnf.Path) } + return true, nil } - return nil - }() - if l == nil { - return xerrors.Errorf("Failed to find digest and filename from layers. actual layers: %#v", manifest.Layers) - } - - r, err = repo.Fetch(ctx, *l) - if err != nil { - return xerrors.Errorf("Failed to fetch content. err: %w", err) + return false, xerrors.Errorf("Failed to stat vuls2 db file. err: %w", err) } - defer r.Close() - d, err := zstd.NewReader(content.NewVerifyReader(r, *l)) - if err != nil { - return errors.Wrap(err, "new zstd reader") - } - defer d.Close() - - if err := os.MkdirAll(filepath.Dir(c.Path), 0755); err != nil { - return errors.Wrapf(err, "mkdir %s", filepath.Dir(c.Path)) - } - - f, err := os.Create(c.Path) - if err != nil { - return xerrors.Errorf("Failed to create. file: %s, err:%w", c.Path, err) + if vuls2Cnf.SkipUpdate { + return false, nil } - defer f.Close() - var pb *progressbar.ProgressBar - pb = progressbar.DefaultBytesSilent(-1) - if !c.Quiet { - pb = progressbar.DefaultBytes(-1, "downloading") - } - if _, err := d.WriteTo(io.MultiWriter(f, pb)); err != nil { - return xerrors.Errorf("Failed to write. filename: %s. err: %w", f.Name(), err) - } - _ = pb.Finish() - - return nil -} - -func (c Config) loadLastModified() (time.Time, bool, error) { - if _, err := os.Stat(c.Path); errors.Is(err, os.ErrNotExist) { - return time.Time{}, false, nil - } - - conf := db.Config{ + dbc, err := (&db.Config{ Type: "boltdb", - Path: c.Path, - } - - dbc, err := conf.New() + Path: vuls2Cnf.Path, + }).New() if err != nil { - return time.Time{}, false, xerrors.Errorf("Failed to new vuls2 db. path: %s, err: %w", c.Path, err) + return false, xerrors.Errorf("Failed to new vuls2 db connection. path: %s, err: %w", vuls2Cnf.Path, err) } if err := dbc.Open(); err != nil { - return time.Time{}, false, xerrors.Errorf("Failed to open vuls2 db. path: %s, err: %w", c.Path, err) + return false, xerrors.Errorf("Failed to open vuls2 db. path: %s, err: %w", vuls2Cnf.Path, err) } - defer func() { - _ = dbc.Close() - }() + defer dbc.Close() metadata, err := dbc.GetMetadata() if err != nil { - return time.Time{}, false, xerrors.Errorf("Failed to get vuls2 db metadata. path: %s, err: %w", c.Path, err) + return false, xerrors.Errorf("Failed to get vuls2 db metadata. path: %s, err: %w", vuls2Cnf.Path, err) } if metadata == nil { - return time.Time{}, false, xerrors.Errorf("Unexpected Vuls2 db metadata. metadata: nil,. path: %s", c.Path) + return false, xerrors.Errorf("Unexpected Vuls2 db metadata. metadata: nil,. path: %s", vuls2Cnf.Path) } - if metadata.SchemaVersion != db.SchemaVersion { - return time.Time{}, false, xerrors.Errorf("Unexpected schema version. expected: %d, actual: %d", db.SchemaVersion, metadata.SchemaVersion) + if metadata.Downloaded != nil && now.Before((*metadata.Downloaded).Add(1*time.Hour)) { + return false, nil } - - return metadata.LastModified, true, nil + return metadata.LastModified.Add(6 * time.Hour).Before(now), nil } diff --git a/detector/vuls2/db_test.go b/detector/vuls2/db_test.go new file mode 100644 index 0000000000..87c50fb3ad --- /dev/null +++ b/detector/vuls2/db_test.go @@ -0,0 +1,149 @@ +package vuls2 + +import ( + "path/filepath" + "reflect" + "testing" + "time" + + "golang.org/x/xerrors" + + "github.com/MaineK00n/vuls2/pkg/db/common" + "github.com/MaineK00n/vuls2/pkg/db/common/types" + "github.com/future-architect/vuls/config" +) + +func Test_shouldDownload(t *testing.T) { + type args struct { + vuls2Cnf config.Vuls2DictConf + now time.Time + } + tests := []struct { + name string + args args + metadata *types.Metadata + want bool + wantErr bool + }{ + { + name: "no db file", + args: args{ + vuls2Cnf: config.Vuls2DictConf{}, + now: *parse("2024-01-02T00:00:00Z"), + }, + want: true, + }, + { + name: "no db file, but skip update", + args: args{ + vuls2Cnf: config.Vuls2DictConf{ + SkipUpdate: true, + }, + now: *parse("2024-01-02T00:00:00Z"), + }, + wantErr: true, + }, + { + name: "just created", + args: args{ + vuls2Cnf: config.Vuls2DictConf{}, + now: *parse("2024-01-02T00:00:00Z"), + }, + metadata: &types.Metadata{ + LastModified: *parse("2024-01-02T00:00:00Z"), + Downloaded: parse("2024-01-02T00:00:00Z"), + SchemaVersion: common.SchemaVersion, + }, + want: false, + }, + { + name: "8 hours old", + args: args{ + vuls2Cnf: config.Vuls2DictConf{}, + now: *parse("2024-01-02T08:00:00Z"), + }, + metadata: &types.Metadata{ + LastModified: *parse("2024-01-02T00:00:00Z"), + Downloaded: parse("2024-01-02T00:00:00Z"), + SchemaVersion: common.SchemaVersion, + }, + want: true, + }, + { + name: "8 hours old, but skip update", + args: args{ + vuls2Cnf: config.Vuls2DictConf{ + SkipUpdate: true, + }, + now: *parse("2024-01-02T08:00:00Z"), + }, + metadata: &types.Metadata{ + LastModified: *parse("2024-01-02T00:00:00Z"), + Downloaded: parse("2024-01-02T00:00:00Z"), + SchemaVersion: common.SchemaVersion, + }, + want: false, + }, + { + name: "8 hours old, but download recently", + args: args{ + vuls2Cnf: config.Vuls2DictConf{}, + now: *parse("2024-01-02T08:00:00Z"), + }, + metadata: &types.Metadata{ + LastModified: *parse("2024-01-02T00:00:00Z"), + Downloaded: parse("2024-01-02T07:30:00Z"), + SchemaVersion: common.SchemaVersion, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := t.TempDir() + tt.args.vuls2Cnf.Path = filepath.Join(d, "vuls.db") + if tt.metadata != nil { + if err := putMetadata(*tt.metadata, tt.args.vuls2Cnf.Path); err != nil { + t.Errorf("putMetadata err = %v", err) + return + } + } + got, err := shouldDownload(tt.args.vuls2Cnf, tt.args.now) + if (err != nil) != tt.wantErr { + t.Errorf("shouldDownload() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("shouldDownload() = %v, want %v", got, tt.want) + } + }) + } + +} + +func putMetadata(metadata types.Metadata, path string) error { + c := common.Config{ + Type: "boltdb", + Path: path, + } + dbc, err := c.New() + if err != nil { + return xerrors.Errorf("c.New(). err: %w", err) + } + if err := dbc.Open(); err != nil { + return xerrors.Errorf("dbc.Open(). err: %w", err) + } + defer dbc.Close() + if err := dbc.Initialize(); err != nil { + return xerrors.Errorf("dbc.Initialize(). err: %w", err) + } + if err := dbc.PutMetadata(metadata); err != nil { + return xerrors.Errorf("dbc.PutMetadata(). err: %w", err) + } + return nil +} + +func parse(date string) *time.Time { + t, _ := time.Parse(time.RFC3339, date) + return &t +} diff --git a/detector/vuls2/vuls2.go b/detector/vuls2/vuls2.go index 13cebcb9c0..ccde7416ac 100644 --- a/detector/vuls2/vuls2.go +++ b/detector/vuls2/vuls2.go @@ -31,6 +31,7 @@ import ( "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/constant" + "github.com/future-architect/vuls/logging" "github.com/future-architect/vuls/models" ) @@ -43,28 +44,14 @@ func Detect(r *models.ScanResult, vuls2Cnf config.Vuls2DictConf, noProgress bool return nil } - c := Config{ - Repository: func() string { - if vuls2Cnf.Repository != "" { - return vuls2Cnf.Repository - } - return DefaultGHCRRepository - }(), - Path: func() string { - if vuls2Cnf.Path != "" { - return vuls2Cnf.Path - } - return DefaultPath - }(), - SkipUpdate: vuls2Cnf.SkipUpdate, - Quiet: noProgress, + if vuls2Cnf.Repository == "" { + vuls2Cnf.Repository = DefaultGHCRRepository } - - if err := c.Refresh(); err != nil { - return xerrors.Errorf("Failed to refresh vuls2 db. err: %w", err) + if vuls2Cnf.Path == "" { + vuls2Cnf.Path = DefaultPath } - dbc, err := c.New() + dbc, err := newDBConnection(vuls2Cnf, noProgress) if err != nil { return xerrors.Errorf("Failed to get new db connection. err: %w", err) } @@ -87,6 +74,8 @@ func Detect(r *models.ScanResult, vuls2Cnf config.Vuls2DictConf, noProgress bool vulnInfos := postConvert(e, r.Family, vuls2Detected, r) r.ScannedCves = vulnInfos + logging.Log.Infof("%s: %d CVEs are detected with vuls2", r.FormatServerName(), len(vulnInfos)) + return nil } @@ -131,7 +120,7 @@ func preConvert(sr *models.ScanResult) scanTypes.ScanResult { } } -// ALmost copied from vuls2 pkg/detect/detect.go +// Almost copied from vuls2 pkg/detect/detect.go func detect(dbc db.DB, sr scanTypes.ScanResult) (detectTypes.DetectResult, error) { detected := make(map[dataTypes.RootID]detectTypes.VulnerabilityData) diff --git a/go.mod b/go.mod index aa8fd4e38c..e3b1799484 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.4.0 github.com/CycloneDX/cyclonedx-go v0.9.2 github.com/MaineK00n/vuls-data-update v0.0.0-20241208090537-1636de60e613 - github.com/MaineK00n/vuls2 v0.0.1-alpha.0.20241208144343-786d30c45984 + github.com/MaineK00n/vuls2 v0.0.1-alpha.0.20241210070054-eb2c0ee8d5a0 github.com/Ullaakut/nmap/v2 v2.2.2 github.com/aquasecurity/trivy v0.58.1 github.com/aquasecurity/trivy-db v0.0.0-20241209111357-8c398f13db0e @@ -34,7 +34,6 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/jesseduffield/gocui v0.3.0 github.com/k0kubun/pp v3.0.1+incompatible - github.com/klauspost/compress v1.17.11 github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f github.com/knqyf263/go-cpe v0.0.0-20230627041855-cb0794d06872 github.com/knqyf263/go-deb-version v0.0.0-20241115132648-6f4aee6ccd23 @@ -44,14 +43,12 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/nlopes/slack v0.6.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/opencontainers/image-spec v1.1.0 github.com/package-url/packageurl-go v0.1.3 github.com/parnurzeal/gorequest v0.3.0 github.com/pkg/errors v0.9.1 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d github.com/samber/lo v1.47.0 - github.com/schollz/progressbar/v3 v3.17.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 github.com/vulsio/go-cti v0.1.0 @@ -66,7 +63,6 @@ require ( golang.org/x/sync v0.10.0 golang.org/x/text v0.21.0 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da - oras.land/oras-go/v2 v2.5.0 ) require ( @@ -226,6 +222,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -268,6 +265,7 @@ require ( github.com/nsf/termbox-go v1.1.1 // indirect github.com/open-policy-agent/opa v0.70.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/owenrumney/squealer v1.2.5 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect @@ -288,6 +286,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/schollz/progressbar/v3 v3.17.1 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/skeema/knownhosts v1.3.0 // indirect @@ -368,6 +367,7 @@ require ( modernc.org/sqlite v1.34.1 // indirect mvdan.cc/sh/v3 v3.10.0 // indirect oras.land/oras-go v1.2.5 // indirect + oras.land/oras-go/v2 v2.5.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.17.2 // indirect sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect diff --git a/go.sum b/go.sum index 641dae7fef..d93df59081 100644 --- a/go.sum +++ b/go.sum @@ -239,8 +239,8 @@ github.com/GoogleCloudPlatform/docker-credential-gcr v2.0.5+incompatible h1:juIa github.com/GoogleCloudPlatform/docker-credential-gcr v2.0.5+incompatible/go.mod h1:BB1eHdMLYEFuFdBlRMb0N7YGVdM5s6Pt0njxgvfbGGs= github.com/MaineK00n/vuls-data-update v0.0.0-20241208090537-1636de60e613 h1:Vnnvrl8aYjq6Lymjj6bQIm+vHpGo1zPE7jZESke+zkg= github.com/MaineK00n/vuls-data-update v0.0.0-20241208090537-1636de60e613/go.mod h1:n5rQrltX0Fp2gKJj+nGE7V81ezSdRBxANuhmLtxkWo4= -github.com/MaineK00n/vuls2 v0.0.1-alpha.0.20241208144343-786d30c45984 h1:yrnzRu3K1S6/SqYwqelPQg5MXcn3CduqlLqlNmd3ryw= -github.com/MaineK00n/vuls2 v0.0.1-alpha.0.20241208144343-786d30c45984/go.mod h1:ZNpPkiaGaFwDqq0dhSjcOUWdVsiwZQhybLoBQJW5DDg= +github.com/MaineK00n/vuls2 v0.0.1-alpha.0.20241210070054-eb2c0ee8d5a0 h1:DW3n2COyJO6lv+aNEgvTXIi96+CGef7V0sxXDlHo2xk= +github.com/MaineK00n/vuls2 v0.0.1-alpha.0.20241210070054-eb2c0ee8d5a0/go.mod h1:ZNpPkiaGaFwDqq0dhSjcOUWdVsiwZQhybLoBQJW5DDg= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=