Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: pass DB dir to trivy-db #7057

Merged
merged 2 commits into from
Jul 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ require (
github.com/aquasecurity/testdocker v0.0.0-20240613070307-2c3868d658ac
github.com/aquasecurity/tml v0.6.1
github.com/aquasecurity/trivy-checks v0.13.0
github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d
github.com/aquasecurity/trivy-db v0.0.0-20240701103400-8e907467e9ab
github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48
github.com/aquasecurity/trivy-kubernetes v0.6.7-0.20240627095026-cf9d48837f6d
github.com/aws/aws-sdk-go-v2 v1.27.2
Expand Down Expand Up @@ -192,7 +192,7 @@ require (
github.com/containerd/ttrpc v1.2.4 // indirect
github.com/containerd/typeurl/v2 v2.1.1 // indirect
github.com/cpuguy83/dockercfg v0.3.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
Expand Down
7 changes: 4 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -771,8 +771,8 @@ github.com/aquasecurity/tml v0.6.1 h1:y2ZlGSfrhnn7t4ZJ/0rotuH+v5Jgv6BDDO5jB6A9gw
github.com/aquasecurity/tml v0.6.1/go.mod h1:OnYMWY5lvI9ejU7yH9LCberWaaTBW7hBFsITiIMY2yY=
github.com/aquasecurity/trivy-checks v0.13.0 h1:na6PTdY4U0uK/fjz3HNRYBxvYSJ8vgTb57a5T8Y5t9w=
github.com/aquasecurity/trivy-checks v0.13.0/go.mod h1:Xec/SMVGV66I7RgUqOX9MEr+YxBqHXDVLTYmpspPi3E=
github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d h1:fjI9mkoTUAkbGqpzt9nJsO24RAdfG+ZSiLFj0G2jO8c=
github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d/go.mod h1:cj9/QmD9N3OZnKQMp+/DvdV+ym3HyIkd4e+F0ZM3ZGs=
github.com/aquasecurity/trivy-db v0.0.0-20240701103400-8e907467e9ab h1:EmpLGFgRJOstPWDpL4KW+Xap4zRYxyctXDTj5luMQdE=
github.com/aquasecurity/trivy-db v0.0.0-20240701103400-8e907467e9ab/go.mod h1:f+wSW9D5txv8S+tw4D4WNOibaUJYwvNnQuQlGQ8gO6c=
github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48 h1:JVgBIuIYbwG+ekC5lUHUpGJboPYiCcxiz06RCtz8neI=
github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48/go.mod h1:Ldya37FLi0e/5Cjq2T5Bty7cFkzUDwTcPeQua+2M8i8=
github.com/aquasecurity/trivy-kubernetes v0.6.7-0.20240627095026-cf9d48837f6d h1:z5Ug+gqNjgHzCo7rmv6wKTmyJ8E3bAVEU2AASo3740s=
Expand Down Expand Up @@ -1019,8 +1019,9 @@ github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
Expand Down
13 changes: 3 additions & 10 deletions integration/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,11 @@ import (
"github.com/stretchr/testify/require"
"github.com/xeipuuv/gojsonschema"

"github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy-db/pkg/metadata"

"github.com/aquasecurity/trivy/internal/dbtest"
"github.com/aquasecurity/trivy/pkg/clock"
"github.com/aquasecurity/trivy/pkg/commands"
"github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/types"
"github.com/aquasecurity/trivy/pkg/uuid"

Expand All @@ -56,15 +55,9 @@ func initDB(t *testing.T) string {
}

cacheDir := dbtest.InitDB(t, fixtures)
defer db.Close()

dbDir := filepath.Dir(db.Path(cacheDir))

metadataFile := filepath.Join(dbDir, "metadata.json")
f, err := os.Create(metadataFile)
require.NoError(t, err)
defer dbtest.Close()

err = json.NewEncoder(f).Encode(metadata.Metadata{
err = metadata.NewClient(db.Dir(cacheDir)).Update(metadata.Metadata{
Version: db.SchemaVersion,
NextUpdate: time.Now().Add(24 * time.Hour),
UpdatedAt: time.Now(),
Expand Down
13 changes: 7 additions & 6 deletions internal/dbtest/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,18 @@ import (
"github.com/stretchr/testify/require"

fixtures "github.com/aquasecurity/bolt-fixtures"
"github.com/aquasecurity/trivy-db/pkg/db"
trivydb "github.com/aquasecurity/trivy-db/pkg/db"
jdb "github.com/aquasecurity/trivy-java-db/pkg/db"
"github.com/aquasecurity/trivy/pkg/db"
)

// InitDB initializes testing database.
func InitDB(t *testing.T, fixtureFiles []string) string {
// Create a temp dir
dir := t.TempDir()
cacheDir := t.TempDir()

dbPath := db.Path(dir)
dbDir := filepath.Dir(dbPath)
dbDir := db.Dir(cacheDir)
dbPath := trivydb.Path(dbDir)
err := os.MkdirAll(dbDir, 0700)
require.NoError(t, err)

Expand All @@ -30,9 +31,9 @@ func InitDB(t *testing.T, fixtureFiles []string) string {
require.NoError(t, loader.Close())

// Initialize DB
require.NoError(t, db.Init(dir))
require.NoError(t, db.Init(dbDir))

return dir
return cacheDir
}

func Close() error {
Expand Down
4 changes: 2 additions & 2 deletions pkg/commands/artifact/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ import (
"github.com/spf13/viper"
"golang.org/x/xerrors"

"github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy/pkg/cache"
"github.com/aquasecurity/trivy/pkg/commands/operation"
"github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
"github.com/aquasecurity/trivy/pkg/fanal/artifact"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
Expand Down Expand Up @@ -295,7 +295,7 @@ func (r *runner) initDB(ctx context.Context, opts flag.Options) error {
return SkipScan
}

if err := db.Init(opts.CacheDir); err != nil {
if err := db.Init(db.Dir(opts.CacheDir)); err != nil {
return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
}
r.dbOpen = true
Expand Down
2 changes: 1 addition & 1 deletion pkg/commands/clean/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func cleanScanCache(ctx context.Context, opts flag.Options) error {

func cleanVulnerabilityDB(ctx context.Context, opts flag.Options) error {
log.InfoContext(ctx, "Removing vulnerability database...")
if err := db.NewClient(opts.CacheDir, true).Clear(ctx); err != nil {
if err := db.NewClient(db.Dir(opts.CacheDir), true).Clear(ctx); err != nil {
return xerrors.Errorf("clear vulnerability database: %w", err)

}
Expand Down
19 changes: 4 additions & 15 deletions pkg/commands/operation/operation.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"github.com/google/go-containerregistry/pkg/name"
"golang.org/x/xerrors"

"github.com/aquasecurity/trivy-db/pkg/metadata"
"github.com/aquasecurity/trivy/pkg/db"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/flag"
Expand All @@ -24,7 +23,8 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n
mu.Lock()
defer mu.Unlock()

client := db.NewClient(cacheDir, quiet, db.WithDBRepository(dbRepository))
dbDir := db.Dir(cacheDir)
client := db.NewClient(dbDir, quiet, db.WithDBRepository(dbRepository))
needsUpdate, err := client.NeedsUpdate(ctx, appVersion, skipUpdate)
if err != nil {
return xerrors.Errorf("database error: %w", err)
Expand All @@ -33,29 +33,18 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n
if needsUpdate {
log.Info("Need to update DB")
log.Info("Downloading DB...", log.String("repository", dbRepository.String()))
if err = client.Download(ctx, cacheDir, opt); err != nil {
if err = client.Download(ctx, dbDir, opt); err != nil {
return xerrors.Errorf("failed to download vulnerability DB: %w", err)
}
}

// for debug
if err = showDBInfo(cacheDir); err != nil {
if err = client.ShowInfo(); err != nil {
return xerrors.Errorf("failed to show database info: %w", err)
}
return nil
}

func showDBInfo(cacheDir string) error {
m := metadata.NewClient(cacheDir)
meta, err := m.Get()
if err != nil {
return xerrors.Errorf("something wrong with DB: %w", err)
}
log.Debug("DB info", log.Int("schema", meta.Version), log.Time("updated_at", meta.UpdatedAt),
log.Time("next_update", meta.NextUpdate), log.Time("downloaded_at", meta.DownloadedAt))
return nil
}

// InitBuiltinPolicies downloads the built-in policies and loads them
func InitBuiltinPolicies(ctx context.Context, cacheDir string, quiet, skipUpdate bool, checkBundleRepository string, registryOpts ftypes.RegistryOptions) ([]string, error) {
mu.Lock()
Expand Down
4 changes: 2 additions & 2 deletions pkg/commands/server/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ import (

"golang.org/x/xerrors"

"github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy/pkg/cache"
"github.com/aquasecurity/trivy/pkg/commands/operation"
"github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/flag"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/module"
Expand Down Expand Up @@ -35,7 +35,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) {
return nil
}

if err = db.Init(opts.CacheDir); err != nil {
if err = db.Init(db.Dir(opts.CacheDir)); err != nil {
return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
}

Expand Down
37 changes: 28 additions & 9 deletions pkg/db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"time"

"github.com/google/go-containerregistry/pkg/name"
Expand All @@ -28,6 +29,10 @@ const (
var (
DefaultRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-db", db.SchemaVersion)
defaultRepository, _ = name.NewTag(DefaultRepository)

Init = db.Init
Close = db.Close
Path = db.Path
)

type options struct {
Expand Down Expand Up @@ -56,13 +61,17 @@ func WithDBRepository(dbRepository name.Reference) Option {
type Client struct {
*options

cacheDir string
dbDir string
metadata metadata.Client
quiet bool
}

func Dir(cacheDir string) string {
return filepath.Join(cacheDir, "db")
}

// NewClient is the factory method for DB client
func NewClient(cacheDir string, quiet bool, opts ...Option) *Client {
func NewClient(dbDir string, quiet bool, opts ...Option) *Client {
o := &options{
dbRepository: defaultRepository,
}
Expand All @@ -73,8 +82,8 @@ func NewClient(cacheDir string, quiet bool, opts ...Option) *Client {

return &Client{
options: o,
cacheDir: cacheDir,
metadata: metadata.NewClient(cacheDir),
dbDir: dbDir,
metadata: metadata.NewClient(dbDir),
quiet: quiet,
}
}
Expand Down Expand Up @@ -149,7 +158,7 @@ func (c *Client) Download(ctx context.Context, dst string, opt types.RegistryOpt
return xerrors.Errorf("OCI artifact error: %w", err)
}

if err = art.Download(ctx, db.Dir(dst), oci.DownloadOption{MediaType: dbMediaType}); err != nil {
if err = art.Download(ctx, dst, oci.DownloadOption{MediaType: dbMediaType}); err != nil {
return xerrors.Errorf("database download error: %w", err)
}

Expand All @@ -159,19 +168,19 @@ func (c *Client) Download(ctx context.Context, dst string, opt types.RegistryOpt
return nil
}

func (c *Client) Clear(ctx context.Context) error {
if err := os.RemoveAll(db.Dir(c.cacheDir)); err != nil {
func (c *Client) Clear(_ context.Context) error {
if err := os.RemoveAll(c.dbDir); err != nil {
return xerrors.Errorf("failed to remove vulnerability database: %w", err)
}
return nil
}

func (c *Client) updateDownloadedAt(ctx context.Context, dst string) error {
func (c *Client) updateDownloadedAt(ctx context.Context, dbDir string) error {
log.Debug("Updating database metadata...")

// We have to initialize a metadata client here
// since the destination may be different from the cache directory.
client := metadata.NewClient(dst)
client := metadata.NewClient(dbDir)
meta, err := client.Get()
if err != nil {
return xerrors.Errorf("unable to get metadata: %w", err)
Expand Down Expand Up @@ -207,3 +216,13 @@ func (c *Client) initOCIArtifact(opt types.RegistryOptions) (*oci.Artifact, erro
}
return art, nil
}

func (c *Client) ShowInfo() error {
meta, err := c.metadata.Get()
if err != nil {
return xerrors.Errorf("something wrong with DB: %w", err)
}
log.Debug("DB info", log.Int("schema", meta.Version), log.Time("updated_at", meta.UpdatedAt),
log.Time("next_update", meta.NextUpdate), log.Time("downloaded_at", meta.DownloadedAt))
return nil
}
Loading