feat(db): store metadata as a file (#464)

* refactor: wrap errors

* feat(db): add the metadata file

* test(db): re-generate mocks

* fix(app): read metadata from the file in showVersion

* fix: open the database after downloading it

* fix(operation): use UpdateMetadata

* chore(mod): update dependency

* test(integration): fix tests

* fix(conf): rename TRIVY_NONSSL to TRIVY_NON_SSL

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/kylelemons/godebug v1.1.0
 	github.com/mattn/go-colorable v0.1.4 // indirect
 	github.com/olekukonko/tablewriter v0.0.2-0.20190607075207-195002e6e56a
+	github.com/spf13/afero v1.2.2
 	github.com/stretchr/testify v1.4.0
 	github.com/twitchtv/twirp v5.10.1+incompatible
 	github.com/urfave/cli v1.22.1

integration/client_server_test.go

@@ -330,7 +330,8 @@ func TestClientServer(t *testing.T) {
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			// Copy DB file
-			cacheDir := gunzipDB()
+			cacheDir, err := gunzipDB()
+			require.NoError(t, err)
 			defer os.RemoveAll(cacheDir)
 
 			port, err := getFreePort()

integration/docker_engine_test.go

@@ -237,7 +237,8 @@ func TestRun_WithDockerEngine(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			// Copy DB file
-			cacheDir := gunzipDB()
+			cacheDir, err := gunzipDB()
+			require.NoError(t, err)
 			defer os.RemoveAll(cacheDir)
 
 			ctx := context.Background()

integration/integration_test.go

@@ -8,46 +8,62 @@ import (
 	"flag"
 	"io"
 	"io/ioutil"
-	"log"
 	"net"
 	"os"
 	"path/filepath"
 	"time"
+
+	dbFile "github.com/aquasecurity/trivy/pkg/db"
+
+	"github.com/aquasecurity/trivy-db/pkg/db"
+	"github.com/spf13/afero"
 )
 
 var update = flag.Bool("update", false, "update golden files")
 
-func gunzipDB() string {
+func gunzipDB() (string, error) {
 	gz, err := os.Open("testdata/trivy.db.gz")
 	if err != nil {
-		log.Panic(err)
+		return "", err
 	}
 	zr, err := gzip.NewReader(gz)
 	if err != nil {
-		log.Panic(err)
+		return "", err
 	}
 
 	tmpDir, err := ioutil.TempDir("", "integration")
 	if err != nil {
-		log.Panic(err)
+		return "", err
 	}
-	dbDir := filepath.Join(tmpDir, "db")
+
+	dbPath := db.Path(tmpDir)
+	dbDir := filepath.Dir(dbPath)
 	err = os.MkdirAll(dbDir, 0700)
 	if err != nil {
-		log.Panic(err)
+		return "", err
 	}
 
-	file, err := os.Create(filepath.Join(dbDir, "trivy.db"))
+	file, err := os.Create(dbPath)
 	if err != nil {
-		log.Panic(err)
+		return "", err
 	}
 	defer file.Close()
 
-	_, err = io.Copy(file, zr)
+	if _, err = io.Copy(file, zr); err != nil {
+		return "", err
+	}
+
+	err = dbFile.NewMetadata(afero.NewOsFs(), tmpDir).Store(db.Metadata{
+		Version:    1,
+		Type:       1,
+		NextUpdate: time.Time{},
+		UpdatedAt:  time.Time{},
+	})
 	if err != nil {
-		log.Panic(err)
+		return "", err
 	}
-	return tmpDir
+
+	return tmpDir, nil
 }
 
 func getFreePort() (int, error) {

integration/standalone_test.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/aquasecurity/trivy/internal"
 
 	"github.com/stretchr/testify/assert"
@@ -346,7 +348,8 @@ func TestRun_WithTar(t *testing.T) {
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			// Copy DB file
-			cacheDir := gunzipDB()
+			cacheDir, err := gunzipDB()
+			require.NoError(t, err)
 			defer os.RemoveAll(cacheDir)
 
 			// Setup CLI App

internal/app.go

@@ -7,14 +7,15 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aquasecurity/trivy-db/pkg/db"
-
+	"github.com/spf13/afero"
 	"github.com/urfave/cli"
 
+	"github.com/aquasecurity/trivy-db/pkg/db"
 	"github.com/aquasecurity/trivy-db/pkg/types"
 	"github.com/aquasecurity/trivy/internal/client"
 	"github.com/aquasecurity/trivy/internal/server"
 	"github.com/aquasecurity/trivy/internal/standalone"
+	tdb "github.com/aquasecurity/trivy/pkg/db"
 	"github.com/aquasecurity/trivy/pkg/utils"
 	"github.com/aquasecurity/trivy/pkg/vulnerability"
 )
@@ -249,16 +250,13 @@ OPTIONS:
 func showVersion(cacheDir, outputFormat, version string, outputWriter io.Writer) {
 	var dbMeta *db.Metadata
 
-	err := db.Init(cacheDir)
-	if err == nil {
-		metadata, _ := db.Config{}.GetMetadata()
-		if !metadata.UpdatedAt.IsZero() && !metadata.NextUpdate.IsZero() && metadata.Version != 0 {
-			dbMeta = &db.Metadata{
-				Version:    metadata.Version,
-				Type:       metadata.Type,
-				NextUpdate: metadata.NextUpdate.UTC(),
-				UpdatedAt:  metadata.UpdatedAt.UTC(),
-			}
+	metadata, _ := tdb.NewMetadata(afero.NewOsFs(), cacheDir).Get()
+	if !metadata.UpdatedAt.IsZero() && !metadata.NextUpdate.IsZero() && metadata.Version != 0 {
+		dbMeta = &db.Metadata{
+			Version:    metadata.Version,
+			Type:       metadata.Type,
+			NextUpdate: metadata.NextUpdate.UTC(),
+			UpdatedAt:  metadata.UpdatedAt.UTC(),
 		}
 	}
 

internal/app_test.go

@@ -1,25 +1,23 @@
 package internal
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
+	dbFile "github.com/aquasecurity/trivy/pkg/db"
+	"github.com/spf13/afero"
+
 	"github.com/aquasecurity/trivy-db/pkg/db"
 
 	"github.com/stretchr/testify/assert"
 )
 
-type fakeIOWriter struct {
-	written []byte
-}
-
-func (f *fakeIOWriter) Write(p []byte) (n int, err error) {
-	f.written = append(f.written, p...)
-	return len(p), nil
-}
-
 func Test_showVersion(t *testing.T) {
 	type args struct {
 		cacheDir     string
@@ -85,27 +83,26 @@ Vulnerability DB:
 				cacheDir = tt.args.cacheDir
 			default:
 				cacheDir, _ = ioutil.TempDir("", "Test_showVersion-*")
-				defer func() {
-					os.RemoveAll(cacheDir)
-				}()
+				defer os.RemoveAll(cacheDir)
 			}
 
 			if tt.createDB {
-				db.Init(cacheDir)
-				db.Config{}.SetMetadata(db.Metadata{
+				m := dbFile.NewMetadata(afero.NewOsFs(), cacheDir)
+				err := os.MkdirAll(filepath.Join(cacheDir, "db"), os.ModePerm)
+				require.NoError(t, err)
+
+				err = m.Store(db.Metadata{
 					Version:    42,
 					Type:       1,
 					NextUpdate: time.Unix(1584403020, 0),
 					UpdatedAt:  time.Unix(1584402020, 0),
 				})
-				db.Close()
+				require.NoError(t, err)
 			}
 
-			var wb []byte
-			fw := fakeIOWriter{written: wb}
-
-			showVersion(cacheDir, tt.args.outputFormat, tt.args.version, &fw)
-			assert.Equal(t, tt.expectedOutput, string(fw.written), tt.name)
+			fw := new(bytes.Buffer)
+			showVersion(cacheDir, tt.args.outputFormat, tt.args.version, fw)
+			assert.Equal(t, tt.expectedOutput, fw.String(), tt.name)
 		})
 	}
 }

internal/operation/inject.go

@@ -7,7 +7,7 @@ import (
 	"github.com/google/wire"
 )
 
-func initializeDBClient(quiet bool) db.Client {
+func initializeDBClient(cacheDir string, quiet bool) db.Client {
 	wire.Build(db.SuperSet)
 	return db.Client{}
 }

internal/operation/operation.go

@@ -4,11 +4,13 @@ import (
 	"context"
 	"os"
 
+	"github.com/spf13/afero"
+
 	"github.com/google/wire"
 	"golang.org/x/xerrors"
 
 	"github.com/aquasecurity/fanal/cache"
-	"github.com/aquasecurity/trivy-db/pkg/db"
+	"github.com/aquasecurity/trivy/pkg/db"
 	"github.com/aquasecurity/trivy/pkg/log"
 	"github.com/aquasecurity/trivy/pkg/utils"
 )
@@ -29,63 +31,59 @@ func NewCache(client cache.LocalImageCache) Cache {
 
 func (c Cache) Reset() (err error) {
 	if err := c.ClearDB(); err != nil {
-		return err
+		return xerrors.Errorf("failed to clear the database: %w", err)
 	}
 	if err := c.ClearImages(); err != nil {
-		return err
+		return xerrors.Errorf("failed to clear the image cache: %w", err)
 	}
 	return nil
 }
 
 func (c Cache) ClearDB() (err error) {
 	log.Logger.Info("Removing DB file...")
 	if err = os.RemoveAll(utils.CacheDir()); err != nil {
-		return xerrors.New("failed to remove cache")
+		return xerrors.Errorf("failed to remove the directory (%s) : %w", utils.CacheDir(), err)
 	}
 	return nil
 }
 
 func (c Cache) ClearImages() error {
 	log.Logger.Info("Removing image caches...")
 	if err := c.client.Clear(); err != nil {
-		return xerrors.New("failed to remove image layer cache")
+		return xerrors.Errorf("failed to remove the cache: %w", err)
 	}
 	return nil
 }
 
 func DownloadDB(appVersion, cacheDir string, quiet, light, skipUpdate bool) error {
-	client := initializeDBClient(quiet)
+	client := initializeDBClient(cacheDir, quiet)
 	ctx := context.Background()
-	needsUpdate, err := client.NeedsUpdate(ctx, appVersion, light, skipUpdate)
+	needsUpdate, err := client.NeedsUpdate(appVersion, light, skipUpdate)
 	if err != nil {
 		return xerrors.Errorf("database error: %w", err)
 	}
 
 	if needsUpdate {
 		log.Logger.Info("Need to update DB")
-		if err = db.Close(); err != nil {
-			return xerrors.Errorf("failed db close: %w", err)
-		}
 		log.Logger.Info("Downloading DB...")
 		if err := client.Download(ctx, cacheDir, light); err != nil {
 			return xerrors.Errorf("failed to download vulnerability DB: %w", err)
 		}
-
-		log.Logger.Info("Reopening DB...")
-		if err = db.Init(cacheDir); err != nil {
-			return xerrors.Errorf("failed db close: %w", err)
+		if err = client.UpdateMetadata(cacheDir); err != nil {
+			return xerrors.Errorf("unable to update database metadata: %w", err)
 		}
 	}
 
 	// for debug
-	if err := showDBInfo(); err != nil {
-		return xerrors.Errorf("failed to show database info")
+	if err := showDBInfo(cacheDir); err != nil {
+		return xerrors.Errorf("failed to show database info: %w", err)
 	}
 	return nil
 }
 
-func showDBInfo() error {
-	metadata, err := db.Config{}.GetMetadata()
+func showDBInfo(cacheDir string) error {
+	m := db.NewMetadata(afero.NewOsFs(), cacheDir)
+	metadata, err := m.Get()
 	if err != nil {
 		return xerrors.Errorf("something wrong with DB: %w", err)
 	}

internal/server/run.go

@@ -1,10 +1,10 @@
 package server
 
 import (
-	"github.com/aquasecurity/fanal/cache"
 	"github.com/urfave/cli"
 	"golang.org/x/xerrors"
 
+	"github.com/aquasecurity/fanal/cache"
 	"github.com/aquasecurity/trivy-db/pkg/db"
 	"github.com/aquasecurity/trivy/internal/operation"
 	"github.com/aquasecurity/trivy/internal/server/config"
@@ -42,10 +42,6 @@ func run(c config.Config) (err error) {
 		return cacheOperation.ClearDB()
 	}
 
-	if err = db.Init(c.CacheDir); err != nil {
-		return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
-	}
-
 	// download the database file
 	if err = operation.DownloadDB(c.AppVersion, c.CacheDir, true, false, c.SkipUpdate); err != nil {
 		return err
@@ -55,5 +51,9 @@ func run(c config.Config) (err error) {
 		return nil
 	}
 
+	if err = db.Init(c.CacheDir); err != nil {
+		return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
+	}
+
 	return server.ListenAndServe(c, fsCache)
 }