feat(cache): support Redis (#770)

* feat(config): add --cache-backend

* feat(operation): embed cache.Cache into operation.Cache

* feat(cache): support redis://

* test(integration): add redis test

* chore(README): add --cache-backend

* chore(mod): update

* chore: add disclaimer

README.md

@@ -55,6 +55,7 @@ A Simple and Comprehensive Vulnerability Scanner for Containers and other Artifa
     + [Specify exit code](#specify-exit-code)
     + [Ignore the specified vulnerabilities](#ignore-the-specified-vulnerabilities)
     + [Specify cache directory](#specify-cache-directory)
+    + [Specify cache backend](#specify-cache-backend)
     + [Clear caches](#clear-caches)
     + [Reset](#reset)
     + [Use lightweight DB](#use-lightweight-db)
@@ -1331,6 +1332,21 @@ Total: 0 (UNKNOWN: 0, LOW: 0, MEDIUM: 0, HIGH: 0, CRITICAL: 0)
 $ trivy --cache-dir /tmp/trivy/ image python:3.4-alpine3.9
 ```
 
+### Specify cache backend
+[EXPERIMENTAL] This feature might change without preserving backwards compatibility.
+
+Trivy supports local filesystem and Redis as the cache backend. This option is useful especially for client/server mode.
+
+Two options:
+- `fs` 
+  - the cache path can be specified by `--cache-dir`
+- `redis://`
+  - `redis://[HOST]:[PORT]`
+
+```
+$ trivy server --cache-backend redis://localhost:6379
+```
+
 ### Clear caches
 
 The `--clear-cache` option removes caches. This option is useful if the image which has the same tag is updated (such as when using `latest` tag).

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.3
 	github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7
 	github.com/docker/go-connections v0.4.0
+	github.com/go-redis/redis/v8 v8.4.0
 	github.com/golang/protobuf v1.4.2
 	github.com/google/go-containerregistry v0.0.0-20200331213917-3d03ed9b1ca2
 	github.com/google/go-github/v28 v28.1.1

go.sum

@@ -477,14 +477,12 @@ github.com/olekukonko/tablewriter v0.0.2-0.20190607075207-195002e6e56a/go.mod h1
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
 github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=

integration/client_server_test.go

@@ -12,8 +12,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/go-connections/nat"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	testcontainers "github.com/testcontainers/testcontainers-go"
 	"github.com/urfave/cli/v2"
 
 	"github.com/aquasecurity/trivy/internal"
@@ -332,7 +334,7 @@ func TestClientServer(t *testing.T) {
 		},
 	}
 
-	app, addr, cacheDir := setup(t, "", "")
+	app, addr, cacheDir := setup(t, setupOptions{})
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -394,7 +396,10 @@ func TestClientServerWithToken(t *testing.T) {
 
 	serverToken := "token"
 	serverTokenHeader := "Trivy-Token"
-	app, addr, cacheDir := setup(t, serverToken, serverTokenHeader)
+	app, addr, cacheDir := setup(t, setupOptions{
+		token:       serverToken,
+		tokenHeader: serverTokenHeader,
+	})
 	defer os.RemoveAll(cacheDir)
 
 	for _, c := range cases {
@@ -418,7 +423,54 @@ func TestClientServerWithToken(t *testing.T) {
 	}
 }
 
-func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) {
+func TestClientServerWithRedis(t *testing.T) {
+	// Set up a Redis container
+	ctx := context.Background()
+	redisC, addr := setupRedis(t, ctx)
+
+	// Set up Trivy server
+	app, addr, cacheDir := setup(t, setupOptions{cacheBackend: addr})
+	defer os.RemoveAll(cacheDir)
+
+	// Test parameters
+	testArgs := args{
+		Version: "dev",
+		Input:   "testdata/fixtures/centos-7.tar.gz",
+	}
+	golden := "testdata/centos-7.json.golden"
+
+	t.Run("centos 7", func(t *testing.T) {
+		osArgs, outputFile, cleanup := setupClient(t, testArgs, addr, cacheDir, golden)
+		defer cleanup()
+
+		// Run Trivy client
+		err := app.Run(osArgs)
+		require.NoError(t, err)
+
+		compare(t, golden, outputFile)
+	})
+
+	// Terminate the Redis container
+	require.NoError(t, redisC.Terminate(ctx))
+
+	t.Run("sad path", func(t *testing.T) {
+		osArgs, _, cleanup := setupClient(t, testArgs, addr, cacheDir, golden)
+		defer cleanup()
+
+		// Run Trivy client
+		err := app.Run(osArgs)
+		require.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect: connection refused")
+	})
+}
+
+type setupOptions struct {
+	token        string
+	tokenHeader  string
+	cacheBackend string
+}
+
+func setup(t *testing.T, options setupOptions) (*cli.App, string, string) {
 	t.Helper()
 	version := "dev"
 
@@ -434,7 +486,7 @@ func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) {
 		// Setup CLI App
 		app := internal.NewApp(version)
 		app.Writer = ioutil.Discard
-		osArgs := setupServer(addr, token, tokenHeader, cacheDir)
+		osArgs := setupServer(addr, options.token, options.tokenHeader, cacheDir, options.cacheBackend)
 
 		// Run Trivy server
 		app.Run(osArgs)
@@ -451,11 +503,14 @@ func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) {
 	return app, addr, cacheDir
 }
 
-func setupServer(addr, token, tokenHeader, cacheDir string) []string {
+func setupServer(addr, token, tokenHeader, cacheDir, cacheBackend string) []string {
 	osArgs := []string{"trivy", "server", "--skip-update", "--cache-dir", cacheDir, "--listen", addr}
 	if token != "" {
 		osArgs = append(osArgs, []string{"--token", token, "--token-header", tokenHeader}...)
 	}
+	if cacheBackend != "" {
+		osArgs = append(osArgs, "--cache-backend", cacheBackend)
+	}
 	return osArgs
 }
 
@@ -519,6 +574,32 @@ func setupClient(t *testing.T, c args, addr string, cacheDir string, golden stri
 	return osArgs, outputFile, cleanup
 }
 
+func setupRedis(t *testing.T, ctx context.Context) (testcontainers.Container, string) {
+	t.Helper()
+	imageName := "redis:5.0"
+	port := "6379/tcp"
+	req := testcontainers.ContainerRequest{
+		Name:         "redis",
+		Image:        imageName,
+		ExposedPorts: []string{port},
+	}
+
+	redis, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	require.NoError(t, err)
+
+	ip, err := redis.Host(ctx)
+	require.NoError(t, err)
+
+	p, err := redis.MappedPort(ctx, nat.Port(port))
+	require.NoError(t, err)
+
+	addr := fmt.Sprintf("redis://%s:%s", ip, p.Port())
+	return redis, addr
+}
+
 func compare(t *testing.T, wantFile, gotFile string) {
 	t.Helper()
 	// Compare want and got

internal/app.go

@@ -144,6 +144,13 @@ var (
 		EnvVars: []string{"TRIVY_CACHE_DIR"},
 	}
 
+	cacheBackendFlag = cli.StringFlag{
+		Name:    "cache-backend",
+		Value:   "fs",
+		Usage:   "cache backend (e.g. redis://localhost:6379)",
+		EnvVars: []string{"TRIVY_CACHE_BACKEND"},
+	}
+
 	ignoreFileFlag = cli.StringFlag{
 		Name:    "ignorefile",
 		Value:   vulnerability.DefaultIgnoreFile,
@@ -229,6 +236,7 @@ var (
 		&listAllPackages,
 		&skipFiles,
 		&skipDirectories,
+		&cacheBackendFlag,
 	}
 
 	// deprecated options
@@ -385,6 +393,7 @@ func NewFilesystemCommand() *cli.Command {
 			&vulnTypeFlag,
 			&ignoreFileFlag,
 			&cacheDirFlag,
+			&cacheBackendFlag,
 			&timeoutFlag,
 			&noProgressFlag,
 			&ignorePolicy,
@@ -419,6 +428,7 @@ func NewRepositoryCommand() *cli.Command {
 			&vulnTypeFlag,
 			&ignoreFileFlag,
 			&cacheDirFlag,
+			&cacheBackendFlag,
 			&timeoutFlag,
 			&noProgressFlag,
 			&ignorePolicy,
@@ -487,6 +497,7 @@ func NewServerCommand() *cli.Command {
 			&quietFlag,
 			&debugFlag,
 			&cacheDirFlag,
+			&cacheBackendFlag,
 
 			// original flags
 			&token,

internal/artifact/config/config.go

@@ -14,6 +14,7 @@ type Config struct {
 	config.DBConfig
 	config.ImageConfig
 	config.ReportConfig
+	config.CacheConfig
 
 	// deprecated
 	onlyUpdate string
@@ -36,6 +37,7 @@ func New(c *cli.Context) (Config, error) {
 		DBConfig:       config.NewDBConfig(c),
 		ImageConfig:    config.NewImageConfig(c),
 		ReportConfig:   config.NewReportConfig(c),
+		CacheConfig:    config.NewCacheConfig(c),
 
 		onlyUpdate:  c.String("only-update"),
 		refresh:     c.Bool("refresh"),
@@ -45,13 +47,11 @@ func New(c *cli.Context) (Config, error) {
 
 // Init initializes the artifact config
 func (c *Config) Init(image bool) error {
-	if err := c.ReportConfig.Init(c.Logger); err != nil {
-		return err
-	}
 	if c.onlyUpdate != "" || c.refresh || c.autoRefresh {
 		c.Logger.Warn("--only-update, --refresh and --auto-refresh are unnecessary and ignored now. These commands will be removed in the next version.")
 	}
-	if err := c.DBConfig.Init(); err != nil {
+
+	if err := c.initPreScanConfigs(); err != nil {
 		return err
 	}
 
@@ -73,6 +73,19 @@ func (c *Config) Init(image bool) error {
 	return nil
 }
 
+func (c *Config) initPreScanConfigs() error {
+	if err := c.ReportConfig.Init(c.Logger); err != nil {
+		return err
+	}
+	if err := c.DBConfig.Init(); err != nil {
+		return err
+	}
+	if err := c.CacheConfig.Init(); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *Config) skipScan() bool {
 	if c.ClearCache || c.DownloadDBOnly || c.Reset {
 		return true

internal/artifact/run.go

@@ -32,20 +32,18 @@ func run(c config.Config, initializeScanner InitializeScanner) error {
 
 	// configure cache dir
 	utils.SetCacheDir(c.CacheDir)
-	cacheClient, err := cache.NewFSCache(c.CacheDir)
+	cache, err := operation.NewCache(c.CacheBackend)
 	if err != nil {
 		return xerrors.Errorf("unable to initialize the cache: %w", err)
 	}
-	defer cacheClient.Close()
-
-	cacheOperation := operation.NewCache(cacheClient)
+	defer cache.Close()
 	log.Logger.Debugf("cache dir:  %s", utils.CacheDir())
 
 	if c.Reset {
-		return cacheOperation.Reset()
+		return cache.Reset()
 	}
 	if c.ClearCache {
-		return cacheOperation.ClearImages()
+		return cache.ClearImages()
 	}
 
 	// download the database file
@@ -70,7 +68,7 @@ func run(c config.Config, initializeScanner InitializeScanner) error {
 
 	ctx, cancel := context.WithTimeout(context.Background(), c.Timeout)
 	defer cancel()
-	scanner, cleanup, err := initializeScanner(ctx, target, cacheClient, cacheClient, c.Timeout)
+	scanner, cleanup, err := initializeScanner(ctx, target, cache, cache, c.Timeout)
 	if err != nil {
 		return xerrors.Errorf("unable to initialize a scanner: %w", err)
 	}

internal/config/cache.go

@@ -0,0 +1,31 @@
+package config
+
+import (
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+// CacheConfig holds the config for cache
+type CacheConfig struct {
+	CacheBackend string
+}
+
+// NewCacheConfig returns an instance of CacheConfig
+func NewCacheConfig(c *cli.Context) CacheConfig {
+	return CacheConfig{
+		CacheBackend: c.String("cache-backend"),
+	}
+}
+
+// Init initialize the CacheConfig
+func (c *CacheConfig) Init() error {
+	// "redis://" or "fs" are allowed for now
+	// An empty value is also allowed for testability
+	if !strings.HasPrefix(c.CacheBackend, "redis://") &&
+		c.CacheBackend != "fs" && c.CacheBackend != "" {
+		return xerrors.Errorf("unsupported cache backend: %s", c.CacheBackend)
+	}
+	return nil
+}