From 547363a7c0f8e09f19396a863a4445b3b57a7c6f Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 3 Jul 2024 15:04:41 +0400 Subject: [PATCH 01/55] feat: add vex subcommand Signed-off-by: knqyf263 --- pkg/commands/app.go | 65 ++++++++++++ pkg/downloader/download.go | 6 +- pkg/plugin/manager.go | 4 +- pkg/plugin/plugin.go | 2 +- pkg/utils/fsutils/fs.go | 4 + pkg/vex/repo/repo.go | 200 +++++++++++++++++++++++++++++++++++++ 6 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 pkg/vex/repo/repo.go diff --git a/pkg/commands/app.go b/pkg/commands/app.go index 5ca651193c35..45b16b6376e9 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -27,6 +27,7 @@ import ( "github.com/aquasecurity/trivy/pkg/types" "github.com/aquasecurity/trivy/pkg/version" "github.com/aquasecurity/trivy/pkg/version/app" + vexrepo "github.com/aquasecurity/trivy/pkg/vex/repo" xstrings "github.com/aquasecurity/trivy/pkg/x/strings" ) @@ -98,6 +99,7 @@ func NewApp() *cobra.Command { NewVersionCommand(globalFlags), NewVMCommand(globalFlags), NewCleanCommand(globalFlags), + NewVEXCommand(globalFlags), ) if plugins := loadPluginCommands(); len(plugins) > 0 { @@ -1226,6 +1228,69 @@ func NewCleanCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { return cmd } +func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { + vexFlags := &flag.Flags{ + GlobalFlagGroup: globalFlags, + } + var vexOptions flag.Options + cmd := &cobra.Command{ + Use: "vex subcommand", + Aliases: []string{"p"}, + GroupID: groupManagement, + Short: "VEX utilities", + SilenceErrors: true, + SilenceUsage: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { + vexOptions, err = vexFlags.ToOptions(args) + if err != nil { + return err + } + return nil + }, + } + + repoCmd := &cobra.Command{ + Use: "repo subcommand", + Short: "Manage VEX repositories", + SilenceErrors: true, + SilenceUsage: true, + } + + repoCmd.AddCommand( + &cobra.Command{ + Use: "init", + Short: "Initialize a configuration file", + SilenceErrors: true, + SilenceUsage: true, + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + if err := vexrepo.NewManager().Init(cmd.Context()); err != nil { + return xerrors.Errorf("config init error: %w", err) + } + return nil + }, + }, + &cobra.Command{ + Use: "update [REPO_NAMES]", + Short: "Update the local copy of the VEX repositories", + DisableFlagsInUseLine: true, + SilenceErrors: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if err := vexrepo.NewManager().Update(cmd.Context(), args, vexrepo.Options{Insecure: vexOptions.Insecure}); err != nil { + return xerrors.Errorf("repository update error: %w", err) + } + return nil + }, + }, + ) + + cmd.AddCommand(repoCmd) + + return cmd +} + func NewVersionCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { var versionFormat string cmd := &cobra.Command{ diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index 7190d3d3d0a3..8c4ff685dfa6 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -7,6 +7,8 @@ import ( getter "github.com/hashicorp/go-getter" "golang.org/x/xerrors" + + "github.com/aquasecurity/trivy/pkg/utils/fsutils" ) // DownloadToTempDir downloads the configured source to a temp dir. @@ -31,7 +33,9 @@ func DownloadToTempDir(ctx context.Context, url string, insecure bool) (string, // Download downloads the configured source to the destination. func Download(ctx context.Context, src, dst, pwd string, insecure bool) error { // go-getter doesn't allow the dst directory already exists if the src is directory. - _ = os.RemoveAll(dst) + if fsutils.DirExists(src) { + _ = os.RemoveAll(dst) + } var opts []getter.ClientOption if insecure { diff --git a/pkg/plugin/manager.go b/pkg/plugin/manager.go index c0f9bf431c87..c0e5cadb5c7e 100644 --- a/pkg/plugin/manager.go +++ b/pkg/plugin/manager.go @@ -23,7 +23,7 @@ import ( const configFile = "plugin.yaml" var ( - pluginsRelativeDir = filepath.Join(".trivy", "plugins") + pluginsDir = "plugins" _defaultManager *Manager ) @@ -58,7 +58,7 @@ type Manager struct { } func NewManager(opts ...ManagerOption) *Manager { - root := filepath.Join(fsutils.HomeDir(), pluginsRelativeDir) + root := filepath.Join(fsutils.TrivyHomeDir(), pluginsDir) m := &Manager{ w: os.Stdout, indexURL: indexURL, diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 56c33644f854..498f8d19c112 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -165,5 +165,5 @@ func (p *Plugin) Dir() string { if p.dir != "" { return p.dir } - return filepath.Join(fsutils.HomeDir(), pluginsRelativeDir, p.Name) + return filepath.Join(fsutils.TrivyHomeDir(), p.Name) } diff --git a/pkg/utils/fsutils/fs.go b/pkg/utils/fsutils/fs.go index c15302deed1d..e0518236f9a8 100644 --- a/pkg/utils/fsutils/fs.go +++ b/pkg/utils/fsutils/fs.go @@ -28,6 +28,10 @@ func HomeDir() string { return homeDir } +func TrivyHomeDir() string { + return filepath.Join(HomeDir(), ".trivy") +} + // CopyFile copies the file content from scr to dst func CopyFile(src, dst string) (int64, error) { sourceFileStat, err := os.Stat(src) diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go new file mode 100644 index 000000000000..ad2be83195fc --- /dev/null +++ b/pkg/vex/repo/repo.go @@ -0,0 +1,200 @@ +package repo + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "slices" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "golang.org/x/xerrors" + "gopkg.in/yaml.v3" + + "github.com/aquasecurity/trivy/pkg/downloader" + "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/utils/fsutils" +) + +// const defaultVEXHubURL = "git@github.com:aquasecurity/vexhub.git" +const defaultVEXHubURL = "https://github.com/aquasecurity/vuln-list-update.git" + +type ManagerOption func(indexer *Manager) + +func WithWriter(w io.Writer) ManagerOption { + return func(manager *Manager) { + manager.w = w + } +} + +func WithLogger(logger *log.Logger) ManagerOption { + return func(manager *Manager) { + manager.logger = logger + } +} + +type Config struct { + Repositories []Repository `json:"repositories"` +} + +type Repository struct { + Name string + URL string + DB string `yaml:",omitempty"` // TODO: support pre-built DB +} + +type Options struct { + Insecure bool +} + +// Manager manages the plugins +type Manager struct { + w io.Writer + indexURL string + logger *log.Logger + configFile string + repoDir string +} + +func NewManager(opts ...ManagerOption) *Manager { + root := filepath.Join(fsutils.TrivyHomeDir(), "vex") + m := &Manager{ + w: os.Stdout, + logger: log.WithPrefix("vex"), + configFile: filepath.Join(root, "config.yaml"), + repoDir: filepath.Join(root, "repositories"), + } + for _, opt := range opts { + opt(m) + } + + return m +} + +func (m *Manager) writeConfig(conf Config) error { + if err := os.MkdirAll(filepath.Dir(m.configFile), 0700); err != nil { + return xerrors.Errorf("failed to mkdir: %w", err) + } + f, err := os.Create(m.configFile) + if err != nil { + return xerrors.Errorf("failed to create a file: %w", err) + } + defer f.Close() + + e := yaml.NewEncoder(f) + e.SetIndent(2) + if err = e.Encode(conf); err != nil { + return xerrors.Errorf("JSON encode error: %w", err) + } + + return nil +} + +func (m *Manager) readConfig() (Config, error) { + if !fsutils.FileExists(m.configFile) { + return Config{}, xerrors.Errorf("config file not found, run 'trivy vex repo init' first") + } + + f, err := os.Open(m.configFile) + if err != nil { + return Config{}, xerrors.Errorf("unable to open a file: %w", err) + } + defer f.Close() + + var conf Config + if err = yaml.NewDecoder(f).Decode(&conf); err != nil { + return conf, xerrors.Errorf("unable to decode metadata: %w", err) + } + return conf, nil +} + +func (m *Manager) Init(ctx context.Context) error { + if fsutils.FileExists(m.configFile) { + m.logger.InfoContext(ctx, "The configuration file already exists", log.String("path", m.configFile)) + return nil + } + + err := m.writeConfig(Config{ + Repositories: []Repository{ + { + Name: "default", + URL: defaultVEXHubURL, + }, + }, + }) + if err != nil { + return xerrors.Errorf("failed to write the default config: %w", err) + } + log.InfoContext(ctx, "The default configuration file has been created", log.FilePath(m.configFile)) + return nil +} + +func (m *Manager) Update(ctx context.Context, names []string, opts Options) error { + conf, err := m.readConfig() + if err != nil { + return xerrors.Errorf("unable to read config: %w", err) + } else if len(conf.Repositories) == 0 { + return xerrors.Errorf("no repositories found in config: %s", m.configFile) + } + + for _, repo := range conf.Repositories { + if len(names) > 0 && !slices.Contains(names, repo.Name) { + continue + } + m.logger.InfoContext(ctx, "Updating the repository...", log.String("name", repo.Name), log.String("url", repo.URL)) + if err = m.download(ctx, repo, opts); err != nil { + return xerrors.Errorf("failed to update the repository: %w", err) + } + } + return nil +} + +func (m *Manager) download(ctx context.Context, repo Repository, opts Options) error { + // Force git protocol + // cf. https://github.com/hashicorp/go-getter/blob/5a63fd9c0d5b8da8a6805e8c283f46f0dacb30b3/README.md#forced-protocol + url := "git::" + repo.URL + "?depth=1" + + dst := filepath.Join(m.repoDir, repo.Name) + if fsutils.DirExists(dst) { + defaultBranch, err := findDefaultBranch(dst) + if err != nil { + m.logger.DebugContext(ctx, "failed to find the default branch", log.String("path", dst), log.Err(err)) + defaultBranch = "main" + } + url += "&ref=" + defaultBranch + } + + m.logger.DebugContext(ctx, "Downloading the repository...", log.String("url", url), log.String("dst", dst)) + if err := downloader.Download(ctx, url, dst, ".", opts.Insecure); err != nil { + return xerrors.Errorf("failed to download the repository: %w", err) + } + return nil +} + +func findDefaultBranch(repoPath string) (string, error) { + repo, err := git.PlainOpen(repoPath) + if err != nil { + return "", err + } + + remote, err := repo.Remote("origin") + if err != nil { + return "", err + } + + refs, err := remote.List(&git.ListOptions{}) + if err != nil { + return "", err + } + + for _, ref := range refs { + if ref.Name() == "HEAD" { + if ref.Type() == plumbing.SymbolicReference { + return ref.Target().Short(), nil + } + } + } + return "", fmt.Errorf("HEAD reference not found") +} From 6ab7fc8858570b3eacd8628211c7299064731722 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 4 Jul 2024 21:30:10 +0400 Subject: [PATCH 02/55] feat: adhere to VEX Repository Specification Signed-off-by: knqyf263 --- go.mod | 2 + go.sum | 4 + pkg/commands/app.go | 9 +- pkg/downloader/download.go | 14 +- pkg/vex/repo/manager.go | 140 +++++++++++++++++ pkg/vex/repo/repo.go | 303 ++++++++++++++++++++++--------------- 6 files changed, 347 insertions(+), 125 deletions(-) create mode 100644 pkg/vex/repo/manager.go diff --git a/go.mod b/go.mod index 98dab8d4fff2..4de6baa4c836 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( github.com/go-redis/redis/v8 v8.11.5 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/go-containerregistry v0.20.0 + github.com/google/go-github/v62 v62.0.0 github.com/google/licenseclassifier/v2 v2.0.0 github.com/google/uuid v1.6.0 github.com/google/wire v0.6.0 @@ -243,6 +244,7 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect diff --git a/go.sum b/go.sum index 114f8585b370..fa33982b4c1c 100644 --- a/go.sum +++ b/go.sum @@ -1357,6 +1357,10 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/google/go-containerregistry v0.20.0 h1:wRqHpOeVh3DnenOrPy9xDOLdnLatiGuuNRVelR2gSbg= github.com/google/go-containerregistry v0.20.0/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= +github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= diff --git a/pkg/commands/app.go b/pkg/commands/app.go index 45b16b6376e9..78cdbd6539fc 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -1241,6 +1241,8 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { SilenceErrors: true, SilenceUsage: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { + cmd.SetContext(log.WithContextPrefix(cmd.Context(), "vex")) + vexOptions, err = vexFlags.ToOptions(args) if err != nil { return err @@ -1273,13 +1275,13 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { }, &cobra.Command{ Use: "update [REPO_NAMES]", - Short: "Update the local copy of the VEX repositories", + Short: "Update the local copy of the VEX repository manifests", DisableFlagsInUseLine: true, SilenceErrors: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - if err := vexrepo.NewManager().Update(cmd.Context(), args, vexrepo.Options{Insecure: vexOptions.Insecure}); err != nil { - return xerrors.Errorf("repository update error: %w", err) + if err := vexrepo.NewManager().UpdateManifest(cmd.Context(), args, vexrepo.Options{Insecure: vexOptions.Insecure}); err != nil { + return xerrors.Errorf("repository manifest update error: %w", err) } return nil }, @@ -1287,7 +1289,6 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { ) cmd.AddCommand(repoCmd) - return cmd } diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index 8c4ff685dfa6..e4451fb21ad6 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -3,8 +3,10 @@ package downloader import ( "context" "maps" + "net/url" "os" + "github.com/google/go-github/v62/github" getter "github.com/hashicorp/go-getter" "golang.org/x/xerrors" @@ -51,8 +53,18 @@ func Download(ctx context.Context, src, dst, pwd string, insecure bool) error { // Since "httpGetter" is a global pointer and the state is shared, // once it is executed without "WithInsecure()", // it cannot enable WithInsecure() afterwards because its state is preserved. + // Therefore, we need to create a new "HttpGetter" instance every time. // cf. https://github.com/hashicorp/go-getter/blob/5a63fd9c0d5b8da8a6805e8c283f46f0dacb30b3/get.go#L63-L65 - httpGetter := &getter.HttpGetter{Netrc: true} + httpGetter := &getter.HttpGetter{ + Netrc: true, + } + if u, err := url.Parse(src); err == nil && u.Host == "github.com" { + client := github.NewClient(nil) + if t := os.Getenv("GITHUB_TOKEN"); t != "" { + client.WithAuthToken(t) + } + httpGetter.Client = client.Client() + } getters["http"] = httpGetter getters["https"] = httpGetter diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go new file mode 100644 index 000000000000..278b2645dffe --- /dev/null +++ b/pkg/vex/repo/manager.go @@ -0,0 +1,140 @@ +package repo + +import ( + "context" + "io" + "os" + "path/filepath" + "slices" + + "golang.org/x/xerrors" + "gopkg.in/yaml.v3" + + "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/utils/fsutils" +) + +const ( + // const defaultVEXHubURL = "git@github.com:aquasecurity/vexhub.git" + defaultVEXHubURL = "https://github.com/aquasecurity/vuln-list-update.git" +) + +type ManagerOption func(indexer *Manager) + +func WithWriter(w io.Writer) ManagerOption { + return func(manager *Manager) { + manager.w = w + } +} + +type Config struct { + Repositories []Repository `json:"repositories"` +} + +type Options struct { + Insecure bool +} + +// Manager manages the plugins +type Manager struct { + w io.Writer + indexURL string + configFile string + repoDir string +} + +func NewManager(opts ...ManagerOption) *Manager { + root := filepath.Join(fsutils.TrivyHomeDir(), "vex") + m := &Manager{ + w: os.Stdout, + configFile: filepath.Join(root, "config.yaml"), + repoDir: filepath.Join(root, "repositories"), + } + for _, opt := range opts { + opt(m) + } + + return m +} + +func (m *Manager) writeConfig(conf Config) error { + if err := os.MkdirAll(filepath.Dir(m.configFile), 0700); err != nil { + return xerrors.Errorf("failed to mkdir: %w", err) + } + f, err := os.Create(m.configFile) + if err != nil { + return xerrors.Errorf("failed to create a file: %w", err) + } + defer f.Close() + + e := yaml.NewEncoder(f) + e.SetIndent(2) + if err = e.Encode(conf); err != nil { + return xerrors.Errorf("JSON encode error: %w", err) + } + + return nil +} + +func (m *Manager) Config() (Config, error) { + if !fsutils.FileExists(m.configFile) { + return Config{}, xerrors.Errorf("config file not found, run 'trivy vex repo init' first") + } + + f, err := os.Open(m.configFile) + if err != nil { + return Config{}, xerrors.Errorf("unable to open a file: %w", err) + } + defer f.Close() + + var conf Config + if err = yaml.NewDecoder(f).Decode(&conf); err != nil { + return conf, xerrors.Errorf("unable to decode metadata: %w", err) + } + + for i := range conf.Repositories { + conf.Repositories[i].dir = m.repoDir + } + return conf, nil +} + +func (m *Manager) Init(ctx context.Context) error { + if fsutils.FileExists(m.configFile) { + log.InfoContext(ctx, "The configuration file already exists", log.String("path", m.configFile)) + return nil + } + + err := m.writeConfig(Config{ + Repositories: []Repository{ + { + Name: "default", + URL: defaultVEXHubURL, + }, + }, + }) + if err != nil { + return xerrors.Errorf("failed to write the default config: %w", err) + } + log.InfoContext(ctx, "The default configuration file has been created", log.FilePath(m.configFile)) + return nil +} + +func (m *Manager) UpdateManifest(ctx context.Context, names []string, opts Options) error { + conf, err := m.Config() + if err != nil { + return xerrors.Errorf("unable to read config: %w", err) + } else if len(conf.Repositories) == 0 { + return xerrors.Errorf("no repositories found in config: %s", m.configFile) + } + + for _, repo := range conf.Repositories { + if len(names) > 0 && !slices.Contains(names, repo.Name) { + continue + } + log.InfoContext(ctx, "Updating the repository...", log.String("name", repo.Name), log.String("url", repo.URL)) + if err = repo.downloadManifest(ctx, opts); err != nil { + return xerrors.Errorf("failed to update the repository: %w", err) + } + } + return nil +} diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index ad2be83195fc..79e13014f9eb 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -2,199 +2,262 @@ package repo import ( "context" - "fmt" + "encoding/json" + "errors" "io" + "net/url" "os" + "path" "path/filepath" - "slices" + "strings" + "time" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" + "github.com/google/go-github/v62/github" + "github.com/samber/lo" "golang.org/x/xerrors" - "gopkg.in/yaml.v3" + "github.com/aquasecurity/trivy/pkg/clock" "github.com/aquasecurity/trivy/pkg/downloader" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/utils/fsutils" ) -// const defaultVEXHubURL = "git@github.com:aquasecurity/vexhub.git" -const defaultVEXHubURL = "https://github.com/aquasecurity/vuln-list-update.git" +const ( + SchemaVersion = "0.1" + manifestFile = "vex-repository.json" + indexFile = "index.json" +) -type ManagerOption func(indexer *Manager) +type Manifest struct { + Name string `json:"name"` + Description string `json:"description"` + Versions map[string]Version `json:"versions"` + LatestVersion string `json:"latest_version"` +} -func WithWriter(w io.Writer) ManagerOption { - return func(manager *Manager) { - manager.w = w - } +type Version struct { + SpecVersion string `json:"spec_version"` + Locations []Location `json:"locations"` + UpdateInterval Duration `json:"update_interval"` +} + +// Duration is a wrapper around time.Duration that implements UnmarshalJSON +type Duration struct { + time.Duration } -func WithLogger(logger *log.Logger) ManagerOption { - return func(manager *Manager) { - manager.logger = logger +// UnmarshalJSON implements the json.Unmarshaler interface +func (d *Duration) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return xerrors.Errorf("duration unmarshal error: %w", err) + } + + var err error + d.Duration, err = time.ParseDuration(s) + if err != nil { + return xerrors.Errorf("duration parse error: %w", err) } + return nil +} + +type Location struct { + URL string `json:"url"` +} + +type Index struct { + UpdatedAt time.Time `json:"updated_at"` + Packages map[string]PackageEntry `json:"packages"` +} + +type PackageEntry struct { + ID string `json:"id"` + Location string `json:"location"` + Format string `json:"format"` } -type Config struct { - Repositories []Repository `json:"repositories"` +type rawIndex struct { + UpdatedAt time.Time `json:"updated_at"` + Packages []PackageEntry `json:"packages"` } type Repository struct { Name string URL string - DB string `yaml:",omitempty"` // TODO: support pre-built DB -} -type Options struct { - Insecure bool + dir string // Root directory for this VEX repository, $CACHE_DIR/vex/repositories/$REPO_NAME/ } -// Manager manages the plugins -type Manager struct { - w io.Writer - indexURL string - logger *log.Logger - configFile string - repoDir string -} +func (r *Repository) Manifest(ctx context.Context) (Manifest, error) { + filePath := filepath.Join(r.dir, manifestFile) + log.DebugContext(ctx, "Reading the repository metadata...", log.String("name", r.Name), log.FilePath(filePath)) -func NewManager(opts ...ManagerOption) *Manager { - root := filepath.Join(fsutils.TrivyHomeDir(), "vex") - m := &Manager{ - w: os.Stdout, - logger: log.WithPrefix("vex"), - configFile: filepath.Join(root, "config.yaml"), - repoDir: filepath.Join(root, "repositories"), - } - for _, opt := range opts { - opt(m) + f, err := os.Open(filePath) + if err != nil { + return Manifest{}, xerrors.Errorf("failed to open the file: %w", err) } + defer f.Close() - return m + var manifest Manifest + if err = json.NewDecoder(f).Decode(&manifest); err != nil { + return Manifest{}, xerrors.Errorf("failed to decode the metadata: %w", err) + } + return manifest, nil } -func (m *Manager) writeConfig(conf Config) error { - if err := os.MkdirAll(filepath.Dir(m.configFile), 0700); err != nil { - return xerrors.Errorf("failed to mkdir: %w", err) - } - f, err := os.Create(m.configFile) +func (r *Repository) Index(ctx context.Context) (Index, error) { + filePath := filepath.Join(r.dir, indexFile) + log.DebugContext(ctx, "Reading the repository index...", log.String("name", r.Name), log.FilePath(filePath)) + + f, err := os.Open(filePath) if err != nil { - return xerrors.Errorf("failed to create a file: %w", err) + return Index{}, xerrors.Errorf("failed to open the file: %w", err) } defer f.Close() - e := yaml.NewEncoder(f) - e.SetIndent(2) - if err = e.Encode(conf); err != nil { - return xerrors.Errorf("JSON encode error: %w", err) + var raw rawIndex + if err = json.NewDecoder(f).Decode(&raw); err != nil { + return Index{}, xerrors.Errorf("failed to decode the index: %w", err) } - return nil + return Index{ + UpdatedAt: raw.UpdatedAt, + Packages: lo.KeyBy(raw.Packages, func(p PackageEntry) string { return p.ID }), + }, nil } -func (m *Manager) readConfig() (Config, error) { - if !fsutils.FileExists(m.configFile) { - return Config{}, xerrors.Errorf("config file not found, run 'trivy vex repo init' first") +func (r *Repository) downloadManifest(ctx context.Context, opts Options) error { + if err := os.MkdirAll(r.dir, 0700); err != nil { + return xerrors.Errorf("failed to mkdir: %w", err) } - f, err := os.Open(m.configFile) + u, err := url.Parse(r.URL) if err != nil { - return Config{}, xerrors.Errorf("unable to open a file: %w", err) + return xerrors.Errorf("failed to parse the URL: %w", err) + } + + if u.Host == "github.com" { + if err = r.githubGet(ctx, u, r.dir); err != nil { + return xerrors.Errorf("failed to get the repository metadata: %w", err) + } + return nil } - defer f.Close() - var conf Config - if err = yaml.NewDecoder(f).Decode(&conf); err != nil { - return conf, xerrors.Errorf("unable to decode metadata: %w", err) + u.Path += path.Join(u.Path, ".well-known", manifestFile) + log.DebugContext(ctx, "Downloading the repository metadata...", log.String("url", u.String()), log.String("dst", r.dir)) + if err := downloader.Download(ctx, u.String(), r.dir, ".", opts.Insecure); err != nil { + return xerrors.Errorf("failed to download the repository: %w", err) } - return conf, nil + return nil } -func (m *Manager) Init(ctx context.Context) error { - if fsutils.FileExists(m.configFile) { - m.logger.InfoContext(ctx, "The configuration file already exists", log.String("path", m.configFile)) - return nil +func (r *Repository) githubGet(ctx context.Context, u *url.URL, dstDir string) error { + ss := strings.SplitN(u.Path, "/", 4) + if len(ss) < 3 { + return xerrors.Errorf("invalid GitHub URL: %s", u) + } + owner := ss[1] + repo := ss[2] + filePath := manifestFile + if len(ss) == 4 { + filePath = path.Join(ss[3], filePath) + } + + client := github.NewClient(nil) + if t := os.Getenv("GITHUB_TOKEN"); t != "" { + client = client.WithAuthToken(t) } - err := m.writeConfig(Config{ - Repositories: []Repository{ - { - Name: "default", - URL: defaultVEXHubURL, - }, - }, - }) + log.DebugContext(ctx, "Downloading the repository metadata from GitHub...", + log.String("owner", owner), log.String("repo", repo), log.String("path", filePath), + log.String("dst", dstDir)) + rc, _, err := client.Repositories.DownloadContents(ctx, owner, repo, filePath, nil) if err != nil { - return xerrors.Errorf("failed to write the default config: %w", err) + return xerrors.Errorf("failed to get the file content: %w", err) } - log.InfoContext(ctx, "The default configuration file has been created", log.FilePath(m.configFile)) - return nil -} + defer rc.Close() -func (m *Manager) Update(ctx context.Context, names []string, opts Options) error { - conf, err := m.readConfig() + f, err := os.Create(filepath.Join(dstDir, manifestFile)) if err != nil { - return xerrors.Errorf("unable to read config: %w", err) - } else if len(conf.Repositories) == 0 { - return xerrors.Errorf("no repositories found in config: %s", m.configFile) + return xerrors.Errorf("failed to create a file: %w", err) } + defer f.Close() - for _, repo := range conf.Repositories { - if len(names) > 0 && !slices.Contains(names, repo.Name) { - continue - } - m.logger.InfoContext(ctx, "Updating the repository...", log.String("name", repo.Name), log.String("url", repo.URL)) - if err = m.download(ctx, repo, opts); err != nil { - return xerrors.Errorf("failed to update the repository: %w", err) - } + if _, err = io.Copy(f, rc); err != nil { + return xerrors.Errorf("failed to copy the file: %w", err) } return nil } -func (m *Manager) download(ctx context.Context, repo Repository, opts Options) error { - // Force git protocol - // cf. https://github.com/hashicorp/go-getter/blob/5a63fd9c0d5b8da8a6805e8c283f46f0dacb30b3/README.md#forced-protocol - url := "git::" + repo.URL + "?depth=1" +func (r *Repository) Update(ctx context.Context, opts Options) error { + manifest, err := r.Manifest(ctx) + if err != nil { + return xerrors.Errorf("failed to get the repository metadata: %w", err) + } - dst := filepath.Join(m.repoDir, repo.Name) - if fsutils.DirExists(dst) { - defaultBranch, err := findDefaultBranch(dst) - if err != nil { - m.logger.DebugContext(ctx, "failed to find the default branch", log.String("path", dst), log.Err(err)) - defaultBranch = "main" - } - url += "&ref=" + defaultBranch + majorVersion, _, ok := strings.Cut(SchemaVersion, ".") + if !ok { + return xerrors.New("invalid schema version") + } + majorVersion = "v" + majorVersion + ver, ok := manifest.Versions[majorVersion] + if !ok { + // TODO: improve error + return xerrors.Errorf("version %s not found", majorVersion) + } + + versionDir := filepath.Join(r.dir, majorVersion) + if !r.needUpdate(ctx, ver, majorVersion) { + log.InfoContext(ctx, "Need to update repository", log.String("name", r.Name)) + return nil } - m.logger.DebugContext(ctx, "Downloading the repository...", log.String("url", url), log.String("dst", dst)) - if err := downloader.Download(ctx, url, dst, ".", opts.Insecure); err != nil { + log.InfoContext(ctx, "Need to update repository", log.String("name", r.Name)) + log.InfoContext(ctx, "Downloading repository...", log.String("name", r.Name), log.String("url", r.URL)) + if err = r.download(ctx, ver, versionDir, opts); err != nil { return xerrors.Errorf("failed to download the repository: %w", err) } - return nil + return err } -func findDefaultBranch(repoPath string) (string, error) { - repo, err := git.PlainOpen(repoPath) - if err != nil { - return "", err +func (r *Repository) needUpdate(ctx context.Context, ver Version, versionDir string) bool { + if !fsutils.DirExists(versionDir) { + return true } - remote, err := repo.Remote("origin") + index, err := r.Index(ctx) if err != nil { - return "", err + log.DebugContext(ctx, "Failed to get the repository index", log.String("name", r.Name), log.Err(err)) + return true } - refs, err := remote.List(&git.ListOptions{}) - if err != nil { - return "", err + now := clock.Clock(ctx).Now() + if now.After(index.UpdatedAt.Add(ver.UpdateInterval.Duration)) { + return true + } + + // TODO: use local metadata.json + + return false +} + +func (r *Repository) download(ctx context.Context, ver Version, dst string, opts Options) error { + if len(ver.Locations) == 0 { + return xerrors.Errorf("no locations found for version %s", ver.SpecVersion) + } + + if err := os.MkdirAll(dst, 0700); err != nil { + return xerrors.Errorf("failed to mkdir: %w", err) } - for _, ref := range refs { - if ref.Name() == "HEAD" { - if ref.Type() == plumbing.SymbolicReference { - return ref.Target().Short(), nil - } + var errs error + for _, loc := range ver.Locations { + log.DebugContext(ctx, "Downloading repository ...", log.String("url", loc.URL), log.String("dir", dst)) + if err := downloader.Download(ctx, loc.URL, dst, ".", opts.Insecure); err != nil { + errs = errors.Join(errs, err) + } else { + return nil } } - return "", fmt.Errorf("HEAD reference not found") + return errs } From 476dcfca0bdba9418b3471af49831611e5eabfdd Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 4 Jul 2024 22:16:24 +0400 Subject: [PATCH 03/55] fix: inherit logger options for fatal errors Signed-off-by: knqyf263 --- pkg/log/logger.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/log/logger.go b/pkg/log/logger.go index f46eb46fc87f..1f0b8e32e1a2 100644 --- a/pkg/log/logger.go +++ b/pkg/log/logger.go @@ -68,7 +68,12 @@ func Errorf(format string, args ...any) { slog.Default().Error(fmt.Sprintf(forma // Fatal for logging fatal errors func Fatal(msg string, args ...any) { // Fatal errors should be logged to stderr even if the logger is disabled. - New(NewHandler(os.Stderr, &Options{})).Log(context.Background(), LevelFatal, msg, args...) + if h, ok := slog.Default().Handler().(*ColorHandler); ok { + h.out = os.Stderr + } else { + slog.SetDefault(New(NewHandler(os.Stderr, &Options{}))) + } + slog.Default().Log(context.Background(), LevelFatal, msg, args...) os.Exit(1) } From d885ecf0b61f31508be03acd0d94bd855b668433 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 4 Jul 2024 22:21:10 +0400 Subject: [PATCH 04/55] feat: download VEX repos Signed-off-by: knqyf263 --- pkg/commands/app.go | 4 ++-- pkg/commands/artifact/run.go | 5 +++++ pkg/commands/operation/operation.go | 30 +++++++++++++++++++++++++-- pkg/downloader/download.go | 2 +- pkg/vex/repo/manager.go | 32 +++++++++++++++++------------ 5 files changed, 55 insertions(+), 18 deletions(-) diff --git a/pkg/commands/app.go b/pkg/commands/app.go index 78cdbd6539fc..eadb50cec207 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -1267,7 +1267,7 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { DisableFlagsInUseLine: true, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { - if err := vexrepo.NewManager().Init(cmd.Context()); err != nil { + if err := vexrepo.NewManager(vexOptions.CacheDir).Init(cmd.Context()); err != nil { return xerrors.Errorf("config init error: %w", err) } return nil @@ -1280,7 +1280,7 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { SilenceErrors: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - if err := vexrepo.NewManager().UpdateManifest(cmd.Context(), args, vexrepo.Options{Insecure: vexOptions.Insecure}); err != nil { + if err := vexrepo.NewManager(vexOptions.CacheDir).UpdateManifest(cmd.Context(), args, vexrepo.Options{Insecure: vexOptions.Insecure}); err != nil { return xerrors.Errorf("repository manifest update error: %w", err) } return nil diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go index 1343f4be61f0..3e928fa9c72d 100644 --- a/pkg/commands/artifact/run.go +++ b/pkg/commands/artifact/run.go @@ -119,6 +119,11 @@ func NewRunner(ctx context.Context, cliOptions flag.Options, opts ...RunnerOptio return nil, xerrors.Errorf("DB error: %w", err) } + // Update the VEX repositories if needed + if err := operation.DownloadVEXRepositories(ctx, cliOptions.CacheDir, cliOptions.SkipDBUpdate, cliOptions.Insecure); err != nil { + return nil, xerrors.Errorf("VEX repositories download error: %w", err) + } + // Initialize WASM modules m, err := module.NewManager(ctx, module.Options{ Dir: cliOptions.ModuleDir, diff --git a/pkg/commands/operation/operation.go b/pkg/commands/operation/operation.go index 92e45e5e696e..5dd83ece96c9 100644 --- a/pkg/commands/operation/operation.go +++ b/pkg/commands/operation/operation.go @@ -2,6 +2,7 @@ package operation import ( "context" + "errors" "sync" "github.com/google/go-containerregistry/pkg/name" @@ -13,6 +14,7 @@ import ( "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/policy" "github.com/aquasecurity/trivy/pkg/types" + "github.com/aquasecurity/trivy/pkg/vex/repo" ) var mu sync.Mutex @@ -23,6 +25,7 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n mu.Lock() defer mu.Unlock() + ctx = log.WithContextPrefix(ctx, "db") dbDir := db.Dir(cacheDir) client := db.NewClient(dbDir, quiet, db.WithDBRepository(dbRepository)) needsUpdate, err := client.NeedsUpdate(ctx, appVersion, skipUpdate) @@ -31,8 +34,8 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n } if needsUpdate { - log.Info("Need to update DB") - log.Info("Downloading DB...", log.String("repository", dbRepository.String())) + log.InfoContext(ctx, "Need to update DB") + log.InfoContext(ctx, "Downloading DB...", log.String("repository", dbRepository.String())) if err = client.Download(ctx, dbDir, opt); err != nil { return xerrors.Errorf("failed to download vulnerability DB: %w", err) } @@ -45,6 +48,29 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n return nil } +func DownloadVEXRepositories(ctx context.Context, cacheDir string, skipUpdate, insecure bool) error { + mu.Lock() + defer mu.Unlock() + + ctx = log.WithContextPrefix(ctx, "vex") + config, err := repo.NewManager(cacheDir).Config(ctx) + if errors.Is(err, repo.ErrNoConfig) { + return nil + } else if err != nil { + return xerrors.Errorf("failed to get vex repository config: %w", err) + } + + for _, r := range config.Repositories { + // TODO: support skip update + if err = r.Update(ctx, repo.Options{Insecure: insecure}); err != nil { + return xerrors.Errorf("failed to update vex repository: %w", err) + } + } + + return nil + +} + // InitBuiltinPolicies downloads the built-in policies and loads them func InitBuiltinPolicies(ctx context.Context, cacheDir string, quiet, skipUpdate bool, checkBundleRepository string, registryOpts ftypes.RegistryOptions) ([]string, error) { mu.Lock() diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index e4451fb21ad6..bc89dcf47912 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -61,7 +61,7 @@ func Download(ctx context.Context, src, dst, pwd string, insecure bool) error { if u, err := url.Parse(src); err == nil && u.Host == "github.com" { client := github.NewClient(nil) if t := os.Getenv("GITHUB_TOKEN"); t != "" { - client.WithAuthToken(t) + client = client.WithAuthToken(t) } httpGetter.Client = client.Client() } diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go index 278b2645dffe..36350326a968 100644 --- a/pkg/vex/repo/manager.go +++ b/pkg/vex/repo/manager.go @@ -2,6 +2,7 @@ package repo import ( "context" + "errors" "io" "os" "path/filepath" @@ -15,10 +16,13 @@ import ( ) const ( - // const defaultVEXHubURL = "git@github.com:aquasecurity/vexhub.git" - defaultVEXHubURL = "https://github.com/aquasecurity/vuln-list-update.git" + defaultVEXHubURL = "https://github.com/aquasecurity/vexhub" + vexDir = "vex" + repoDir = "repositories" ) +var ErrNoConfig = errors.New("no config found") + type ManagerOption func(indexer *Manager) func WithWriter(w io.Writer) ManagerOption { @@ -40,15 +44,14 @@ type Manager struct { w io.Writer indexURL string configFile string - repoDir string + cacheDir string } -func NewManager(opts ...ManagerOption) *Manager { - root := filepath.Join(fsutils.TrivyHomeDir(), "vex") +func NewManager(cacheRoot string, opts ...ManagerOption) *Manager { m := &Manager{ w: os.Stdout, - configFile: filepath.Join(root, "config.yaml"), - repoDir: filepath.Join(root, "repositories"), + configFile: filepath.Join(fsutils.TrivyHomeDir(), vexDir, "repository.yaml"), + cacheDir: filepath.Join(cacheRoot, vexDir), } for _, opt := range opts { opt(m) @@ -76,9 +79,10 @@ func (m *Manager) writeConfig(conf Config) error { return nil } -func (m *Manager) Config() (Config, error) { +func (m *Manager) Config(ctx context.Context) (Config, error) { if !fsutils.FileExists(m.configFile) { - return Config{}, xerrors.Errorf("config file not found, run 'trivy vex repo init' first") + log.DebugContext(ctx, "No config found", log.String("path", m.configFile)) + return Config{}, ErrNoConfig } f, err := os.Open(m.configFile) @@ -92,8 +96,8 @@ func (m *Manager) Config() (Config, error) { return conf, xerrors.Errorf("unable to decode metadata: %w", err) } - for i := range conf.Repositories { - conf.Repositories[i].dir = m.repoDir + for i, r := range conf.Repositories { + conf.Repositories[i].dir = filepath.Join(m.cacheDir, repoDir, r.Name) } return conf, nil } @@ -120,8 +124,10 @@ func (m *Manager) Init(ctx context.Context) error { } func (m *Manager) UpdateManifest(ctx context.Context, names []string, opts Options) error { - conf, err := m.Config() - if err != nil { + conf, err := m.Config(ctx) + if errors.Is(err, ErrNoConfig) { + return errors.New("no config found, run 'trivy vex repo init' first") + } else if err != nil { return xerrors.Errorf("unable to read config: %w", err) } else if len(conf.Repositories) == 0 { return xerrors.Errorf("no repositories found in config: %s", m.configFile) From 8ba0871c7e82378b6ef4c29cfb160a6446836c6d Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 4 Jul 2024 22:21:24 +0400 Subject: [PATCH 05/55] feat: add --vex-repos to 'trivy clean' Signed-off-by: knqyf263 --- pkg/commands/clean/run.go | 18 +++++++++++++++++- pkg/flag/clean_flags.go | 18 ++++++++++++++---- pkg/vex/repo/manager.go | 6 +++++- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/pkg/commands/clean/run.go b/pkg/commands/clean/run.go index 9d00d431b962..c60227360d19 100644 --- a/pkg/commands/clean/run.go +++ b/pkg/commands/clean/run.go @@ -12,13 +12,15 @@ import ( "github.com/aquasecurity/trivy/pkg/javadb" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/policy" + "github.com/aquasecurity/trivy/pkg/vex/repo" ) func Run(ctx context.Context, opts flag.Options) error { ctx, cancel := context.WithTimeout(ctx, opts.Timeout) defer cancel() - if !opts.CleanAll && !opts.CleanScanCache && !opts.CleanVulnerabilityDB && !opts.CleanJavaDB && !opts.CleanChecksBundle { + if !opts.CleanAll && !opts.CleanScanCache && !opts.CleanVulnerabilityDB && !opts.CleanJavaDB && + !opts.CleanChecksBundle && !opts.CleanVEXRepositories { return xerrors.New("no clean option is specified") } @@ -49,6 +51,12 @@ func Run(ctx context.Context, opts flag.Options) error { return xerrors.Errorf("check bundle clean error: %w", err) } } + + if opts.CleanVEXRepositories { + if err := cleanVEXRepositories(opts); err != nil { + return xerrors.Errorf("VEX repositories clean error: %w", err) + } + } return nil } @@ -102,3 +110,11 @@ func cleanCheckBundle(opts flag.Options) error { } return nil } + +func cleanVEXRepositories(opts flag.Options) error { + log.Info("Removing VEX repositories...") + if err := repo.NewManager(opts.CacheDir).Clear(); err != nil { + return xerrors.Errorf("clear VEX repositories: %w", err) + } + return nil +} diff --git a/pkg/flag/clean_flags.go b/pkg/flag/clean_flags.go index 7a898c38ad63..57cf1d3376fc 100644 --- a/pkg/flag/clean_flags.go +++ b/pkg/flag/clean_flags.go @@ -27,31 +27,39 @@ var ( ConfigName: "clean.checks-bundle", Usage: "remove checks bundle", } + CleanVEXRepos = Flag[bool]{ + Name: "vex-repos", + ConfigName: "clean.vex-repos", + Usage: "remove VEX repositories", + } ) type CleanFlagGroup struct { CleanAll *Flag[bool] + CleanScanCache *Flag[bool] CleanVulnerabilityDB *Flag[bool] CleanJavaDB *Flag[bool] CleanChecksBundle *Flag[bool] - CleanScanCache *Flag[bool] + CleanVEXRepositories *Flag[bool] } type CleanOptions struct { CleanAll bool + CleanScanCache bool CleanVulnerabilityDB bool CleanJavaDB bool CleanChecksBundle bool - CleanScanCache bool + CleanVEXRepositories bool } func NewCleanFlagGroup() *CleanFlagGroup { return &CleanFlagGroup{ CleanAll: CleanAll.Clone(), + CleanScanCache: CleanScanCache.Clone(), CleanVulnerabilityDB: CleanVulnerabilityDB.Clone(), CleanJavaDB: CleanJavaDB.Clone(), CleanChecksBundle: CleanChecksBundle.Clone(), - CleanScanCache: CleanScanCache.Clone(), + CleanVEXRepositories: CleanVEXRepos.Clone(), } } @@ -62,10 +70,11 @@ func (fg *CleanFlagGroup) Name() string { func (fg *CleanFlagGroup) Flags() []Flagger { return []Flagger{ fg.CleanAll, + fg.CleanScanCache, fg.CleanVulnerabilityDB, fg.CleanJavaDB, fg.CleanChecksBundle, - fg.CleanScanCache, + fg.CleanVEXRepositories, } } @@ -80,5 +89,6 @@ func (fg *CleanFlagGroup) ToOptions() (CleanOptions, error) { CleanJavaDB: fg.CleanJavaDB.Value(), CleanChecksBundle: fg.CleanChecksBundle.Value(), CleanScanCache: fg.CleanScanCache.Value(), + CleanVEXRepositories: fg.CleanVEXRepositories.Value(), }, nil } diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go index 36350326a968..7846e8107456 100644 --- a/pkg/vex/repo/manager.go +++ b/pkg/vex/repo/manager.go @@ -137,10 +137,14 @@ func (m *Manager) UpdateManifest(ctx context.Context, names []string, opts Optio if len(names) > 0 && !slices.Contains(names, repo.Name) { continue } - log.InfoContext(ctx, "Updating the repository...", log.String("name", repo.Name), log.String("url", repo.URL)) + log.InfoContext(ctx, "Updating the repository manifest...", log.String("name", repo.Name), log.String("url", repo.URL)) if err = repo.downloadManifest(ctx, opts); err != nil { return xerrors.Errorf("failed to update the repository: %w", err) } } return nil } + +func (m *Manager) Clear() error { + return os.RemoveAll(m.cacheDir) +} From 319eb65f6ebbca86851604835615f64b6c411de8 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 5 Jul 2024 21:07:13 +0400 Subject: [PATCH 06/55] add 'trivy vex repo download' Signed-off-by: knqyf263 --- pkg/commands/app.go | 13 ++++++++++--- pkg/vex/repo/manager.go | 9 +++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/commands/app.go b/pkg/commands/app.go index eadb50cec207..42f687330a53 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -1256,6 +1256,9 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { Short: "Manage VEX repositories", SilenceErrors: true, SilenceUsage: true, + Example: ` # Initialize the configuration file + $ trivy vex repo init +`, } repoCmd.AddCommand( @@ -1274,13 +1277,17 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { }, }, &cobra.Command{ - Use: "update [REPO_NAMES]", - Short: "Update the local copy of the VEX repository manifests", + Use: "download [REPO_NAMES]", + Short: "Download the VEX repositories", DisableFlagsInUseLine: true, SilenceErrors: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - if err := vexrepo.NewManager(vexOptions.CacheDir).UpdateManifest(cmd.Context(), args, vexrepo.Options{Insecure: vexOptions.Insecure}); err != nil { + err := vexrepo.NewManager(vexOptions.CacheDir).DownloadRepositories(cmd.Context(), args, + vexrepo.Options{Insecure: vexOptions.Insecure}) + if errors.Is(err, vexrepo.ErrNoConfig) { + return errors.New("no config found, run 'trivy vex repo init' first") + } else if err != nil { return xerrors.Errorf("repository manifest update error: %w", err) } return nil diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go index 7846e8107456..75a4603e5efc 100644 --- a/pkg/vex/repo/manager.go +++ b/pkg/vex/repo/manager.go @@ -123,11 +123,9 @@ func (m *Manager) Init(ctx context.Context) error { return nil } -func (m *Manager) UpdateManifest(ctx context.Context, names []string, opts Options) error { +func (m *Manager) DownloadRepositories(ctx context.Context, names []string, opts Options) error { conf, err := m.Config(ctx) - if errors.Is(err, ErrNoConfig) { - return errors.New("no config found, run 'trivy vex repo init' first") - } else if err != nil { + if err != nil { return xerrors.Errorf("unable to read config: %w", err) } else if len(conf.Repositories) == 0 { return xerrors.Errorf("no repositories found in config: %s", m.configFile) @@ -137,8 +135,7 @@ func (m *Manager) UpdateManifest(ctx context.Context, names []string, opts Optio if len(names) > 0 && !slices.Contains(names, repo.Name) { continue } - log.InfoContext(ctx, "Updating the repository manifest...", log.String("name", repo.Name), log.String("url", repo.URL)) - if err = repo.downloadManifest(ctx, opts); err != nil { + if err = repo.Update(ctx, opts); err != nil { return xerrors.Errorf("failed to update the repository: %w", err) } } From 2b188789efab54ae58f9d0949f2d1ee2f6fee8af Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 5 Jul 2024 21:08:21 +0400 Subject: [PATCH 07/55] feat: cache ETag Signed-off-by: knqyf263 --- pkg/commands/operation/operation.go | 11 +-- pkg/downloader/download.go | 117 ++++++++++++++++++++++++---- pkg/oci/artifact.go | 2 +- pkg/plugin/index.go | 3 +- pkg/plugin/manager.go | 2 +- pkg/plugin/plugin.go | 2 +- 6 files changed, 109 insertions(+), 28 deletions(-) diff --git a/pkg/commands/operation/operation.go b/pkg/commands/operation/operation.go index 5dd83ece96c9..152c109f39ff 100644 --- a/pkg/commands/operation/operation.go +++ b/pkg/commands/operation/operation.go @@ -53,20 +53,15 @@ func DownloadVEXRepositories(ctx context.Context, cacheDir string, skipUpdate, i defer mu.Unlock() ctx = log.WithContextPrefix(ctx, "vex") - config, err := repo.NewManager(cacheDir).Config(ctx) + err := repo.NewManager(cacheDir).DownloadRepositories(ctx, nil, repo.Options{ + Insecure: insecure, + }) if errors.Is(err, repo.ErrNoConfig) { return nil } else if err != nil { return xerrors.Errorf("failed to get vex repository config: %w", err) } - for _, r := range config.Repositories { - // TODO: support skip update - if err = r.Update(ctx, repo.Options{Insecure: insecure}); err != nil { - return xerrors.Errorf("failed to update vex repository: %w", err) - } - } - return nil } diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index bc89dcf47912..624e3d7d2288 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -2,9 +2,13 @@ package downloader import ( "context" + "errors" "maps" + "net/http" "net/url" "os" + "strings" + "time" "github.com/google/go-github/v62/github" getter "github.com/hashicorp/go-getter" @@ -13,8 +17,15 @@ import ( "github.com/aquasecurity/trivy/pkg/utils/fsutils" ) +var ErrSkipDownload = errors.New("skip download") + +type Options struct { + Insecure bool + ETag string +} + // DownloadToTempDir downloads the configured source to a temp dir. -func DownloadToTempDir(ctx context.Context, url string, insecure bool) (string, error) { +func DownloadToTempDir(ctx context.Context, url string, opts Options) (string, error) { tempDir, err := os.MkdirTemp("", "trivy-download") if err != nil { return "", xerrors.Errorf("failed to create a temp dir: %w", err) @@ -25,7 +36,7 @@ func DownloadToTempDir(ctx context.Context, url string, insecure bool) (string, return "", xerrors.Errorf("unable to get the current dir: %w", err) } - if err = Download(ctx, url, tempDir, pwd, insecure); err != nil { + if _, err = Download(ctx, url, tempDir, pwd, opts); err != nil { return "", xerrors.Errorf("download error: %w", err) } @@ -33,15 +44,15 @@ func DownloadToTempDir(ctx context.Context, url string, insecure bool) (string, } // Download downloads the configured source to the destination. -func Download(ctx context.Context, src, dst, pwd string, insecure bool) error { +func Download(ctx context.Context, src, dst, pwd string, opts Options) (string, error) { // go-getter doesn't allow the dst directory already exists if the src is directory. if fsutils.DirExists(src) { _ = os.RemoveAll(dst) } - var opts []getter.ClientOption - if insecure { - opts = append(opts, getter.WithInsecure()) + var clientOpts []getter.ClientOption + if opts.Insecure { + clientOpts = append(clientOpts, getter.WithInsecure()) } // Clone the global map so that it will not be accessed concurrently. @@ -55,15 +66,13 @@ func Download(ctx context.Context, src, dst, pwd string, insecure bool) error { // it cannot enable WithInsecure() afterwards because its state is preserved. // Therefore, we need to create a new "HttpGetter" instance every time. // cf. https://github.com/hashicorp/go-getter/blob/5a63fd9c0d5b8da8a6805e8c283f46f0dacb30b3/get.go#L63-L65 + transport := NewCustomTransport(opts.ETag) httpGetter := &getter.HttpGetter{ Netrc: true, - } - if u, err := url.Parse(src); err == nil && u.Host == "github.com" { - client := github.NewClient(nil) - if t := os.Getenv("GITHUB_TOKEN"); t != "" { - client = client.WithAuthToken(t) - } - httpGetter.Client = client.Client() + Client: &http.Client{ + Transport: transport, + Timeout: time.Minute * 5, + }, } getters["http"] = httpGetter getters["https"] = httpGetter @@ -76,12 +85,88 @@ func Download(ctx context.Context, src, dst, pwd string, insecure bool) error { Pwd: pwd, Getters: getters, Mode: getter.ClientModeAny, - Options: opts, + Options: clientOpts, } if err := client.Get(); err != nil { - return xerrors.Errorf("failed to download: %w", err) + return "", xerrors.Errorf("failed to download: %w", err) + } + + return transport.newETag, nil +} + +type CustomTransport struct { + cachedETag string + newETag string +} + +func NewCustomTransport(etag string) *CustomTransport { + return &CustomTransport{cachedETag: etag} +} + +func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if t.cachedETag != "" { + req.Header.Set("If-None-Match", t.cachedETag) + } + + transport := http.DefaultTransport + if req.URL.Host == "github.com" { + transport = NewGitHubTransport(req.URL) + } + + res, err := transport.RoundTrip(req) + if err != nil { + return nil, xerrors.Errorf("failed to round trip: %w", err) } - return nil + switch res.StatusCode { + case http.StatusOK, http.StatusPartialContent: + // Update the ETag + t.newETag = res.Header.Get("ETag") + case http.StatusNotModified: + return nil, ErrSkipDownload + } + + return res, nil +} + +func NewGitHubTransport(u *url.URL) http.RoundTripper { + client := newGitHubClient() + ss := strings.SplitN(u.Path, "/", 4) + if len(ss) < 4 || strings.HasPrefix(ss[3], "archive/") { + // Use the default transport from go-github for authentication + return client.Client().Transport + } + + return &GitHubContentTransport{ + owner: ss[1], + repo: ss[2], + filePath: ss[3], + client: client, + } +} + +// GitHubContentTransport is a round tripper for downloading the GitHub content. +type GitHubContentTransport struct { + owner string + repo string + filePath string + client *github.Client +} + +// RoundTrip calls the GitHub API to download the content. +func (t *GitHubContentTransport) RoundTrip(req *http.Request) (*http.Response, error) { + _, res, err := t.client.Repositories.DownloadContents(req.Context(), t.owner, t.repo, t.filePath, nil) + if err != nil { + return nil, xerrors.Errorf("failed to get the file content: %w", err) + } + return res.Response, nil +} + +func newGitHubClient() *github.Client { + client := github.NewClient(nil) + if token := os.Getenv("GITHUB_TOKEN"); token != "" { + client = client.WithAuthToken(token) + } + return client } diff --git a/pkg/oci/artifact.go b/pkg/oci/artifact.go index 8cd4460ec919..04d62ee15d36 100644 --- a/pkg/oci/artifact.go +++ b/pkg/oci/artifact.go @@ -189,7 +189,7 @@ func (a *Artifact) download(ctx context.Context, layer v1.Layer, fileName, dir s // Decompress the downloaded file if it is compressed and copy it into the dst // NOTE: it's local copying, the insecure option doesn't matter. - if err = downloader.Download(ctx, f.Name(), dir, dir, false); err != nil { + if _, err = downloader.Download(ctx, f.Name(), dir, dir, downloader.Options{}); err != nil { return xerrors.Errorf("download error: %w", err) } diff --git a/pkg/plugin/index.go b/pkg/plugin/index.go index 58beeaa5f9c7..57c6f0260877 100644 --- a/pkg/plugin/index.go +++ b/pkg/plugin/index.go @@ -34,7 +34,8 @@ type Index struct { func (m *Manager) Update(ctx context.Context, opts Options) error { m.logger.InfoContext(ctx, "Updating the plugin index...", log.String("url", m.indexURL)) - if err := downloader.Download(ctx, m.indexURL, filepath.Dir(m.indexPath), "", opts.Insecure); err != nil { + if _, err := downloader.Download(ctx, m.indexURL, filepath.Dir(m.indexPath), "", + downloader.Options{Insecure: opts.Insecure}); err != nil { return xerrors.Errorf("unable to download the plugin index: %w", err) } return nil diff --git a/pkg/plugin/manager.go b/pkg/plugin/manager.go index c0e5cadb5c7e..799481b98f14 100644 --- a/pkg/plugin/manager.go +++ b/pkg/plugin/manager.go @@ -111,7 +111,7 @@ func (m *Manager) Install(ctx context.Context, arg string, opts Options) (Plugin } func (m *Manager) install(ctx context.Context, src string, opts Options) (Plugin, error) { - tempDir, err := downloader.DownloadToTempDir(ctx, src, opts.Insecure) + tempDir, err := downloader.DownloadToTempDir(ctx, src, downloader.Options{Insecure: opts.Insecure}) if err != nil { return Plugin{}, xerrors.Errorf("download failed: %w", err) } diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 498f8d19c112..b427ed12e423 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -155,7 +155,7 @@ func (p *Plugin) install(ctx context.Context, dst, pwd string, opts Options) err p.Installed.Platform = lo.FromPtr(platform.Selector) log.DebugContext(ctx, "Downloading the execution file...", log.String("uri", platform.URI)) - if err = downloader.Download(ctx, platform.URI, dst, pwd, opts.Insecure); err != nil { + if _, err = downloader.Download(ctx, platform.URI, dst, pwd, downloader.Options{Insecure: opts.Insecure}); err != nil { return xerrors.Errorf("unable to download the execution file (%s): %w", platform.URI, err) } return nil From d2ec22734c42efb04ee8f35e35bfd88328ea5de1 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 5 Jul 2024 23:45:49 +0400 Subject: [PATCH 08/55] TODO: refactor CycloneDX Signed-off-by: knqyf263 --- pkg/vex/cyclonedx.go | 58 ++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/pkg/vex/cyclonedx.go b/pkg/vex/cyclonedx.go index 7bee16d32c81..c1c676428ac5 100644 --- a/pkg/vex/cyclonedx.go +++ b/pkg/vex/cyclonedx.go @@ -11,58 +11,52 @@ import ( type CycloneDX struct { sbom *core.BOM - statements []Statement + statements map[string]Statement logger *log.Logger } type Statement struct { - VulnerabilityID string - Affects []string - Status types.FindingStatus - Justification string + Affects []string + Status types.FindingStatus + Justification string } func newCycloneDX(sbom *core.BOM, vex *cdx.BOM) *CycloneDX { - var stmts []Statement + statements := make(map[string]Statement) for _, vuln := range lo.FromPtr(vex.Vulnerabilities) { affects := lo.Map(lo.FromPtr(vuln.Affects), func(item cdx.Affects, index int) string { return item.Ref }) analysis := lo.FromPtr(vuln.Analysis) - stmts = append(stmts, Statement{ - VulnerabilityID: vuln.ID, - Affects: affects, - Status: cdxStatus(analysis.State), - Justification: string(analysis.Justification), - }) + statements[vuln.ID] = Statement{ + Affects: affects, + Status: cdxStatus(analysis.State), + Justification: string(analysis.Justification), + } } return &CycloneDX{ sbom: sbom, - statements: stmts, + statements: statements, logger: log.WithPrefix("vex").With(log.String("format", "CycloneDX")), } } -func (v *CycloneDX) Filter(result *types.Result, _ *core.BOM) { - result.Vulnerabilities = lo.Filter(result.Vulnerabilities, func(vuln types.DetectedVulnerability, _ int) bool { - stmt, ok := lo.Find(v.statements, func(item Statement) bool { - return item.VulnerabilityID == vuln.VulnerabilityID - }) - if !ok { - return true - } - if !v.affected(vuln, stmt) { - result.ModifiedFindings = append(result.ModifiedFindings, - types.NewModifiedFinding(vuln, stmt.Status, stmt.Justification, "CycloneDX VEX")) - return false - } - return true - }) +func (v *CycloneDX) Filter(result *types.Result, bom *core.BOM) { + filterVulnerabilities(result, bom, v.NotAffected) } -func (v *CycloneDX) affected(vuln types.DetectedVulnerability, stmt Statement) bool { +func (v *CycloneDX) NotAffected(vuln types.DetectedVulnerability, product, _ *core.Component) (types.ModifiedFinding, bool) { + stmt, ok := v.statements[vuln.VulnerabilityID] + if !ok { + return types.ModifiedFinding{}, false + } + for _, affect := range stmt.Affects { + if stmt.Status != types.FindingStatusNotAffected && stmt.Status != types.FindingStatusFixed { + continue + } + // Affect must be BOM-Link at the moment link, err := cdx.ParseBOMLink(affect) if err != nil { @@ -75,11 +69,11 @@ func (v *CycloneDX) affected(vuln types.DetectedVulnerability, stmt Statement) b log.Int("version", link.Version())) continue } - if vuln.PkgIdentifier.Match(link.Reference()) && (stmt.Status == types.FindingStatusNotAffected || stmt.Status == types.FindingStatusFixed) { - return false + if product.PkgIdentifier.Match(link.Reference()) { + return types.NewModifiedFinding(vuln, stmt.Status, stmt.Justification, "CycloneDX VEX"), false } } - return true + return types.ModifiedFinding{}, false } func cdxStatus(s cdx.ImpactAnalysisState) types.FindingStatus { From c548e384ad439a02336603036ad2ddaa18bef69e Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 6 Jul 2024 10:14:23 +0400 Subject: [PATCH 09/55] refactor: always create parent map Signed-off-by: knqyf263 --- pkg/sbom/core/bom.go | 5 +---- pkg/vex/vex_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pkg/sbom/core/bom.go b/pkg/sbom/core/bom.go index 51875bff8738..cd238b535a11 100644 --- a/pkg/sbom/core/bom.go +++ b/pkg/sbom/core/bom.go @@ -203,7 +203,6 @@ type Vulnerability struct { type Options struct { GenerateBOMRef bool // Generate BOMRef for CycloneDX - Parents bool // Hold parent maps } func NewBOM(opts Options) *BOM { @@ -263,9 +262,7 @@ func (b *BOM) AddRelationship(parent, child *Component, relationshipType Relatio Dependency: child.id, }) - if b.opts.Parents { - b.parents[child.id] = append(b.parents[child.id], parent.id) - } + b.parents[child.id] = append(b.parents[child.id], parent.id) } func (b *BOM) AddVulnerabilities(c *Component, vulns []Vulnerability) { diff --git a/pkg/vex/vex_test.go b/pkg/vex/vex_test.go index d951b1795908..8a1b10a84f09 100644 --- a/pkg/vex/vex_test.go +++ b/pkg/vex/vex_test.go @@ -500,7 +500,7 @@ func newTestBOM1() *core.BOM { // - oci:debian?tag=12 // - pkg:maven/org.springframework.boot/spring-boot@2.6.0 // - pkg:deb/debian/bash@5.3 - bom := core.NewBOM(core.Options{Parents: true}) + bom := core.NewBOM(core.Options{}) bom.AddComponent(&ociComponent) bom.AddComponent(&springComponent) bom.AddComponent(&bashComponent) @@ -541,7 +541,7 @@ func newTestBOM3() *core.BOM { // - pkg:golang/github.com/aquasecurity/go-module@1.0.0 // - pkg:golang/github.com/aquasecurity/go-direct1@2.0.0 // - pkg:golang/github.com/aquasecurity/go-transitive@4.0.0 - bom := core.NewBOM(core.Options{Parents: true}) + bom := core.NewBOM(core.Options{}) bom.AddComponent(&fsComponent) bom.AddComponent(&goModuleComponent) bom.AddComponent(&goDirectComponent1) @@ -559,7 +559,7 @@ func newTestBOM4() *core.BOM { // - pkg:golang/github.com/aquasecurity/go-transitive@5.0.0 // - pkg:golang/github.com/aquasecurity/go-direct2@4.0.0 // - pkg:golang/github.com/aquasecurity/go-transitive@5.0.0 - bom := core.NewBOM(core.Options{Parents: true}) + bom := core.NewBOM(core.Options{}) bom.AddComponent(&fsComponent) bom.AddComponent(&goModuleComponent) bom.AddComponent(&goDirectComponent1) @@ -577,7 +577,7 @@ func newTestBOM5() *core.BOM { // - oci:debian?tag=12 // - pkg:bitnami/argo-cd@2.9.3-2?arch=amd64&distro=debian-12 // - pkg:golang/k8s.io/client-go@0.24.2 - bom := core.NewBOM(core.Options{Parents: true}) + bom := core.NewBOM(core.Options{}) bom.AddComponent(&ociComponent) bom.AddComponent(&argoComponent) bom.AddComponent(&clientGoComponent) @@ -589,7 +589,7 @@ func newTestBOM5() *core.BOM { func newTestBOM6() *core.BOM { // - oci:debian?tag=12 // - pkg:golang/k8s.io/client-go@0.24.2 - bom := core.NewBOM(core.Options{Parents: true}) + bom := core.NewBOM(core.Options{}) bom.AddComponent(&ociComponent) bom.AddComponent(&clientGoComponent) bom.AddRelationship(&ociComponent, &clientGoComponent, core.RelationshipContains) From 9787609b4b7e2f7455e35e0d307b9a16275e120b Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 6 Jul 2024 10:44:42 +0400 Subject: [PATCH 10/55] refactor: rename library to package Signed-off-by: knqyf263 --- pkg/sbom/io/decode.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sbom/io/decode.go b/pkg/sbom/io/decode.go index 379e1af32d52..6cfc79a0871c 100644 --- a/pkg/sbom/io/decode.go +++ b/pkg/sbom/io/decode.go @@ -137,11 +137,11 @@ func (m *Decoder) decodeComponents(ctx context.Context, sbom *types.SBOM) error // Third-party SBOMs may contain packages in types other than "Library" if c.Type == core.TypeLibrary || c.PkgIdentifier.PURL != nil { - pkg, err := m.decodeLibrary(ctx, c) + pkg, err := m.decodePackage(ctx, c) if errors.Is(err, ErrUnsupportedType) || errors.Is(err, ErrPURLEmpty) { continue } else if err != nil { - return xerrors.Errorf("failed to decode library: %w", err) + return xerrors.Errorf("failed to decode package: %w", err) } m.pkgs[id] = pkg } @@ -184,7 +184,7 @@ func (m *Decoder) decodeApplication(c *core.Component) *ftypes.Application { return &app } -func (m *Decoder) decodeLibrary(ctx context.Context, c *core.Component) (*ftypes.Package, error) { +func (m *Decoder) decodePackage(ctx context.Context, c *core.Component) (*ftypes.Package, error) { p := (*purl.PackageURL)(c.PkgIdentifier.PURL) if p == nil { m.logger.DebugContext(ctx, "Skipping a component without PURL", From 71b8ca30535da6a3e3721669eaa75a777ab502f0 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 6 Jul 2024 11:01:51 +0400 Subject: [PATCH 11/55] refactor: rename FilterOption to FilterOptions Signed-off-by: knqyf263 --- pkg/flag/options.go | 4 ++-- pkg/result/filter.go | 14 +++++++------- pkg/result/filter_test.go | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/flag/options.go b/pkg/flag/options.go index 33190fb76fbe..4e8a24a2a2a6 100644 --- a/pkg/flag/options.go +++ b/pkg/flag/options.go @@ -437,8 +437,8 @@ func (o *Options) RegistryOpts() ftypes.RegistryOptions { } // FilterOpts returns options for filtering -func (o *Options) FilterOpts() result.FilterOption { - return result.FilterOption{ +func (o *Options) FilterOpts() result.FilterOptions { + return result.FilterOptions{ Severities: o.Severities, IgnoreStatuses: o.IgnoreStatuses, IncludeNonFailures: o.IncludeNonFailures, diff --git a/pkg/result/filter.go b/pkg/result/filter.go index 7d4ead524ccc..f5aaced49225 100644 --- a/pkg/result/filter.go +++ b/pkg/result/filter.go @@ -24,7 +24,7 @@ const ( DefaultIgnoreFile = ".trivyignore" ) -type FilterOption struct { +type FilterOptions struct { Severities []dbTypes.Severity IgnoreStatuses []dbTypes.Status IncludeNonFailures bool @@ -35,20 +35,20 @@ type FilterOption struct { } // Filter filters out the report -func Filter(ctx context.Context, report types.Report, opt FilterOption) error { - ignoreConf, err := ParseIgnoreFile(ctx, opt.IgnoreFile) +func Filter(ctx context.Context, report types.Report, opts FilterOptions) error { + ignoreConf, err := ParseIgnoreFile(ctx, opts.IgnoreFile) if err != nil { - return xerrors.Errorf("%s error: %w", opt.IgnoreFile, err) + return xerrors.Errorf("%s error: %w", opts.IgnoreFile, err) } for i := range report.Results { - if err = FilterResult(ctx, &report.Results[i], ignoreConf, opt); err != nil { + if err = FilterResult(ctx, &report.Results[i], ignoreConf, opts); err != nil { return xerrors.Errorf("unable to filter vulnerabilities: %w", err) } } // Filter out vulnerabilities based on the given VEX document. - if err = filterByVEX(report, opt); err != nil { + if err = vex.Filter(&report, vex.Options{VEXPath: opts.VEXPath}); err != nil { return xerrors.Errorf("VEX error: %w", err) } @@ -56,7 +56,7 @@ func Filter(ctx context.Context, report types.Report, opt FilterOption) error { } // FilterResult filters out the result -func FilterResult(ctx context.Context, result *types.Result, ignoreConf IgnoreConfig, opt FilterOption) error { +func FilterResult(ctx context.Context, result *types.Result, ignoreConf IgnoreConfig, opt FilterOptions) error { // Convert dbTypes.Severity to string severities := lo.Map(opt.Severities, func(s dbTypes.Severity, _ int) string { return s.String() diff --git a/pkg/result/filter_test.go b/pkg/result/filter_test.go index 5c1eeb6771c3..c8eb2e3e3bcf 100644 --- a/pkg/result/filter_test.go +++ b/pkg/result/filter_test.go @@ -1007,7 +1007,7 @@ func TestFilter(t *testing.T) { fakeTime := time.Date(2020, 8, 10, 7, 28, 17, 958601, time.UTC) ctx := clock.With(context.Background(), fakeTime) - err := result.Filter(ctx, tt.args.report, result.FilterOption{ + err := result.Filter(ctx, tt.args.report, result.FilterOptions{ Severities: tt.args.severities, VEXPath: tt.args.vexPath, IgnoreStatuses: tt.args.ignoreStatuses, From 6a0533d3a297330448d3001d24eeab562cee82ba Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 6 Jul 2024 11:02:08 +0400 Subject: [PATCH 12/55] fix: re-use the existing UID Signed-off-by: knqyf263 --- pkg/dependency/id.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/dependency/id.go b/pkg/dependency/id.go index 577ed5d0ac41..3d1b5765639b 100644 --- a/pkg/dependency/id.go +++ b/pkg/dependency/id.go @@ -37,6 +37,9 @@ func ID(ltype types.LangType, name, version string) string { // UID calculates the hash of the package for the unique ID func UID(filePath string, pkg types.Package) string { + if pkg.Identifier.UID != "" { + return pkg.Identifier.UID + } v := map[string]any{ "filePath": filePath, // To differentiate the hash of the same package but different file path "pkg": pkg, From 7b35ccc605c22d02f37226fb9fdf46d3b8f5ea53 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 6 Jul 2024 13:07:05 +0400 Subject: [PATCH 13/55] refactor: re-define interface Signed-off-by: knqyf263 --- pkg/result/filter.go | 27 --------------------------- pkg/vex/csaf.go | 2 +- pkg/vex/cyclonedx.go | 6 +----- pkg/vex/openvex.go | 2 +- pkg/vex/vex.go | 40 +++++++++++++++++++++++++++++++++++----- 5 files changed, 38 insertions(+), 39 deletions(-) diff --git a/pkg/result/filter.go b/pkg/result/filter.go index f5aaced49225..f1e038a3782c 100644 --- a/pkg/result/filter.go +++ b/pkg/result/filter.go @@ -13,8 +13,6 @@ import ( "golang.org/x/xerrors" dbTypes "github.com/aquasecurity/trivy-db/pkg/types" - "github.com/aquasecurity/trivy/pkg/sbom/core" - sbomio "github.com/aquasecurity/trivy/pkg/sbom/io" "github.com/aquasecurity/trivy/pkg/types" "github.com/aquasecurity/trivy/pkg/vex" ) @@ -77,31 +75,6 @@ func FilterResult(ctx context.Context, result *types.Result, ignoreConf IgnoreCo return nil } -// filterByVEX determines whether a detected vulnerability should be filtered out based on the provided VEX document. -// If the VEX document is not nil and the vulnerability is either not affected or fixed according to the VEX statement, -// the vulnerability is filtered out. -func filterByVEX(report types.Report, opt FilterOption) error { - vexDoc, err := vex.New(opt.VEXPath, report) - if err != nil { - return err - } else if vexDoc == nil { - return nil - } - - bom, err := sbomio.NewEncoder(core.Options{Parents: true}).Encode(report) - if err != nil { - return xerrors.Errorf("unable to encode the SBOM: %w", err) - } - - for i, result := range report.Results { - if len(result.Vulnerabilities) == 0 { - continue - } - vexDoc.Filter(&report.Results[i], bom) - } - return nil -} - func filterVulnerabilities(result *types.Result, severities []string, ignoreStatuses []dbTypes.Status, ignoreConfig IgnoreConfig) { uniqVulns := make(map[string]types.DetectedVulnerability) for _, vuln := range result.Vulnerabilities { diff --git a/pkg/vex/csaf.go b/pkg/vex/csaf.go index 35680a8ddfc5..99f45b19a7cb 100644 --- a/pkg/vex/csaf.go +++ b/pkg/vex/csaf.go @@ -20,7 +20,7 @@ type relationship struct { SubProducts []*purl.PackageURL } -func newCSAF(advisory csaf.Advisory) VEX { +func newCSAF(advisory csaf.Advisory) *CSAF { return &CSAF{ advisory: advisory, logger: log.WithPrefix("vex").With(log.String("format", "CSAF")), diff --git a/pkg/vex/cyclonedx.go b/pkg/vex/cyclonedx.go index c1c676428ac5..771cc71be253 100644 --- a/pkg/vex/cyclonedx.go +++ b/pkg/vex/cyclonedx.go @@ -42,10 +42,6 @@ func newCycloneDX(sbom *core.BOM, vex *cdx.BOM) *CycloneDX { } } -func (v *CycloneDX) Filter(result *types.Result, bom *core.BOM) { - filterVulnerabilities(result, bom, v.NotAffected) -} - func (v *CycloneDX) NotAffected(vuln types.DetectedVulnerability, product, _ *core.Component) (types.ModifiedFinding, bool) { stmt, ok := v.statements[vuln.VulnerabilityID] if !ok { @@ -70,7 +66,7 @@ func (v *CycloneDX) NotAffected(vuln types.DetectedVulnerability, product, _ *co continue } if product.PkgIdentifier.Match(link.Reference()) { - return types.NewModifiedFinding(vuln, stmt.Status, stmt.Justification, "CycloneDX VEX"), false + return types.NewModifiedFinding(vuln, stmt.Status, stmt.Justification, "CycloneDX VEX"), true } } return types.ModifiedFinding{}, false diff --git a/pkg/vex/openvex.go b/pkg/vex/openvex.go index 36ab7808d559..1e62ebfea938 100644 --- a/pkg/vex/openvex.go +++ b/pkg/vex/openvex.go @@ -11,7 +11,7 @@ type OpenVEX struct { vex openvex.VEX } -func newOpenVEX(vex openvex.VEX) VEX { +func newOpenVEX(vex openvex.VEX) *OpenVEX { return &OpenVEX{ vex: vex, } diff --git a/pkg/vex/vex.go b/pkg/vex/vex.go index f4cf265997a0..f88574946355 100644 --- a/pkg/vex/vex.go +++ b/pkg/vex/vex.go @@ -17,6 +17,7 @@ import ( "github.com/aquasecurity/trivy/pkg/sbom" "github.com/aquasecurity/trivy/pkg/sbom/core" "github.com/aquasecurity/trivy/pkg/sbom/cyclonedx" + sbomio "github.com/aquasecurity/trivy/pkg/sbom/io" "github.com/aquasecurity/trivy/pkg/types" "github.com/aquasecurity/trivy/pkg/uuid" ) @@ -25,10 +26,39 @@ import ( // Note: This is in the experimental stage and does not yet support many specifications. // The implementation may change significantly. type VEX interface { - Filter(*types.Result, *core.BOM) + NotAffected(vuln types.DetectedVulnerability, product, subComponent *core.Component) (types.ModifiedFinding, bool) } -func New(filePath string, report types.Report) (VEX, error) { +type Options struct { + VEXPath string +} + +// Filter determines whether a detected vulnerability should be filtered out based on the provided VEX document. +// If the VEX document is passed and the vulnerability is either not affected or fixed according to the VEX statement, +// the vulnerability is filtered out. +func Filter(report *types.Report, opts Options) error { + vexDoc, err := New(opts.VEXPath, report) + if err != nil { + return xerrors.Errorf("unable to load VEX: %w", err) + } else if vexDoc == nil { + return nil + } + + bom, err := sbomio.NewEncoder(core.Options{}).Encode(*report) + if err != nil { + return xerrors.Errorf("unable to encode the SBOM: %w", err) + } + + for i, result := range report.Results { + if len(result.Vulnerabilities) == 0 { + continue + } + filterVulnerabilities(&report.Results[i], bom, vexDoc.NotAffected) + } + return nil +} + +func New(filePath string, report *types.Report) (VEX, error) { if filePath == "" { return nil, nil } @@ -63,7 +93,7 @@ func New(filePath string, report types.Report) (VEX, error) { return nil, xerrors.Errorf("unable to load VEX: %w", errs) } -func decodeCycloneDXJSON(r io.ReadSeeker, report types.Report) (VEX, error) { +func decodeCycloneDXJSON(r io.ReadSeeker, report *types.Report) (*CycloneDX, error) { if _, err := r.Seek(0, io.SeekStart); err != nil { return nil, xerrors.Errorf("seek error: %w", err) } @@ -77,7 +107,7 @@ func decodeCycloneDXJSON(r io.ReadSeeker, report types.Report) (VEX, error) { return newCycloneDX(report.BOM, vex), nil } -func decodeOpenVEX(r io.ReadSeeker) (VEX, error) { +func decodeOpenVEX(r io.ReadSeeker) (*OpenVEX, error) { // openvex/go-vex outputs log messages by default logrus.SetOutput(io.Discard) @@ -94,7 +124,7 @@ func decodeOpenVEX(r io.ReadSeeker) (VEX, error) { return newOpenVEX(openVEX), nil } -func decodeCSAF(r io.ReadSeeker) (VEX, error) { +func decodeCSAF(r io.ReadSeeker) (*CSAF, error) { if _, err := r.Seek(0, io.SeekStart); err != nil { return nil, xerrors.Errorf("seek error: %w", err) } From abd6ab92c77983f2664df82f751594923eb9e4a0 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 6 Jul 2024 13:07:37 +0400 Subject: [PATCH 14/55] test: fit for new interface Signed-off-by: knqyf263 --- pkg/vex/testdata/csaf-relationships.json | 40 +- pkg/vex/testdata/csaf.json | 18 +- pkg/vex/testdata/cyclonedx.json | 6 +- pkg/vex/testdata/openvex-oci-mismatch.json | 26 + pkg/vex/vex.go | 14 +- pkg/vex/vex_test.go | 734 ++++++++++----------- 6 files changed, 405 insertions(+), 433 deletions(-) create mode 100644 pkg/vex/testdata/openvex-oci-mismatch.json diff --git a/pkg/vex/testdata/csaf-relationships.json b/pkg/vex/testdata/csaf-relationships.json index ab58d2bfe492..2e823d17a2c6 100644 --- a/pkg/vex/testdata/csaf-relationships.json +++ b/pkg/vex/testdata/csaf-relationships.json @@ -37,22 +37,22 @@ "branches": [ { "category": "product_version", - "name": "2.9.3-2", + "name": "2.0.0", "product": { - "name": "Argo CD 2.9.3-2", - "product_id": "argo-cd-2.9.3-2-amd64-debian-12", + "name": "go-direct1 v2.0.0", + "product_id": "go-direct1-v2.0.0", "product_identification_helper": { - "purl": "pkg:bitnami/argo-cd@2.9.3-2?arch=amd64\u0026distro=debian-12" + "purl": "pkg:golang/github.com/aquasecurity/go-direct1@2.0.0" } } } ], "category": "product_name", - "name": "Argo CD" + "name": "go-direct1" } ], "category": "vendor", - "name": "VMWare, Inc." + "name": "bar" }, { "branches": [ @@ -60,45 +60,45 @@ "branches": [ { "category": "product_version", - "name": "v0.24.2", + "name": "v4.0.0", "product": { - "name": "client-go v0.24.2", - "product_id": "client-go-v0.24.2", + "name": "go-transitive v4.0.0", + "product_id": "go-transitive-v4.0.0", "product_identification_helper": { - "purl": "pkg:golang/k8s.io/client-go@0.24.2" + "purl": "pkg:golang/github.com/aquasecurity/go-transitive@4.0.0" } } } ], "category": "product_name", - "name": "client-go" + "name": "go-transitive" } ], "category": "vendor", - "name": "k8s.io" + "name": "foo" } ], "relationships": [ { - "product_reference": "client-go-v0.24.2", + "product_reference": "go-transitive-v4.0.0", "category": "default_component_of", - "relates_to_product_reference": "argo-cd-2.9.3-2-amd64-debian-12", + "relates_to_product_reference": "go-direct1-v2.0.0", "full_product_name": { - "product_id": "argo-cd-2.9.3-2-amd64-debian-12-client-go", - "name": "Argo CD uses kubernetes golang library" + "product_id": "go-direct1-v2.0.0-go-transitive-v4.0.0", + "name": "go-direct1 uses go-transitive" } } ] }, "vulnerabilities": [ { - "cve": "CVE-2023-2727", + "cve": "CVE-2024-0001", "flags": [ { "date": "2024-01-04T17:17:25+01:00", "label": "vulnerable_code_cannot_be_controlled_by_adversary", "product_ids": [ - "argo-cd-2.9.3-2-amd64-debian-12-client-go" + "go-direct1-v2.0.0-go-transitive-v4.0.0" ] } ], @@ -111,14 +111,14 @@ ], "product_status": { "known_not_affected": [ - "argo-cd-2.9.3-2-amd64-debian-12-client-go" + "go-direct1-v2.0.0-go-transitive-v4.0.0" ] }, "threats": [ { "category": "impact", "date": "2024-01-04T17:17:25+01:00", - "details": "The asset uses the component as a dependency in the code, but the vulnerability only affects Kubernetes clusters https://github.com/kubernetes/kubernetes/issues/118640" + "details": "vulnerable_code_not_in_execute_path" } ] } diff --git a/pkg/vex/testdata/csaf.json b/pkg/vex/testdata/csaf.json index 28389af24fcc..70afefe70205 100644 --- a/pkg/vex/testdata/csaf.json +++ b/pkg/vex/testdata/csaf.json @@ -45,28 +45,28 @@ "branches": [ { "category": "product_version", - "name": "v0.24.2", + "name": "v4.0.0", "product": { - "name": "client-go v0.24.2", - "product_id": "client-go-v0.24.2", + "name": "go-transitive v4.0.0", + "product_id": "go-transitive-v4.0.0", "product_identification_helper": { - "purl": "pkg:golang/k8s.io/client-go@0.24.2" + "purl": "pkg:golang/github.com/aquasecurity/go-transitive@4.0.0" } } } ], "category": "product_name", - "name": "client-go" + "name": "go-transitive" } ], "category": "vendor", - "name": "k8s.io" + "name": "foo" } ] }, "vulnerabilities": [ { - "cve": "CVE-2023-2727", + "cve": "CVE-2024-0001", "notes": [ { "category": "description", @@ -76,13 +76,13 @@ ], "product_status": { "known_not_affected": [ - "client-go-v0.24.2" + "go-transitive-v4.0.0" ] }, "threats": [ { "category": "impact", - "details": "The asset uses the component as a dependency in the code, but the vulnerability only affects Kubernetes clusters https://github.com/kubernetes/kubernetes/issues/118640" + "details": "vulnerable_code_not_in_execute_path" } ] } diff --git a/pkg/vex/testdata/cyclonedx.json b/pkg/vex/testdata/cyclonedx.json index ccc4396981b5..fb6463600c65 100644 --- a/pkg/vex/testdata/cyclonedx.json +++ b/pkg/vex/testdata/cyclonedx.json @@ -4,10 +4,10 @@ "version": 1, "vulnerabilities": [ { - "id": "CVE-2018-7489", + "id": "CVE-2021-44228", "source": { "name": "NVD", - "url": "https://nvd.nist.gov/vuln/detail/CVE-2019-9997" + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-44228" }, "analysis": { "state": "not_affected", @@ -15,7 +15,7 @@ }, "affects": [ { - "ref": "urn:cdx:3e671687-395b-41f5-a30f-a58921a69b79/1#pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.8.0" + "ref": "urn:cdx:3e671687-395b-41f5-a30f-a58921a69b79/1#pkg:maven/org.springframework.boot/spring-boot@2.6.0" } ] }, diff --git a/pkg/vex/testdata/openvex-oci-mismatch.json b/pkg/vex/testdata/openvex-oci-mismatch.json new file mode 100644 index 000000000000..e022d4854abb --- /dev/null +++ b/pkg/vex/testdata/openvex-oci-mismatch.json @@ -0,0 +1,26 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "author": "Aqua Security", + "role": "Project Release Bot", + "timestamp": "2023-01-16T19:07:16.853479631-06:00", + "version": 1, + "statements": [ + { + "vulnerability": { + "name": "CVE-2022-3715" + }, + "products": [ + { + "@id": "pkg:oci/mismatch", + "subcomponents": [ + { + "@id": "pkg:deb/debian/bash" + } + ] + } + ], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path" + } + ] +} diff --git a/pkg/vex/vex.go b/pkg/vex/vex.go index f88574946355..52dda5082234 100644 --- a/pkg/vex/vex.go +++ b/pkg/vex/vex.go @@ -152,16 +152,20 @@ func filterVulnerabilities(result *types.Result, bom *core.BOM, fn NotAffected) return true // Should never reach here } + var modified types.ModifiedFinding notAffectedFn := func(c, leaf *core.Component) bool { - modified, notAffected := fn(vuln, c, leaf) + m, notAffected := fn(vuln, c, leaf) if notAffected { - result.ModifiedFindings = append(result.ModifiedFindings, modified) - return true + modified = m // Take the last modified finding if multiple VEX states "not affected" } - return false + return notAffected } - return reachRoot(c, bom.Components(), bom.Parents(), notAffectedFn) + if !reachRoot(c, bom.Components(), bom.Parents(), notAffectedFn) { + result.ModifiedFindings = append(result.ModifiedFindings, modified) + return false + } + return true }) } diff --git a/pkg/vex/vex_test.go b/pkg/vex/vex_test.go index 8a1b10a84f09..febeee729c01 100644 --- a/pkg/vex/vex_test.go +++ b/pkg/vex/vex_test.go @@ -4,6 +4,7 @@ import ( "os" "testing" + "github.com/google/go-containerregistry/pkg/v1" "github.com/package-url/packageurl-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,41 +17,19 @@ import ( "github.com/aquasecurity/trivy/pkg/vex" ) +const ( + vulnerableCodeNotInExecutePath = "vulnerable_code_not_in_execute_path" + codeNotReachable = "code_not_reachable" +) + var ( - ociComponent = core.Component{ - Root: true, - Type: core.TypeContainerImage, - Name: "debian:12", - PkgIdentifier: ftypes.PkgIdentifier{ - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeOCI, - Name: "debian", - Version: "sha256:4482958b4461ff7d9fabc24b3a9ab1e9a2c85ece07b2db1840c7cbc01d053e90", - Qualifiers: packageurl.Qualifiers{ - { - Key: "tag", - Value: "12", - }, - { - Key: "repository_url", - Value: "docker.io/library/debian", - }, - }, - }, - }, - } - fsComponent = core.Component{ - Root: true, - Type: core.TypeFilesystem, - Name: ".", - } - springComponent = core.Component{ - Type: core.TypeLibrary, - Group: "org.springframework.boot", - Name: "spring-boot", + springPackage = ftypes.Package{ + ID: "org.springframework.boot:spring-boot:2.6.0", + Name: "org.springframework.boot:spring-boot", Version: "2.6.0", - PkgIdentifier: ftypes.PkgIdentifier{ - UID: "01", + Identifier: ftypes.PkgIdentifier{ + UID: "01", + BOMRef: "pkg:maven/org.springframework.boot/spring-boot@2.6.0", PURL: &packageurl.PackageURL{ Type: packageurl.TypeMaven, Namespace: "org.springframework.boot", @@ -59,25 +38,26 @@ var ( }, }, } - bashComponent = core.Component{ - Type: core.TypeLibrary, + bashPackage = ftypes.Package{ + ID: "bash@5.3", Name: "bash", Version: "5.3", - PkgIdentifier: ftypes.PkgIdentifier{ + Identifier: ftypes.PkgIdentifier{ UID: "02", PURL: &packageurl.PackageURL{ Type: packageurl.TypeDebian, Namespace: "debian", Name: "bash", - Version: "5.2.15", + Version: "5.3", }, }, } - goModuleComponent = core.Component{ - Type: core.TypeLibrary, - Name: "github.com/aquasecurity/go-module", - Version: "1.0.0", - PkgIdentifier: ftypes.PkgIdentifier{ + goModulePackage = ftypes.Package{ + ID: "github.com/aquasecurity/go-module@1.0.0", + Name: "github.com/aquasecurity/go-module", + Version: "1.0.0", + Relationship: ftypes.RelationshipRoot, + Identifier: ftypes.PkgIdentifier{ UID: "03", PURL: &packageurl.PackageURL{ Type: packageurl.TypeGolang, @@ -87,11 +67,12 @@ var ( }, }, } - goDirectComponent1 = core.Component{ - Type: core.TypeLibrary, - Name: "github.com/aquasecurity/go-direct1", - Version: "2.0.0", - PkgIdentifier: ftypes.PkgIdentifier{ + goDirectPackage1 = ftypes.Package{ + ID: "github.com/aquasecurity/go-direct1@2.0.0", + Name: "github.com/aquasecurity/go-direct1", + Version: "2.0.0", + Relationship: ftypes.RelationshipDirect, + Identifier: ftypes.PkgIdentifier{ UID: "04", PURL: &packageurl.PackageURL{ Type: packageurl.TypeGolang, @@ -101,11 +82,12 @@ var ( }, }, } - goDirectComponent2 = core.Component{ - Type: core.TypeLibrary, - Name: "github.com/aquasecurity/go-direct2", - Version: "3.0.0", - PkgIdentifier: ftypes.PkgIdentifier{ + goDirectPackage2 = ftypes.Package{ + ID: "github.com/aquasecurity/go-direct2@3.0.0", + Name: "github.com/aquasecurity/go-direct2", + Version: "3.0.0", + Relationship: ftypes.RelationshipDirect, + Identifier: ftypes.PkgIdentifier{ UID: "05", PURL: &packageurl.PackageURL{ Type: packageurl.TypeGolang, @@ -115,11 +97,12 @@ var ( }, }, } - goTransitiveComponent = core.Component{ - Type: core.TypeLibrary, - Name: "github.com/aquasecurity/go-transitive", - Version: "4.0.0", - PkgIdentifier: ftypes.PkgIdentifier{ + goTransitivePackage = ftypes.Package{ + ID: "github.com/aquasecurity/go-transitive@4.0.0", + Name: "github.com/aquasecurity/go-transitive", + Version: "4.0.0", + Relationship: ftypes.RelationshipIndirect, + Identifier: ftypes.PkgIdentifier{ UID: "06", PURL: &packageurl.PackageURL{ Type: packageurl.TypeGolang, @@ -129,87 +112,29 @@ var ( }, }, } - argoComponent = core.Component{ - Type: core.TypeLibrary, - Name: "argo-cd", - Version: "2.9.3-2", - PkgIdentifier: ftypes.PkgIdentifier{ - UID: "07", - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeBitnami, - Name: "argo-cd", - Version: "2.9.3-2", - Qualifiers: packageurl.Qualifiers{ - { - Key: "arch", - Value: "amd64", - }, - { - Key: "distro", - Value: "debian-12", - }, - }, - }, - }, - } - clientGoComponent = core.Component{ - Type: core.TypeLibrary, - Name: "k8s.io/client-go", - Version: "0.24.2", - PkgIdentifier: ftypes.PkgIdentifier{ - UID: "08", - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeGolang, - Namespace: "k8s.io", - Name: "client-go", - Version: "0.24.2", - }, - }, - } vuln1 = types.DetectedVulnerability{ VulnerabilityID: "CVE-2021-44228", - PkgName: springComponent.Name, - InstalledVersion: springComponent.Version, - PkgIdentifier: ftypes.PkgIdentifier{ - UID: springComponent.PkgIdentifier.UID, - PURL: springComponent.PkgIdentifier.PURL, - }, + PkgName: springPackage.Name, + InstalledVersion: springPackage.Version, + PkgIdentifier: springPackage.Identifier, } vuln2 = types.DetectedVulnerability{ VulnerabilityID: "CVE-2021-0001", - PkgName: springComponent.Name, - InstalledVersion: springComponent.Version, - PkgIdentifier: ftypes.PkgIdentifier{ - UID: springComponent.PkgIdentifier.UID, - PURL: springComponent.PkgIdentifier.PURL, - }, + PkgName: springPackage.Name, + InstalledVersion: springPackage.Version, + PkgIdentifier: springPackage.Identifier, } vuln3 = types.DetectedVulnerability{ VulnerabilityID: "CVE-2022-3715", - PkgName: bashComponent.Name, - InstalledVersion: bashComponent.Version, - PkgIdentifier: ftypes.PkgIdentifier{ - UID: bashComponent.PkgIdentifier.UID, - PURL: bashComponent.PkgIdentifier.PURL, - }, + PkgName: bashPackage.Name, + InstalledVersion: bashPackage.Version, + PkgIdentifier: bashPackage.Identifier, } vuln4 = types.DetectedVulnerability{ VulnerabilityID: "CVE-2024-0001", - PkgName: goTransitiveComponent.Name, - InstalledVersion: goTransitiveComponent.Version, - PkgIdentifier: ftypes.PkgIdentifier{ - UID: goTransitiveComponent.PkgIdentifier.UID, - PURL: goTransitiveComponent.PkgIdentifier.PURL, - }, - } - vuln5 = types.DetectedVulnerability{ - VulnerabilityID: "CVE-2023-2727", - PkgName: clientGoComponent.Name, - InstalledVersion: clientGoComponent.Version, - PkgIdentifier: ftypes.PkgIdentifier{ - UID: clientGoComponent.PkgIdentifier.UID, - PURL: clientGoComponent.PkgIdentifier.PURL, - }, + PkgName: goTransitivePackage.Name, + InstalledVersion: goTransitivePackage.Version, + PkgIdentifier: goTransitivePackage.Identifier, } ) @@ -218,380 +143,397 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestVEX_Filter(t *testing.T) { - type fields struct { - filePath string - report types.Report - } +func TestFilter(t *testing.T) { type args struct { - vulns []types.DetectedVulnerability - bom *core.BOM + report *types.Report + opts vex.Options } tests := []struct { name string - fields fields args args - want []types.DetectedVulnerability + want *types.Report wantErr string }{ { name: "OpenVEX", - fields: fields{ - filePath: "testdata/openvex.json", - }, args: args{ - vulns: []types.DetectedVulnerability{vuln1}, - bom: newTestBOM1(), + // - oci:debian?tag=12 + // - pkg:maven/org.springframework.boot/spring-boot@2.6.0 + report: imageReport([]types.Result{ + springResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln1}, + }), + }), + opts: vex.Options{ + VEXPath: "testdata/openvex.json", + }, }, - want: []types.DetectedVulnerability{}, + want: imageReport([]types.Result{ + springResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln1, vulnerableCodeNotInExecutePath, "OpenVEX")}, + }), + }), }, { name: "OpenVEX, multiple statements", - fields: fields{ - filePath: "testdata/openvex-multiple.json", - }, args: args{ - vulns: []types.DetectedVulnerability{ - vuln1, // filtered by VEX - vuln2, + // - oci:debian?tag=12 + // - pkg:maven/org.springframework.boot/spring-boot@2.6.0 + report: imageReport([]types.Result{ + springResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{ + vuln1, // filtered by VEX + vuln2, + }, + }), + }), + opts: vex.Options{ + VEXPath: "testdata/openvex-multiple.json", }, - bom: newTestBOM1(), - }, - want: []types.DetectedVulnerability{ - vuln2, }, + want: imageReport([]types.Result{ + springResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln2}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln1, vulnerableCodeNotInExecutePath, "OpenVEX")}, + }), + }), }, { name: "OpenVEX, subcomponents, oci image", - fields: fields{ - filePath: "testdata/openvex-oci.json", - }, args: args{ - vulns: []types.DetectedVulnerability{ - vuln3, + // - oci:debian?tag=12 + // - pkg:deb/debian/bash@5.3 + report: imageReport([]types.Result{ + bashResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{ + vuln3, // filtered by VEX + }, + }), + }), + opts: vex.Options{ + VEXPath: "testdata/openvex-oci.json", }, - bom: newTestBOM1(), }, - want: []types.DetectedVulnerability{}, + want: imageReport([]types.Result{ + bashResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln3, vulnerableCodeNotInExecutePath, "OpenVEX")}, + }), + }), }, { - name: "OpenVEX, subcomponents, wrong oci image", - fields: fields{ - filePath: "testdata/openvex-oci.json", - }, + name: "OpenVEX, subcomponents, mismatched oci image", args: args{ - vulns: []types.DetectedVulnerability{vuln3}, - bom: newTestBOM2(), + report: imageReport(types.Results{ + bashResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln3}, + }), + }), + opts: vex.Options{ + VEXPath: "testdata/openvex-oci-mismatch.json", + }, }, - want: []types.DetectedVulnerability{vuln3}, + want: imageReport([]types.Result{ + bashResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln3}, + }), + }), }, { name: "OpenVEX, single path between product and subcomponent", - fields: fields{ - filePath: "testdata/openvex-nested.json", - }, args: args{ - vulns: []types.DetectedVulnerability{vuln4}, - bom: newTestBOM3(), + report: fsReport([]types.Result{ + goSinglePathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{ + vuln4, // filtered by VEX + }, + }), + }), + opts: vex.Options{ + VEXPath: "testdata/openvex-nested.json", + }, }, - want: []types.DetectedVulnerability{}, + want: fsReport([]types.Result{ + goSinglePathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "OpenVEX")}, + }), + }), }, { name: "OpenVEX, multi paths between product and subcomponent", - fields: fields{ - filePath: "testdata/openvex-nested.json", - }, args: args{ - vulns: []types.DetectedVulnerability{vuln4}, - bom: newTestBOM4(), + report: fsReport([]types.Result{ + goMultiPathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{ + vuln4, + }, + }), + }), + opts: vex.Options{ + VEXPath: "testdata/openvex-nested.json", + }, }, - want: []types.DetectedVulnerability{vuln4}, // Will not be filtered because of multi paths + want: fsReport([]types.Result{ + goMultiPathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln4}, // Will not be filtered because of multi paths + }), + }), }, { name: "CycloneDX SBOM with CycloneDX VEX", - fields: fields{ - filePath: "testdata/cyclonedx.json", - report: types.Report{ + args: args{ + report: &types.Report{ ArtifactType: artifact.TypeCycloneDX, BOM: &core.BOM{ SerialNumber: "urn:uuid:3e671687-395b-41f5-a30f-a58921a69b79", Version: 1, }, - }, - }, - args: args{ - vulns: []types.DetectedVulnerability{ - { - VulnerabilityID: "CVE-2018-7489", - PkgName: "jackson-databind", - InstalledVersion: "2.8.0", - PkgIdentifier: ftypes.PkgIdentifier{ - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeMaven, - Namespace: "com.fasterxml.jackson.core", - Name: "jackson-databind", - Version: "2.8.0", - }, - }, - }, - { - VulnerabilityID: "CVE-2018-7490", - PkgName: "jackson-databind", - InstalledVersion: "2.8.0", - PkgIdentifier: ftypes.PkgIdentifier{ - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeMaven, - Namespace: "com.fasterxml.jackson.core", - Name: "jackson-databind", - Version: "2.8.0", - }, - }, - }, - { - VulnerabilityID: "CVE-2022-27943", - PkgID: "libstdc++6@12.3.0-1ubuntu1~22.04", - PkgName: "libstdc++6", - InstalledVersion: "12.3.0-1ubuntu1~22.04", - PkgIdentifier: ftypes.PkgIdentifier{ - BOMRef: "pkg:deb/ubuntu/libstdc%2B%2B6@12.3.0-1ubuntu1~22.04?distro=ubuntu-22.04&arch=amd64", - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeDebian, - Namespace: "ubuntu", - Name: "libstdc++6", - Version: "12.3.0-1ubuntu1~22.04", - Qualifiers: []packageurl.Qualifier{ - { - Key: "arch", - Value: "amd64", - }, - { - Key: "distro", - Value: "ubuntu-22.04", - }, - }, - }, - }, + Results: []types.Result{ + springResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln1}, + }), }, }, + opts: vex.Options{ + VEXPath: "testdata/cyclonedx.json", + }, }, - want: []types.DetectedVulnerability{ - { - VulnerabilityID: "CVE-2018-7490", - PkgName: "jackson-databind", - InstalledVersion: "2.8.0", - PkgIdentifier: ftypes.PkgIdentifier{ - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeMaven, - Namespace: "com.fasterxml.jackson.core", - Name: "jackson-databind", - Version: "2.8.0", - }, - }, + want: &types.Report{ + ArtifactType: artifact.TypeCycloneDX, + BOM: &core.BOM{ + SerialNumber: "urn:uuid:3e671687-395b-41f5-a30f-a58921a69b79", + Version: 1, + }, + Results: []types.Result{ + springResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln1, codeNotReachable, "CycloneDX VEX")}, + }), }, }, }, { name: "CycloneDX VEX wrong URN", - fields: fields{ - filePath: "testdata/cyclonedx.json", - report: types.Report{ + args: args{ + report: &types.Report{ ArtifactType: artifact.TypeCycloneDX, BOM: &core.BOM{ SerialNumber: "urn:uuid:wrong", Version: 1, }, - }, - }, - args: args{ - vulns: []types.DetectedVulnerability{ - { - VulnerabilityID: "CVE-2018-7489", - PkgName: "jackson-databind", - InstalledVersion: "2.8.0", - PkgIdentifier: ftypes.PkgIdentifier{ - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeMaven, - Namespace: "com.fasterxml.jackson.core", - Name: "jackson-databind", - Version: "2.8.0", - }, - }, + Results: []types.Result{ + springResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln1}, + }), }, }, + opts: vex.Options{ + VEXPath: "testdata/cyclonedx.json", + }, }, - want: []types.DetectedVulnerability{ - { - VulnerabilityID: "CVE-2018-7489", - PkgName: "jackson-databind", - InstalledVersion: "2.8.0", - PkgIdentifier: ftypes.PkgIdentifier{ - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeMaven, - Namespace: "com.fasterxml.jackson.core", - Name: "jackson-databind", - Version: "2.8.0", - }, - }, + want: &types.Report{ + ArtifactType: artifact.TypeCycloneDX, + BOM: &core.BOM{ + SerialNumber: "urn:uuid:wrong", + Version: 1, + }, + Results: []types.Result{ + springResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln1}, + }), }, }, }, { name: "CSAF, not affected", - fields: fields{ - filePath: "testdata/csaf.json", - }, args: args{ - bom: newTestBOM5(), - vulns: []types.DetectedVulnerability{vuln5}, + report: imageReport([]types.Result{ + goSinglePathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{ + vuln4, // filtered by VEX + }, + }), + }), + opts: vex.Options{ + VEXPath: "testdata/csaf.json", + }, }, - want: []types.DetectedVulnerability{}, + want: imageReport([]types.Result{ + goSinglePathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "CSAF VEX")}, + }), + }), }, { name: "CSAF with relationships, not affected", - fields: fields{ - filePath: "testdata/csaf-relationships.json", - }, args: args{ - bom: newTestBOM5(), - vulns: []types.DetectedVulnerability{vuln5}, + report: imageReport([]types.Result{ + goSinglePathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{ + vuln4, // filtered by VEX + }, + }), + }), + opts: vex.Options{ + VEXPath: "testdata/csaf-relationships.json", + }, }, - want: []types.DetectedVulnerability{}, + want: imageReport([]types.Result{ + goSinglePathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "CSAF VEX")}, + }), + }), }, { name: "CSAF with relationships, affected", - fields: fields{ - filePath: "testdata/csaf-relationships.json", - }, args: args{ - bom: newTestBOM6(), - vulns: []types.DetectedVulnerability{vuln5}, + report: imageReport([]types.Result{ + goMultiPathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{ + vuln4, + }, + }), + }), + opts: vex.Options{ + VEXPath: "testdata/csaf-relationships.json", + }, }, - want: []types.DetectedVulnerability{vuln5}, + want: imageReport([]types.Result{ + goMultiPathResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln4}, // Will not be filtered because of multi paths + }), + }), }, { name: "unknown format", - fields: fields{ - filePath: "testdata/unknown.json", + args: args{ + report: &types.Report{}, + opts: vex.Options{ + VEXPath: "testdata/unknown.json", + }, }, - args: args{}, wantErr: "unable to load VEX", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - v, err := vex.New(tt.fields.filePath, tt.fields.report) + err := vex.Filter(tt.args.report, tt.args.opts) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return } require.NoError(t, err) - - got := &types.Result{ - Vulnerabilities: tt.args.vulns, - } - v.Filter(got, tt.args.bom) - assert.Equal(t, tt.want, got.Vulnerabilities) + assert.Equal(t, tt.want, tt.args.report) }) } } -func newTestBOM1() *core.BOM { - // - oci:debian?tag=12 - // - pkg:maven/org.springframework.boot/spring-boot@2.6.0 - // - pkg:deb/debian/bash@5.3 - bom := core.NewBOM(core.Options{}) - bom.AddComponent(&ociComponent) - bom.AddComponent(&springComponent) - bom.AddComponent(&bashComponent) - bom.AddRelationship(&ociComponent, &springComponent, core.RelationshipContains) - bom.AddRelationship(&ociComponent, &bashComponent, core.RelationshipContains) - return bom -} - -func newTestBOM2() *core.BOM { - bom := core.NewBOM(core.Options{}) - bom.AddComponent(&core.Component{ - Root: true, - Type: core.TypeContainerImage, - Name: "ubuntu:24.04", - PkgIdentifier: ftypes.PkgIdentifier{ - PURL: &packageurl.PackageURL{ - Type: packageurl.TypeOCI, - Name: "ubuntu", - Version: "sha256:4482958b4461ff7d9fabc24b3a9ab1e9a2c85ece07b2db1840c7cbc01d053e90", - Qualifiers: packageurl.Qualifiers{ - { - Key: "tag", - Value: "24.04", - }, - { - Key: "repository_url", - Value: "docker.io/library/ubuntu", - }, - }, +func imageReport(results types.Results) *types.Report { + return &types.Report{ + ArtifactName: "debian:12", + ArtifactType: artifact.TypeContainerImage, + Metadata: types.Metadata{ + RepoDigests: []string{ + "debian:@sha256:4482958b4461ff7d9fabc24b3a9ab1e9a2c85ece07b2db1840c7cbc01d053e90", + }, + ImageConfig: v1.ConfigFile{ + Architecture: "amd64", }, }, - }) - return bom + Results: results, + } +} + +func fsReport(results types.Results) *types.Report { + return &types.Report{ + ArtifactName: ".", + ArtifactType: artifact.TypeFilesystem, + Results: results, + } } -func newTestBOM3() *core.BOM { - // - filesystem - // - pkg:golang/github.com/aquasecurity/go-module@1.0.0 - // - pkg:golang/github.com/aquasecurity/go-direct1@2.0.0 - // - pkg:golang/github.com/aquasecurity/go-transitive@4.0.0 - bom := core.NewBOM(core.Options{}) - bom.AddComponent(&fsComponent) - bom.AddComponent(&goModuleComponent) - bom.AddComponent(&goDirectComponent1) - bom.AddComponent(&goTransitiveComponent) - bom.AddRelationship(&fsComponent, &goModuleComponent, core.RelationshipContains) - bom.AddRelationship(&goModuleComponent, &goDirectComponent1, core.RelationshipDependsOn) - bom.AddRelationship(&goDirectComponent1, &goTransitiveComponent, core.RelationshipDependsOn) - return bom +func springResult(result types.Result) types.Result { + result.Type = ftypes.Jar + result.Class = types.ClassLangPkg + result.Packages = []ftypes.Package{springPackage} + return result } -func newTestBOM4() *core.BOM { - // - filesystem - // - pkg:golang/github.com/aquasecurity/go-module@2.0.0 - // - pkg:golang/github.com/aquasecurity/go-direct1@3.0.0 - // - pkg:golang/github.com/aquasecurity/go-transitive@5.0.0 - // - pkg:golang/github.com/aquasecurity/go-direct2@4.0.0 - // - pkg:golang/github.com/aquasecurity/go-transitive@5.0.0 - bom := core.NewBOM(core.Options{}) - bom.AddComponent(&fsComponent) - bom.AddComponent(&goModuleComponent) - bom.AddComponent(&goDirectComponent1) - bom.AddComponent(&goDirectComponent2) - bom.AddComponent(&goTransitiveComponent) - bom.AddRelationship(&fsComponent, &goModuleComponent, core.RelationshipContains) - bom.AddRelationship(&goModuleComponent, &goDirectComponent1, core.RelationshipDependsOn) - bom.AddRelationship(&goModuleComponent, &goDirectComponent2, core.RelationshipDependsOn) - bom.AddRelationship(&goDirectComponent1, &goTransitiveComponent, core.RelationshipDependsOn) - bom.AddRelationship(&goDirectComponent2, &goTransitiveComponent, core.RelationshipDependsOn) - return bom +// bashResult wraps the result with the bash package +func bashResult(result types.Result) types.Result { + result.Type = ftypes.Debian + result.Class = types.ClassOSPkg + result.Packages = []ftypes.Package{bashPackage} + return result } -func newTestBOM5() *core.BOM { - // - oci:debian?tag=12 - // - pkg:bitnami/argo-cd@2.9.3-2?arch=amd64&distro=debian-12 - // - pkg:golang/k8s.io/client-go@0.24.2 - bom := core.NewBOM(core.Options{}) - bom.AddComponent(&ociComponent) - bom.AddComponent(&argoComponent) - bom.AddComponent(&clientGoComponent) - bom.AddRelationship(&ociComponent, &argoComponent, core.RelationshipContains) - bom.AddRelationship(&argoComponent, &clientGoComponent, core.RelationshipDependsOn) - return bom +func goSinglePathResult(result types.Result) types.Result { + result.Type = ftypes.GoModule + result.Class = types.ClassLangPkg + + // - pkg:golang/github.com/aquasecurity/go-module@1.0.0 + // - pkg:golang/github.com/aquasecurity/go-direct1@2.0.0 + // - pkg:golang/github.com/aquasecurity/go-transitive@4.0.0 + goModule := clonePackage(goModulePackage) + goDirect1 := clonePackage(goDirectPackage1) + goTransitive := clonePackage(goTransitivePackage) + + goModule.DependsOn = []string{goDirect1.ID} + goDirect1.DependsOn = []string{goTransitive.ID} + result.Packages = []ftypes.Package{ + goModule, + goDirect1, + goTransitive, + } + return result +} + +func goMultiPathResult(result types.Result) types.Result { + result.Type = ftypes.GoModule + result.Class = types.ClassLangPkg + + // - pkg:golang/github.com/aquasecurity/go-module@2.0.0 + // - pkg:golang/github.com/aquasecurity/go-direct1@3.0.0 + // - pkg:golang/github.com/aquasecurity/go-transitive@5.0.0 + // - pkg:golang/github.com/aquasecurity/go-direct2@4.0.0 + // - pkg:golang/github.com/aquasecurity/go-transitive@5.0.0 + goModule := clonePackage(goModulePackage) + goDirect1 := clonePackage(goDirectPackage1) + goDirect2 := clonePackage(goDirectPackage2) + goTransitive := clonePackage(goTransitivePackage) + + goModule.DependsOn = []string{ + goDirect1.ID, + goDirect2.ID, + } + goDirect1.DependsOn = []string{goTransitive.ID} + goDirect2.DependsOn = []string{goTransitive.ID} + result.Packages = []ftypes.Package{ + goModule, + goDirect1, + goDirect2, + goTransitive, + } + return result +} + +func modifiedFinding(vuln types.DetectedVulnerability, statement, source string) types.ModifiedFinding { + return types.ModifiedFinding{ + Type: types.FindingTypeVulnerability, + Status: types.FindingStatusNotAffected, + Statement: statement, + Source: source, + Finding: vuln, + } } -func newTestBOM6() *core.BOM { - // - oci:debian?tag=12 - // - pkg:golang/k8s.io/client-go@0.24.2 - bom := core.NewBOM(core.Options{}) - bom.AddComponent(&ociComponent) - bom.AddComponent(&clientGoComponent) - bom.AddRelationship(&ociComponent, &clientGoComponent, core.RelationshipContains) - return bom +func clonePackage(p ftypes.Package) ftypes.Package { + n := p + n.DependsOn = []string{} + return n } From 7458b32433ddf34477d0cb0e3bd071dc89718978 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 6 Jul 2024 13:08:08 +0400 Subject: [PATCH 15/55] fix: re-use serial number from SBOM Signed-off-by: knqyf263 --- pkg/sbom/io/encode.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/sbom/io/encode.go b/pkg/sbom/io/encode.go index 096abd026b86..45c5dca245c6 100644 --- a/pkg/sbom/io/encode.go +++ b/pkg/sbom/io/encode.go @@ -35,6 +35,10 @@ func (e *Encoder) Encode(report types.Report) (*core.BOM, error) { } e.bom = core.NewBOM(e.opts) + if report.BOM != nil { + e.bom.SerialNumber = report.BOM.SerialNumber + e.bom.Version = report.BOM.Version + } e.bom.AddComponent(root) for _, result := range report.Results { From 1353291c6bec56dace8b305c004ec645cd68c4fc Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 8 Jul 2024 10:45:24 +0400 Subject: [PATCH 16/55] fix: go-github may return nil for transport Signed-off-by: knqyf263 --- pkg/downloader/download.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index 624e3d7d2288..ba83ca731f6a 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -109,10 +109,13 @@ func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) { req.Header.Set("If-None-Match", t.cachedETag) } - transport := http.DefaultTransport + var transport http.RoundTripper if req.URL.Host == "github.com" { transport = NewGitHubTransport(req.URL) } + if transport == nil { + transport = http.DefaultTransport + } res, err := transport.RoundTrip(req) if err != nil { From e969bf814b3e4a67a3a5c3d278386808e18bc8fe Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 8 Jul 2024 11:14:22 +0400 Subject: [PATCH 17/55] feat: store cache metadata for VEX repos Signed-off-by: knqyf263 --- pkg/vex/repo/repo.go | 169 ++++++++++++++++++++++++++----------------- 1 file changed, 102 insertions(+), 67 deletions(-) diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index 79e13014f9eb..f00efdca62ce 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "io" "net/url" "os" "path" @@ -12,7 +11,6 @@ import ( "strings" "time" - "github.com/google/go-github/v62/github" "github.com/samber/lo" "golang.org/x/xerrors" @@ -24,8 +22,10 @@ import ( const ( SchemaVersion = "0.1" - manifestFile = "vex-repository.json" - indexFile = "index.json" + + manifestFile = "vex-repository.json" + indexFile = "index.json" + cacheMetadataFile = "cache.json" ) type Manifest struct { @@ -66,8 +66,9 @@ type Location struct { } type Index struct { - UpdatedAt time.Time `json:"updated_at"` - Packages map[string]PackageEntry `json:"packages"` + Path string // Path to the index file + UpdatedAt time.Time + Packages map[string]PackageEntry } type PackageEntry struct { @@ -88,10 +89,20 @@ type Repository struct { dir string // Root directory for this VEX repository, $CACHE_DIR/vex/repositories/$REPO_NAME/ } +type CacheMetadata struct { + UpdatedAt time.Time // Last updated time + ETags map[string]string // Last ETag for each URL +} + func (r *Repository) Manifest(ctx context.Context) (Manifest, error) { filePath := filepath.Join(r.dir, manifestFile) - log.DebugContext(ctx, "Reading the repository metadata...", log.String("name", r.Name), log.FilePath(filePath)) + if !fsutils.FileExists(filePath) { + if err := r.downloadManifest(ctx, Options{}); err != nil { + return Manifest{}, xerrors.Errorf("failed to download the repository metadata: %w", err) + } + } + log.DebugContext(ctx, "Reading the repository metadata...", log.String("repo", r.Name), log.FilePath(filePath)) f, err := os.Open(filePath) if err != nil { return Manifest{}, xerrors.Errorf("failed to open the file: %w", err) @@ -107,7 +118,7 @@ func (r *Repository) Manifest(ctx context.Context) (Manifest, error) { func (r *Repository) Index(ctx context.Context) (Index, error) { filePath := filepath.Join(r.dir, indexFile) - log.DebugContext(ctx, "Reading the repository index...", log.String("name", r.Name), log.FilePath(filePath)) + log.DebugContext(ctx, "Reading the repository index...", log.String("repo", r.Name), log.FilePath(filePath)) f, err := os.Open(filePath) if err != nil { @@ -121,6 +132,7 @@ func (r *Repository) Index(ctx context.Context) (Index, error) { } return Index{ + Path: filePath, UpdatedAt: raw.UpdatedAt, Packages: lo.KeyBy(raw.Packages, func(p PackageEntry) string { return p.ID }), }, nil @@ -137,58 +149,19 @@ func (r *Repository) downloadManifest(ctx context.Context, opts Options) error { } if u.Host == "github.com" { - if err = r.githubGet(ctx, u, r.dir); err != nil { - return xerrors.Errorf("failed to get the repository metadata: %w", err) - } - return nil + u.Path = path.Join(u.Path, manifestFile) + } else { + u.Path = path.Join(u.Path, ".well-known", manifestFile) } - u.Path += path.Join(u.Path, ".well-known", manifestFile) log.DebugContext(ctx, "Downloading the repository metadata...", log.String("url", u.String()), log.String("dst", r.dir)) - if err := downloader.Download(ctx, u.String(), r.dir, ".", opts.Insecure); err != nil { + if _, err = downloader.Download(ctx, u.String(), r.dir, ".", downloader.Options{Insecure: opts.Insecure}); err != nil { + _ = os.RemoveAll(r.dir) return xerrors.Errorf("failed to download the repository: %w", err) } return nil } -func (r *Repository) githubGet(ctx context.Context, u *url.URL, dstDir string) error { - ss := strings.SplitN(u.Path, "/", 4) - if len(ss) < 3 { - return xerrors.Errorf("invalid GitHub URL: %s", u) - } - owner := ss[1] - repo := ss[2] - filePath := manifestFile - if len(ss) == 4 { - filePath = path.Join(ss[3], filePath) - } - - client := github.NewClient(nil) - if t := os.Getenv("GITHUB_TOKEN"); t != "" { - client = client.WithAuthToken(t) - } - - log.DebugContext(ctx, "Downloading the repository metadata from GitHub...", - log.String("owner", owner), log.String("repo", repo), log.String("path", filePath), - log.String("dst", dstDir)) - rc, _, err := client.Repositories.DownloadContents(ctx, owner, repo, filePath, nil) - if err != nil { - return xerrors.Errorf("failed to get the file content: %w", err) - } - defer rc.Close() - - f, err := os.Create(filepath.Join(dstDir, manifestFile)) - if err != nil { - return xerrors.Errorf("failed to create a file: %w", err) - } - defer f.Close() - - if _, err = io.Copy(f, rc); err != nil { - return xerrors.Errorf("failed to copy the file: %w", err) - } - return nil -} - func (r *Repository) Update(ctx context.Context, opts Options) error { manifest, err := r.Manifest(ctx) if err != nil { @@ -207,13 +180,12 @@ func (r *Repository) Update(ctx context.Context, opts Options) error { } versionDir := filepath.Join(r.dir, majorVersion) - if !r.needUpdate(ctx, ver, majorVersion) { - log.InfoContext(ctx, "Need to update repository", log.String("name", r.Name)) + if !r.needUpdate(ctx, ver, versionDir) { + log.InfoContext(ctx, "No need to check repository updates", log.String("repo", r.Name)) return nil } - log.InfoContext(ctx, "Need to update repository", log.String("name", r.Name)) - log.InfoContext(ctx, "Downloading repository...", log.String("name", r.Name), log.String("url", r.URL)) + log.InfoContext(ctx, "Updating repository...", log.String("repo", r.Name), log.String("url", r.URL)) if err = r.download(ctx, ver, versionDir, opts); err != nil { return xerrors.Errorf("failed to download the repository: %w", err) } @@ -225,19 +197,18 @@ func (r *Repository) needUpdate(ctx context.Context, ver Version, versionDir str return true } - index, err := r.Index(ctx) + m, err := r.cacheMetadata() if err != nil { - log.DebugContext(ctx, "Failed to get the repository index", log.String("name", r.Name), log.Err(err)) + log.DebugContext(ctx, "Failed to get repository cache metadata", log.String("repo", r.Name), log.Err(err)) return true } now := clock.Clock(ctx).Now() - if now.After(index.UpdatedAt.Add(ver.UpdateInterval.Duration)) { + log.DebugContext(ctx, "Checking if the repository needs to be updated...", log.String("repo", r.Name), + log.Time("last_update", m.UpdatedAt), log.Duration("update_interval", ver.UpdateInterval.Duration)) + if now.After(m.UpdatedAt.Add(ver.UpdateInterval.Duration)) { return true } - - // TODO: use local metadata.json - return false } @@ -245,19 +216,83 @@ func (r *Repository) download(ctx context.Context, ver Version, dst string, opts if len(ver.Locations) == 0 { return xerrors.Errorf("no locations found for version %s", ver.SpecVersion) } - if err := os.MkdirAll(dst, 0700); err != nil { return xerrors.Errorf("failed to mkdir: %w", err) } + m, err := r.cacheMetadata() + if err != nil { + return xerrors.Errorf("failed to get the repository cache metadata: %w", err) + } + etags := lo.Ternary(m.ETags == nil, map[string]string{}, m.ETags) + var errs error for _, loc := range ver.Locations { - log.DebugContext(ctx, "Downloading repository ...", log.String("url", loc.URL), log.String("dir", dst)) - if err := downloader.Download(ctx, loc.URL, dst, ".", opts.Insecure); err != nil { + logger := log.With(log.String("repo", r.Name)) + logger.DebugContext(ctx, "Downloading repository to cache dir...", log.String("url", loc.URL), + log.String("dir", dst), log.String("etag", etags[loc.URL])) + etag, err := downloader.Download(ctx, loc.URL, dst, ".", downloader.Options{ + Insecure: opts.Insecure, + ETag: etags[loc.URL], + }) + switch { + case errors.Is(err, downloader.ErrSkipDownload): + logger.DebugContext(ctx, "No updates in the repository", log.String("url", r.URL)) + etag = etags[loc.URL] // Keep the old ETag + // Update last updated time so that Trivy will not try to download the same URL soon + case err != nil: errs = errors.Join(errs, err) - } else { - return nil + continue // Try the next location + default: + // Successfully downloaded } + + // Update the cache metadata + etags[loc.URL] = etag + now := clock.Clock(ctx).Now() + if err = r.updateCacheMetadata(ctx, CacheMetadata{ + UpdatedAt: now, + ETags: etags, + }); err != nil { + return xerrors.Errorf("failed to update the repository cache metadata: %w", err) + } + logger.DebugContext(ctx, "Updated repository cache metadata", log.String("etag", etag), + log.Time("updated_at", now)) + return nil } return errs } + +func (r *Repository) cacheMetadata() (CacheMetadata, error) { + filePath := filepath.Join(r.dir, cacheMetadataFile) + if !fsutils.FileExists(filePath) { + return CacheMetadata{}, nil + } + f, err := os.Open(filePath) + if err != nil { + return CacheMetadata{}, xerrors.Errorf("failed to open the file: %w", err) + } + defer f.Close() + + var metadata CacheMetadata + if err = json.NewDecoder(f).Decode(&metadata); err != nil { + return CacheMetadata{}, xerrors.Errorf("failed to decode the cache metadata: %w", err) + } + return metadata, nil +} + +func (r *Repository) updateCacheMetadata(ctx context.Context, metadata CacheMetadata) error { + filePath := filepath.Join(r.dir, cacheMetadataFile) + log.DebugContext(ctx, "Updating repository cache metadata...", log.FilePath(filePath)) + + f, err := os.Create(filePath) + if err != nil { + return xerrors.Errorf("failed to create the file: %w", err) + } + defer f.Close() + + if err = json.NewEncoder(f).Encode(metadata); err != nil { + return xerrors.Errorf("failed to encode the metadata: %w", err) + } + return nil +} From d098fb640715134e75decebfe24e4ae42f0873b0 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 10 Jul 2024 11:38:08 +0400 Subject: [PATCH 18/55] feat: add RepositorySet Signed-off-by: knqyf263 --- pkg/commands/app.go | 2 +- pkg/flag/options.go | 1 + pkg/result/filter.go | 6 ++- pkg/vex/document.go | 98 +++++++++++++++++++++++++++++++++++ pkg/vex/openvex.go | 10 ++-- pkg/vex/repo.go | 105 ++++++++++++++++++++++++++++++++++++++ pkg/vex/repo/repo.go | 11 ++-- pkg/vex/vex.go | 119 +++++++++++++------------------------------ 8 files changed, 254 insertions(+), 98 deletions(-) create mode 100644 pkg/vex/document.go create mode 100644 pkg/vex/repo.go diff --git a/pkg/commands/app.go b/pkg/commands/app.go index 42f687330a53..ca60bd4c170c 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -1237,7 +1237,7 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { Use: "vex subcommand", Aliases: []string{"p"}, GroupID: groupManagement, - Short: "VEX utilities", + Short: "[EXPERIMENTAL] VEX utilities", SilenceErrors: true, SilenceUsage: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { diff --git a/pkg/flag/options.go b/pkg/flag/options.go index 4e8a24a2a2a6..c960617ef29a 100644 --- a/pkg/flag/options.go +++ b/pkg/flag/options.go @@ -445,6 +445,7 @@ func (o *Options) FilterOpts() result.FilterOptions { IgnoreFile: o.IgnoreFile, PolicyFile: o.IgnorePolicy, IgnoreLicenses: o.IgnoredLicenses, + CacheDir: o.CacheDir, VEXPath: o.VEXPath, } } diff --git a/pkg/result/filter.go b/pkg/result/filter.go index f1e038a3782c..8c7a52e4527e 100644 --- a/pkg/result/filter.go +++ b/pkg/result/filter.go @@ -29,6 +29,7 @@ type FilterOptions struct { IgnoreFile string PolicyFile string IgnoreLicenses []string + CacheDir string VEXPath string } @@ -46,7 +47,10 @@ func Filter(ctx context.Context, report types.Report, opts FilterOptions) error } // Filter out vulnerabilities based on the given VEX document. - if err = vex.Filter(&report, vex.Options{VEXPath: opts.VEXPath}); err != nil { + if err = vex.Filter(ctx, &report, vex.Options{ + CacheDir: opts.CacheDir, + VEXPath: opts.VEXPath, + }); err != nil { return xerrors.Errorf("VEX error: %w", err) } diff --git a/pkg/vex/document.go b/pkg/vex/document.go new file mode 100644 index 000000000000..71358fa88dec --- /dev/null +++ b/pkg/vex/document.go @@ -0,0 +1,98 @@ +package vex + +import ( + "encoding/json" + "io" + "os" + + "github.com/csaf-poc/csaf_distribution/v3/csaf" + "github.com/hashicorp/go-multierror" + openvex "github.com/openvex/go-vex/pkg/vex" + "github.com/sirupsen/logrus" + "golang.org/x/xerrors" + + "github.com/aquasecurity/trivy/pkg/fanal/artifact" + "github.com/aquasecurity/trivy/pkg/sbom" + "github.com/aquasecurity/trivy/pkg/sbom/cyclonedx" + "github.com/aquasecurity/trivy/pkg/types" +) + +func NewDocument(filePath string, report *types.Report) (VEX, error) { + if filePath == "" { + return nil, nil + } + f, err := os.Open(filePath) + if err != nil { + return nil, xerrors.Errorf("file open error: %w", err) + } + defer f.Close() + + var errs error + // Try CycloneDX JSON + if ok, err := sbom.IsCycloneDXJSON(f); err != nil { + errs = multierror.Append(errs, err) + } else if ok { + return decodeCycloneDXJSON(f, report) + } + + // Try OpenVEX + if v, err := decodeOpenVEX(f, filePath); err != nil { + errs = multierror.Append(errs, err) + } else if v != nil { + return v, nil + } + + // Try CSAF + if v, err := decodeCSAF(f); err != nil { + errs = multierror.Append(errs, err) + } else if v != nil { + return v, nil + } + + return nil, xerrors.Errorf("unable to load VEX: %w", errs) +} + +func decodeCycloneDXJSON(r io.ReadSeeker, report *types.Report) (*CycloneDX, error) { + if _, err := r.Seek(0, io.SeekStart); err != nil { + return nil, xerrors.Errorf("seek error: %w", err) + } + vex, err := cyclonedx.DecodeJSON(r) + if err != nil { + return nil, xerrors.Errorf("json decode error: %w", err) + } + if report.ArtifactType != artifact.TypeCycloneDX { + return nil, xerrors.New("CycloneDX VEX can be used with CycloneDX SBOM") + } + return newCycloneDX(report.BOM, vex), nil +} + +func decodeOpenVEX(r io.ReadSeeker, source string) (*OpenVEX, error) { + // openvex/go-vex outputs log messages by default + logrus.SetOutput(io.Discard) + + if _, err := r.Seek(0, io.SeekStart); err != nil { + return nil, xerrors.Errorf("seek error: %w", err) + } + var openVEX openvex.VEX + if err := json.NewDecoder(r).Decode(&openVEX); err != nil { + return nil, err + } + if openVEX.Context == "" { + return nil, nil + } + return newOpenVEX(openVEX, source), nil +} + +func decodeCSAF(r io.ReadSeeker) (*CSAF, error) { + if _, err := r.Seek(0, io.SeekStart); err != nil { + return nil, xerrors.Errorf("seek error: %w", err) + } + var adv csaf.Advisory + if err := json.NewDecoder(r).Decode(&adv); err != nil { + return nil, err + } + if adv.Vulnerabilities == nil { + return nil, nil + } + return newCSAF(adv), nil +} diff --git a/pkg/vex/openvex.go b/pkg/vex/openvex.go index 1e62ebfea938..d300de66ff61 100644 --- a/pkg/vex/openvex.go +++ b/pkg/vex/openvex.go @@ -8,12 +8,14 @@ import ( ) type OpenVEX struct { - vex openvex.VEX + vex openvex.VEX + source string } -func newOpenVEX(vex openvex.VEX) *OpenVEX { +func newOpenVEX(vex openvex.VEX, source string) *OpenVEX { return &OpenVEX{ - vex: vex, + vex: vex, + source: source, } } @@ -32,7 +34,7 @@ func (v *OpenVEX) NotAffected(vuln types.DetectedVulnerability, product, subComp // cf. https://github.com/openvex/spec/blob/fa5ba0c0afedb008dc5ebad418548cacf16a3ca7/OPENVEX-SPEC.md#the-vex-statement stmt := stmts[len(stmts)-1] if stmt.Status == openvex.StatusNotAffected || stmt.Status == openvex.StatusFixed { - modifiedFindings := types.NewModifiedFinding(vuln, findingStatus(stmt.Status), string(stmt.Justification), "OpenVEX") + modifiedFindings := types.NewModifiedFinding(vuln, findingStatus(stmt.Status), string(stmt.Justification), v.source) return modifiedFindings, true } return types.ModifiedFinding{}, false diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go new file mode 100644 index 000000000000..70d5e421c54b --- /dev/null +++ b/pkg/vex/repo.go @@ -0,0 +1,105 @@ +package vex + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "golang.org/x/xerrors" + + "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/sbom/core" + "github.com/aquasecurity/trivy/pkg/types" + "github.com/aquasecurity/trivy/pkg/vex/repo" +) + +// RepositoryIndex wraps the repository index +type RepositoryIndex struct { + Name string + URL string + repo.Index +} + +type RepositorySet struct { + indexes []RepositoryIndex + logger *log.Logger +} + +func NewRepositorySet(ctx context.Context, cacheDir string) (*RepositorySet, error) { + conf, err := repo.NewManager(cacheDir).Config(ctx) + if err != nil { + return nil, xerrors.Errorf("failed to get VEX repository config: %w", err) + } + + var indexes []RepositoryIndex + for _, r := range conf.Repositories { + index, err := r.Index(ctx) + if err != nil { + return nil, xerrors.Errorf("failed to get VEX repository index: %w", err) + } + indexes = append(indexes, RepositoryIndex{ + Name: r.Name, + URL: r.URL, + Index: index, + }) + } + return &RepositorySet{ + indexes: indexes, // In precedence order + logger: log.WithPrefix("vex"), + }, nil +} + +func (rs *RepositorySet) Filter(result *types.Result, bom *core.BOM) { + filterVulnerabilities(result, bom, rs.NotAffected) +} + +func (rs *RepositorySet) NotAffected(vuln types.DetectedVulnerability, product, subComponent *core.Component) (types.ModifiedFinding, bool) { + if product == nil || product.PkgIdentifier.PURL == nil { + return types.ModifiedFinding{}, false + } + p := *product.PkgIdentifier.PURL + p.Version = "" + p.Qualifiers = nil + p.Subpath = "" + + pkgID := p.String() // PURL without version, qualifiers, and subpath + for _, index := range rs.indexes { + entry, ok := index.Packages[pkgID] + if !ok { + continue + } + source := fmt.Sprintf("VEX Repository: %s (%s)", index.Name, index.URL) + doc, err := rs.OpenDocument(source, filepath.Dir(index.Path), entry) + if err != nil { + log.Warn("Failed to open the VEX document", log.String("location", entry.Location), log.Err(err)) + return types.ModifiedFinding{}, false + } + + if m, notAffected := doc.NotAffected(vuln, product, subComponent); notAffected { + return m, notAffected + } + + log.Debug("VEX found, but affected", log.String("vulnerability", vuln.VulnerabilityID), + log.String("package", pkgID), log.String("repo", index.Name), log.String("repo_url", index.URL)) + break // Stop searching for the next VEX document as this repository has higher precedence. + } + return types.ModifiedFinding{}, false +} + +func (rs *RepositorySet) OpenDocument(source, dir string, entry repo.PackageEntry) (VEX, error) { + f, err := os.Open(filepath.Join(dir, entry.Location)) + if err != nil { + return nil, xerrors.Errorf("failed to open the VEX document: %w", err) + } + defer f.Close() + + switch entry.Format { + case "openvex", "": + return decodeOpenVEX(f, source) + case "csaf": + return decodeCSAF(f) + default: + return nil, xerrors.Errorf("unsupported VEX format: %s", entry.Format) + } +} diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index f00efdca62ce..22b540c8fa16 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -21,13 +21,15 @@ import ( ) const ( - SchemaVersion = "0.1" + SchemaVersion = "v0.1" manifestFile = "vex-repository.json" indexFile = "index.json" cacheMetadataFile = "cache.json" ) +var majorVersion, _, _ = strings.Cut(SchemaVersion, ".") + type Manifest struct { Name string `json:"name"` Description string `json:"description"` @@ -117,7 +119,7 @@ func (r *Repository) Manifest(ctx context.Context) (Manifest, error) { } func (r *Repository) Index(ctx context.Context) (Index, error) { - filePath := filepath.Join(r.dir, indexFile) + filePath := filepath.Join(r.dir, majorVersion, indexFile) log.DebugContext(ctx, "Reading the repository index...", log.String("repo", r.Name), log.FilePath(filePath)) f, err := os.Open(filePath) @@ -168,11 +170,6 @@ func (r *Repository) Update(ctx context.Context, opts Options) error { return xerrors.Errorf("failed to get the repository metadata: %w", err) } - majorVersion, _, ok := strings.Cut(SchemaVersion, ".") - if !ok { - return xerrors.New("invalid schema version") - } - majorVersion = "v" + majorVersion ver, ok := manifest.Versions[majorVersion] if !ok { // TODO: improve error diff --git a/pkg/vex/vex.go b/pkg/vex/vex.go index 52dda5082234..4c02242bb643 100644 --- a/pkg/vex/vex.go +++ b/pkg/vex/vex.go @@ -1,25 +1,18 @@ package vex import ( - "encoding/json" - "io" - "os" + "context" + "errors" - "github.com/csaf-poc/csaf_distribution/v3/csaf" - "github.com/hashicorp/go-multierror" - openvex "github.com/openvex/go-vex/pkg/vex" "github.com/samber/lo" - "github.com/sirupsen/logrus" "golang.org/x/xerrors" - "github.com/aquasecurity/trivy/pkg/fanal/artifact" "github.com/aquasecurity/trivy/pkg/log" - "github.com/aquasecurity/trivy/pkg/sbom" "github.com/aquasecurity/trivy/pkg/sbom/core" - "github.com/aquasecurity/trivy/pkg/sbom/cyclonedx" sbomio "github.com/aquasecurity/trivy/pkg/sbom/io" "github.com/aquasecurity/trivy/pkg/types" "github.com/aquasecurity/trivy/pkg/uuid" + vexrepo "github.com/aquasecurity/trivy/pkg/vex/repo" ) // VEX represents Vulnerability Exploitability eXchange. It abstracts multiple VEX formats. @@ -29,18 +22,25 @@ type VEX interface { NotAffected(vuln types.DetectedVulnerability, product, subComponent *core.Component) (types.ModifiedFinding, bool) } +type Client struct { + VEXes []VEX +} + type Options struct { - VEXPath string + CacheDir string + VEXPath string } +type NotAffected func(vuln types.DetectedVulnerability, product, subComponent *core.Component) (types.ModifiedFinding, bool) + // Filter determines whether a detected vulnerability should be filtered out based on the provided VEX document. // If the VEX document is passed and the vulnerability is either not affected or fixed according to the VEX statement, // the vulnerability is filtered out. -func Filter(report *types.Report, opts Options) error { - vexDoc, err := New(opts.VEXPath, report) +func Filter(ctx context.Context, report *types.Report, opts Options) error { + client, err := New(ctx, report, opts) if err != nil { - return xerrors.Errorf("unable to load VEX: %w", err) - } else if vexDoc == nil { + return xerrors.Errorf("VEX error: %w", err) + } else if client == nil { return nil } @@ -53,93 +53,42 @@ func Filter(report *types.Report, opts Options) error { if len(result.Vulnerabilities) == 0 { continue } - filterVulnerabilities(&report.Results[i], bom, vexDoc.NotAffected) + filterVulnerabilities(&report.Results[i], bom, client.NotAffected) } return nil } -func New(filePath string, report *types.Report) (VEX, error) { - if filePath == "" { - return nil, nil - } - f, err := os.Open(filePath) +func New(ctx context.Context, report *types.Report, opts Options) (*Client, error) { + var vexes []VEX + v, err := NewDocument(opts.VEXPath, report) if err != nil { - return nil, xerrors.Errorf("file open error: %w", err) - } - defer f.Close() - - var errs error - // Try CycloneDX JSON - if ok, err := sbom.IsCycloneDXJSON(f); err != nil { - errs = multierror.Append(errs, err) - } else if ok { - return decodeCycloneDXJSON(f, report) - } - - // Try OpenVEX - if v, err := decodeOpenVEX(f); err != nil { - errs = multierror.Append(errs, err) + return nil, xerrors.Errorf("unable to load VEX: %w", err) } else if v != nil { - return v, nil + vexes = append(vexes, v) } - // Try CSAF - if v, err := decodeCSAF(f); err != nil { - errs = multierror.Append(errs, err) - } else if v != nil { - return v, nil + rs, err := NewRepositorySet(ctx, opts.CacheDir) + if !errors.Is(err, vexrepo.ErrNoConfig) && err != nil { + return nil, xerrors.Errorf("failed to create a repository set: %w", err) + } else if rs != nil { + vexes = append(vexes, rs) } - return nil, xerrors.Errorf("unable to load VEX: %w", errs) -} - -func decodeCycloneDXJSON(r io.ReadSeeker, report *types.Report) (*CycloneDX, error) { - if _, err := r.Seek(0, io.SeekStart); err != nil { - return nil, xerrors.Errorf("seek error: %w", err) - } - vex, err := cyclonedx.DecodeJSON(r) - if err != nil { - return nil, xerrors.Errorf("json decode error: %w", err) - } - if report.ArtifactType != artifact.TypeCycloneDX { - return nil, xerrors.New("CycloneDX VEX can be used with CycloneDX SBOM") - } - return newCycloneDX(report.BOM, vex), nil -} - -func decodeOpenVEX(r io.ReadSeeker) (*OpenVEX, error) { - // openvex/go-vex outputs log messages by default - logrus.SetOutput(io.Discard) - - if _, err := r.Seek(0, io.SeekStart); err != nil { - return nil, xerrors.Errorf("seek error: %w", err) - } - var openVEX openvex.VEX - if err := json.NewDecoder(r).Decode(&openVEX); err != nil { - return nil, err - } - if openVEX.Context == "" { + if len(vexes) == 0 { return nil, nil } - return newOpenVEX(openVEX), nil + return &Client{VEXes: vexes}, nil } -func decodeCSAF(r io.ReadSeeker) (*CSAF, error) { - if _, err := r.Seek(0, io.SeekStart); err != nil { - return nil, xerrors.Errorf("seek error: %w", err) - } - var adv csaf.Advisory - if err := json.NewDecoder(r).Decode(&adv); err != nil { - return nil, err - } - if adv.Vulnerabilities == nil { - return nil, nil +func (c *Client) NotAffected(vuln types.DetectedVulnerability, product, subComponent *core.Component) (types.ModifiedFinding, bool) { + for _, v := range c.VEXes { + if m, notAffected := v.NotAffected(vuln, product, subComponent); notAffected { + return m, true + } } - return newCSAF(adv), nil + return types.ModifiedFinding{}, false } -type NotAffected func(vuln types.DetectedVulnerability, product, subComponent *core.Component) (types.ModifiedFinding, bool) - func filterVulnerabilities(result *types.Result, bom *core.BOM, fn NotAffected) { components := lo.MapEntries(bom.Components(), func(id uuid.UUID, component *core.Component) (string, *core.Component) { return component.PkgIdentifier.UID, component From 430954e59ca55e6257475a7fe6c294f0fc0bbc38 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 10 Jul 2024 11:40:41 +0400 Subject: [PATCH 19/55] feat(csaf): support VEX source Signed-off-by: knqyf263 --- pkg/vex/csaf.go | 6 ++++-- pkg/vex/document.go | 6 +++--- pkg/vex/repo.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/vex/csaf.go b/pkg/vex/csaf.go index 99f45b19a7cb..1f9c91fdd001 100644 --- a/pkg/vex/csaf.go +++ b/pkg/vex/csaf.go @@ -12,6 +12,7 @@ import ( type CSAF struct { advisory csaf.Advisory + source string logger *log.Logger } @@ -20,9 +21,10 @@ type relationship struct { SubProducts []*purl.PackageURL } -func newCSAF(advisory csaf.Advisory) *CSAF { +func newCSAF(advisory csaf.Advisory, source string) *CSAF { return &CSAF{ advisory: advisory, + source: source, logger: log.WithPrefix("vex").With(log.String("format", "CSAF")), } } @@ -43,7 +45,7 @@ func (v *CSAF) NotAffected(vuln types.DetectedVulnerability, product, subProduct if status == "" { return types.ModifiedFinding{}, false } - return types.NewModifiedFinding(vuln, status, v.statement(found), "CSAF VEX"), true + return types.NewModifiedFinding(vuln, status, v.statement(found), v.source), true } func (v *CSAF) match(vuln *csaf.Vulnerability, product, subProduct *core.Component) types.FindingStatus { diff --git a/pkg/vex/document.go b/pkg/vex/document.go index 71358fa88dec..450c1676f7ab 100644 --- a/pkg/vex/document.go +++ b/pkg/vex/document.go @@ -43,7 +43,7 @@ func NewDocument(filePath string, report *types.Report) (VEX, error) { } // Try CSAF - if v, err := decodeCSAF(f); err != nil { + if v, err := decodeCSAF(f, filePath); err != nil { errs = multierror.Append(errs, err) } else if v != nil { return v, nil @@ -83,7 +83,7 @@ func decodeOpenVEX(r io.ReadSeeker, source string) (*OpenVEX, error) { return newOpenVEX(openVEX, source), nil } -func decodeCSAF(r io.ReadSeeker) (*CSAF, error) { +func decodeCSAF(r io.ReadSeeker, source string) (*CSAF, error) { if _, err := r.Seek(0, io.SeekStart); err != nil { return nil, xerrors.Errorf("seek error: %w", err) } @@ -94,5 +94,5 @@ func decodeCSAF(r io.ReadSeeker) (*CSAF, error) { if adv.Vulnerabilities == nil { return nil, nil } - return newCSAF(adv), nil + return newCSAF(adv, source), nil } diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go index 70d5e421c54b..b3ceb75a060c 100644 --- a/pkg/vex/repo.go +++ b/pkg/vex/repo.go @@ -98,7 +98,7 @@ func (rs *RepositorySet) OpenDocument(source, dir string, entry repo.PackageEntr case "openvex", "": return decodeOpenVEX(f, source) case "csaf": - return decodeCSAF(f) + return decodeCSAF(f, source) default: return nil, xerrors.Errorf("unsupported VEX format: %s", entry.Format) } From 34ade4b75ccd72ccbac7eb5ee276830d2c3d2d52 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 10 Jul 2024 12:44:02 +0400 Subject: [PATCH 20/55] test: fix sources Signed-off-by: knqyf263 --- pkg/vex/vex_test.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/vex/vex_test.go b/pkg/vex/vex_test.go index febeee729c01..1816b27d535c 100644 --- a/pkg/vex/vex_test.go +++ b/pkg/vex/vex_test.go @@ -1,6 +1,7 @@ package vex_test import ( + "context" "os" "testing" @@ -171,7 +172,7 @@ func TestFilter(t *testing.T) { want: imageReport([]types.Result{ springResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{}, - ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln1, vulnerableCodeNotInExecutePath, "OpenVEX")}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln1, vulnerableCodeNotInExecutePath, "testdata/openvex.json")}, }), }), }, @@ -195,7 +196,7 @@ func TestFilter(t *testing.T) { want: imageReport([]types.Result{ springResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{vuln2}, - ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln1, vulnerableCodeNotInExecutePath, "OpenVEX")}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln1, vulnerableCodeNotInExecutePath, "testdata/openvex-multiple.json")}, }), }), }, @@ -218,7 +219,7 @@ func TestFilter(t *testing.T) { want: imageReport([]types.Result{ bashResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{}, - ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln3, vulnerableCodeNotInExecutePath, "OpenVEX")}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln3, vulnerableCodeNotInExecutePath, "testdata/openvex-oci.json")}, }), }), }, @@ -257,7 +258,7 @@ func TestFilter(t *testing.T) { want: fsReport([]types.Result{ goSinglePathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{}, - ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "OpenVEX")}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "testdata/openvex-nested.json")}, }), }), }, @@ -363,7 +364,7 @@ func TestFilter(t *testing.T) { want: imageReport([]types.Result{ goSinglePathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{}, - ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "CSAF VEX")}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "testdata/csaf.json")}, }), }), }, @@ -384,7 +385,7 @@ func TestFilter(t *testing.T) { want: imageReport([]types.Result{ goSinglePathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{}, - ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "CSAF VEX")}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "testdata/csaf-relationships.json")}, }), }), }, @@ -422,7 +423,7 @@ func TestFilter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := vex.Filter(tt.args.report, tt.args.opts) + err := vex.Filter(context.Background(), tt.args.report, tt.args.opts) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return From 33f820761a280b0fb10a1dc6f1820f2e28899d5d Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 10 Jul 2024 13:31:49 +0400 Subject: [PATCH 21/55] test(vex): add tests for repository set Signed-off-by: knqyf263 --- pkg/vex/repo.go | 4 - pkg/vex/repo_test.go | 134 ++++++++++++++++++ .../vex/repositories/default/v0/index.json | 10 ++ .../repositories/default/v0/openssl-vex.json | 22 +++ .../repositories/high-priority/v0/index.json | 10 ++ .../high-priority/v0/openssl-vex.json | 21 +++ .../vex/repositories/default/v0/index.json | 10 ++ .../repositories/default/v0/openssl-vex.json | 22 +++ 8 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 pkg/vex/repo_test.go create mode 100644 pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json create mode 100644 pkg/vex/testdata/multi-repos/vex/repositories/default/v0/openssl-vex.json create mode 100644 pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json create mode 100644 pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/openssl-vex.json create mode 100644 pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json create mode 100644 pkg/vex/testdata/single-repo/vex/repositories/default/v0/openssl-vex.json diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go index b3ceb75a060c..ea8e5f6c1139 100644 --- a/pkg/vex/repo.go +++ b/pkg/vex/repo.go @@ -50,10 +50,6 @@ func NewRepositorySet(ctx context.Context, cacheDir string) (*RepositorySet, err }, nil } -func (rs *RepositorySet) Filter(result *types.Result, bom *core.BOM) { - filterVulnerabilities(result, bom, rs.NotAffected) -} - func (rs *RepositorySet) NotAffected(vuln types.DetectedVulnerability, product, subComponent *core.Component) (types.ModifiedFinding, bool) { if product == nil || product.PkgIdentifier.PURL == nil { return types.ModifiedFinding{}, false diff --git a/pkg/vex/repo_test.go b/pkg/vex/repo_test.go new file mode 100644 index 000000000000..05a62e1a30d9 --- /dev/null +++ b/pkg/vex/repo_test.go @@ -0,0 +1,134 @@ +package vex_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/package-url/packageurl-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/sbom/core" + "github.com/aquasecurity/trivy/pkg/types" + "github.com/aquasecurity/trivy/pkg/vex" +) + +var ( + opensslComponent = core.Component{ + Name: "openssl", + Version: "1.1.1t-r0", + PkgIdentifier: ftypes.PkgIdentifier{ + UID: "01", + PURL: &packageurl.PackageURL{ + Type: "deb", + Namespace: "debian", + Name: "openssl", + Version: "1.1.1t-r0", + }, + }, + } + opensslVuln1 = types.DetectedVulnerability{ + VulnerabilityID: "CVE-2023-1234", + PkgName: opensslComponent.Name, + InstalledVersion: opensslComponent.Version, + FixedVersion: "1.1.1u-r0", + PkgIdentifier: opensslComponent.PkgIdentifier, + } + opensslVuln2 = types.DetectedVulnerability{ + VulnerabilityID: "CVE-2023-12345", + PkgName: opensslComponent.Name, + InstalledVersion: opensslComponent.Version, + FixedVersion: "1.1.1u-r2", + PkgIdentifier: opensslComponent.PkgIdentifier, + } +) + +func TestRepositorySet_NotAffected(t *testing.T) { + tests := []struct { + name string + cacheDir string + configContent string + vuln types.DetectedVulnerability + product core.Component + wantModified types.ModifiedFinding + wantNotAffected bool + }{ + { + name: "single repository - not affected", + cacheDir: "testdata/single-repo", + configContent: ` +repositories: + - name: default + url: https://example.com/vex/default +`, + vuln: opensslVuln1, + product: opensslComponent, + wantModified: types.ModifiedFinding{ + Type: types.FindingTypeVulnerability, + Finding: opensslVuln1, + Status: types.FindingStatusNotAffected, + Statement: "vulnerable_code_not_in_execute_path", + Source: "VEX Repository: default (https://example.com/vex/default)", + }, + wantNotAffected: true, + }, + { + name: "multiple repositories - high priority affected", + cacheDir: "testdata/multi-repos", + configContent: ` +repositories: + - name: high-priority + url: https://example.com/vex/high-priority + - name: default + url: https://example.com/vex/default`, + vuln: opensslVuln1, + product: opensslComponent, + wantNotAffected: false, + }, + { + name: "no matching VEX data", + cacheDir: "testdata/single-repo", + configContent: ` +repositories: + - name: default + url: https://example.com/vex/default +`, + vuln: opensslVuln2, + product: opensslComponent, + wantNotAffected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a temporary directory for each test + tmpDir := t.TempDir() + + // Set XDG_DATA_HOME to the temporary directory + t.Setenv("XDG_DATA_HOME", tmpDir) + + // Create the vex directory in the temporary directory + vexDir := filepath.Join(tmpDir, ".trivy", "vex") + err := os.MkdirAll(vexDir, 0755) + require.NoError(t, err) + + // Write the config file + configPath := filepath.Join(vexDir, "repository.yaml") + err = os.WriteFile(configPath, []byte(tt.configContent), 0644) + require.NoError(t, err) + + ctx := context.Background() + rs, err := vex.NewRepositorySet(ctx, tt.cacheDir) + require.NoError(t, err) + + modified, notAffected := rs.NotAffected(tt.vuln, &tt.product, nil) + assert.Equal(t, tt.wantNotAffected, notAffected) + if tt.wantNotAffected { + assert.Equal(t, tt.wantModified, modified) + } + }) + } +} diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json b/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json new file mode 100644 index 000000000000..ffe87393b7c9 --- /dev/null +++ b/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json @@ -0,0 +1,10 @@ +{ + "updated_at": "2024-07-01T00:00:00Z", + "packages": [ + { + "id": "pkg:deb/debian/openssl", + "location": "openssl-vex.json", + "format": "openvex" + } + ] +} \ No newline at end of file diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/openssl-vex.json b/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/openssl-vex.json new file mode 100644 index 000000000000..99345699faa7 --- /dev/null +++ b/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/openssl-vex.json @@ -0,0 +1,22 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "https://openvex.dev/docs/public/vex-5d6e2706", + "author": "Example Author", + "role": "Document Creator", + "timestamp": "2023-07-01T00:00:00Z", + "version": 1, + "statements": [ + { + "vulnerability": { + "@id": "CVE-2023-1234" + }, + "products": [ + { + "@id": "pkg:deb/debian/openssl@1.1.1t-r0" + } + ], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path" + } + ] +} \ No newline at end of file diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json new file mode 100644 index 000000000000..ffe87393b7c9 --- /dev/null +++ b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json @@ -0,0 +1,10 @@ +{ + "updated_at": "2024-07-01T00:00:00Z", + "packages": [ + { + "id": "pkg:deb/debian/openssl", + "location": "openssl-vex.json", + "format": "openvex" + } + ] +} \ No newline at end of file diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/openssl-vex.json b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/openssl-vex.json new file mode 100644 index 000000000000..11db6a59bf07 --- /dev/null +++ b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/openssl-vex.json @@ -0,0 +1,21 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "https://openvex.dev/docs/public/vex-5d6e2706", + "author": "Example Author", + "role": "Document Creator", + "timestamp": "2023-07-01T00:00:00Z", + "version": 1, + "statements": [ + { + "vulnerability": { + "@id": "CVE-2023-1234" + }, + "products": [ + { + "@id": "pkg:deb/debian/openssl@1.1.1t-r0" + } + ], + "status": "affected" + } + ] +} \ No newline at end of file diff --git a/pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json b/pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json new file mode 100644 index 000000000000..ffe87393b7c9 --- /dev/null +++ b/pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json @@ -0,0 +1,10 @@ +{ + "updated_at": "2024-07-01T00:00:00Z", + "packages": [ + { + "id": "pkg:deb/debian/openssl", + "location": "openssl-vex.json", + "format": "openvex" + } + ] +} \ No newline at end of file diff --git a/pkg/vex/testdata/single-repo/vex/repositories/default/v0/openssl-vex.json b/pkg/vex/testdata/single-repo/vex/repositories/default/v0/openssl-vex.json new file mode 100644 index 000000000000..99345699faa7 --- /dev/null +++ b/pkg/vex/testdata/single-repo/vex/repositories/default/v0/openssl-vex.json @@ -0,0 +1,22 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "https://openvex.dev/docs/public/vex-5d6e2706", + "author": "Example Author", + "role": "Document Creator", + "timestamp": "2023-07-01T00:00:00Z", + "version": 1, + "statements": [ + { + "vulnerability": { + "@id": "CVE-2023-1234" + }, + "products": [ + { + "@id": "pkg:deb/debian/openssl@1.1.1t-r0" + } + ], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path" + } + ] +} \ No newline at end of file From 3d6e0e30d4b39e18c66e054cfb61ab3283fc3249 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 10 Jul 2024 14:07:20 +0400 Subject: [PATCH 22/55] test: add more vexr tests Signed-off-by: knqyf263 --- pkg/vex/repo_test.go | 50 +++-------- .../v0/{openssl-vex.json => bash-vex.json} | 4 +- .../vex/repositories/default/v0/index.json | 4 +- .../v0/{openssl-vex.json => bash-vex.json} | 4 +- .../repositories/high-priority/v0/index.json | 4 +- .../v0/{openssl-vex.json => bash-vex.json} | 4 +- .../vex/repositories/default/v0/index.json | 4 +- pkg/vex/vex_test.go | 88 ++++++++++++++++--- 8 files changed, 102 insertions(+), 60 deletions(-) rename pkg/vex/testdata/multi-repos/vex/repositories/default/v0/{openssl-vex.json => bash-vex.json} (83%) rename pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/{openssl-vex.json => bash-vex.json} (81%) rename pkg/vex/testdata/single-repo/vex/repositories/default/v0/{openssl-vex.json => bash-vex.json} (83%) diff --git a/pkg/vex/repo_test.go b/pkg/vex/repo_test.go index 05a62e1a30d9..54682e80cccc 100644 --- a/pkg/vex/repo_test.go +++ b/pkg/vex/repo_test.go @@ -6,45 +6,19 @@ import ( "path/filepath" "testing" - "github.com/package-url/packageurl-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/sbom/core" "github.com/aquasecurity/trivy/pkg/types" "github.com/aquasecurity/trivy/pkg/vex" ) -var ( - opensslComponent = core.Component{ - Name: "openssl", - Version: "1.1.1t-r0", - PkgIdentifier: ftypes.PkgIdentifier{ - UID: "01", - PURL: &packageurl.PackageURL{ - Type: "deb", - Namespace: "debian", - Name: "openssl", - Version: "1.1.1t-r0", - }, - }, - } - opensslVuln1 = types.DetectedVulnerability{ - VulnerabilityID: "CVE-2023-1234", - PkgName: opensslComponent.Name, - InstalledVersion: opensslComponent.Version, - FixedVersion: "1.1.1u-r0", - PkgIdentifier: opensslComponent.PkgIdentifier, - } - opensslVuln2 = types.DetectedVulnerability{ - VulnerabilityID: "CVE-2023-12345", - PkgName: opensslComponent.Name, - InstalledVersion: opensslComponent.Version, - FixedVersion: "1.1.1u-r2", - PkgIdentifier: opensslComponent.PkgIdentifier, - } -) +var bashComponent = core.Component{ + Name: bashPackage.Name, + Version: bashPackage.Version, + PkgIdentifier: bashPackage.Identifier, +} func TestRepositorySet_NotAffected(t *testing.T) { tests := []struct { @@ -64,11 +38,11 @@ repositories: - name: default url: https://example.com/vex/default `, - vuln: opensslVuln1, - product: opensslComponent, + vuln: vuln3, + product: bashComponent, wantModified: types.ModifiedFinding{ Type: types.FindingTypeVulnerability, - Finding: opensslVuln1, + Finding: vuln3, Status: types.FindingStatusNotAffected, Statement: "vulnerable_code_not_in_execute_path", Source: "VEX Repository: default (https://example.com/vex/default)", @@ -84,8 +58,8 @@ repositories: url: https://example.com/vex/high-priority - name: default url: https://example.com/vex/default`, - vuln: opensslVuln1, - product: opensslComponent, + vuln: vuln3, + product: bashComponent, wantNotAffected: false, }, { @@ -96,8 +70,8 @@ repositories: - name: default url: https://example.com/vex/default `, - vuln: opensslVuln2, - product: opensslComponent, + vuln: vuln4, + product: bashComponent, wantNotAffected: false, }, } diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/openssl-vex.json b/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/bash-vex.json similarity index 83% rename from pkg/vex/testdata/multi-repos/vex/repositories/default/v0/openssl-vex.json rename to pkg/vex/testdata/multi-repos/vex/repositories/default/v0/bash-vex.json index 99345699faa7..c06426bac376 100644 --- a/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/openssl-vex.json +++ b/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/bash-vex.json @@ -8,11 +8,11 @@ "statements": [ { "vulnerability": { - "@id": "CVE-2023-1234" + "@id": "CVE-2022-3715" }, "products": [ { - "@id": "pkg:deb/debian/openssl@1.1.1t-r0" + "@id": "pkg:deb/debian/bash@5.3" } ], "status": "not_affected", diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json b/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json index ffe87393b7c9..4a746cca5382 100644 --- a/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json +++ b/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json @@ -2,8 +2,8 @@ "updated_at": "2024-07-01T00:00:00Z", "packages": [ { - "id": "pkg:deb/debian/openssl", - "location": "openssl-vex.json", + "id": "pkg:deb/debian/bash", + "location": "bash-vex.json", "format": "openvex" } ] diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/openssl-vex.json b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/bash-vex.json similarity index 81% rename from pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/openssl-vex.json rename to pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/bash-vex.json index 11db6a59bf07..300ecc8e7c3a 100644 --- a/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/openssl-vex.json +++ b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/bash-vex.json @@ -8,11 +8,11 @@ "statements": [ { "vulnerability": { - "@id": "CVE-2023-1234" + "@id": "CVE-2022-3715" }, "products": [ { - "@id": "pkg:deb/debian/openssl@1.1.1t-r0" + "@id": "pkg:deb/debian/bash@5.3" } ], "status": "affected" diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json index ffe87393b7c9..4a746cca5382 100644 --- a/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json +++ b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json @@ -2,8 +2,8 @@ "updated_at": "2024-07-01T00:00:00Z", "packages": [ { - "id": "pkg:deb/debian/openssl", - "location": "openssl-vex.json", + "id": "pkg:deb/debian/bash", + "location": "bash-vex.json", "format": "openvex" } ] diff --git a/pkg/vex/testdata/single-repo/vex/repositories/default/v0/openssl-vex.json b/pkg/vex/testdata/single-repo/vex/repositories/default/v0/bash-vex.json similarity index 83% rename from pkg/vex/testdata/single-repo/vex/repositories/default/v0/openssl-vex.json rename to pkg/vex/testdata/single-repo/vex/repositories/default/v0/bash-vex.json index 99345699faa7..c06426bac376 100644 --- a/pkg/vex/testdata/single-repo/vex/repositories/default/v0/openssl-vex.json +++ b/pkg/vex/testdata/single-repo/vex/repositories/default/v0/bash-vex.json @@ -8,11 +8,11 @@ "statements": [ { "vulnerability": { - "@id": "CVE-2023-1234" + "@id": "CVE-2022-3715" }, "products": [ { - "@id": "pkg:deb/debian/openssl@1.1.1t-r0" + "@id": "pkg:deb/debian/bash@5.3" } ], "status": "not_affected", diff --git a/pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json b/pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json index ffe87393b7c9..4a746cca5382 100644 --- a/pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json +++ b/pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json @@ -2,8 +2,8 @@ "updated_at": "2024-07-01T00:00:00Z", "packages": [ { - "id": "pkg:deb/debian/openssl", - "location": "openssl-vex.json", + "id": "pkg:deb/debian/bash", + "location": "bash-vex.json", "format": "openvex" } ] diff --git a/pkg/vex/vex_test.go b/pkg/vex/vex_test.go index 1816b27d535c..3fa30fd79153 100644 --- a/pkg/vex/vex_test.go +++ b/pkg/vex/vex_test.go @@ -3,6 +3,7 @@ package vex_test import ( "context" "os" + "path/filepath" "testing" "github.com/google/go-containerregistry/pkg/v1" @@ -132,6 +133,12 @@ var ( PkgIdentifier: bashPackage.Identifier, } vuln4 = types.DetectedVulnerability{ + VulnerabilityID: "CVE-2024-10000", + PkgName: bashPackage.Name, + InstalledVersion: bashPackage.Version, + PkgIdentifier: bashPackage.Identifier, + } + vuln5 = types.DetectedVulnerability{ VulnerabilityID: "CVE-2024-0001", PkgName: goTransitivePackage.Name, InstalledVersion: goTransitivePackage.Version, @@ -151,6 +158,7 @@ func TestFilter(t *testing.T) { } tests := []struct { name string + setup func(t *testing.T, tmpDir string) args args want *types.Report wantErr string @@ -247,7 +255,7 @@ func TestFilter(t *testing.T) { report: fsReport([]types.Result{ goSinglePathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{ - vuln4, // filtered by VEX + vuln5, // filtered by VEX }, }), }), @@ -258,7 +266,7 @@ func TestFilter(t *testing.T) { want: fsReport([]types.Result{ goSinglePathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{}, - ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "testdata/openvex-nested.json")}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln5, vulnerableCodeNotInExecutePath, "testdata/openvex-nested.json")}, }), }), }, @@ -268,7 +276,7 @@ func TestFilter(t *testing.T) { report: fsReport([]types.Result{ goMultiPathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{ - vuln4, + vuln5, }, }), }), @@ -278,7 +286,7 @@ func TestFilter(t *testing.T) { }, want: fsReport([]types.Result{ goMultiPathResult(types.Result{ - Vulnerabilities: []types.DetectedVulnerability{vuln4}, // Will not be filtered because of multi paths + Vulnerabilities: []types.DetectedVulnerability{vuln5}, // Will not be filtered because of multi paths }), }), }, @@ -353,7 +361,7 @@ func TestFilter(t *testing.T) { report: imageReport([]types.Result{ goSinglePathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{ - vuln4, // filtered by VEX + vuln5, // filtered by VEX }, }), }), @@ -364,7 +372,7 @@ func TestFilter(t *testing.T) { want: imageReport([]types.Result{ goSinglePathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{}, - ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "testdata/csaf.json")}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln5, vulnerableCodeNotInExecutePath, "testdata/csaf.json")}, }), }), }, @@ -374,7 +382,7 @@ func TestFilter(t *testing.T) { report: imageReport([]types.Result{ goSinglePathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{ - vuln4, // filtered by VEX + vuln5, // filtered by VEX }, }), }), @@ -385,7 +393,7 @@ func TestFilter(t *testing.T) { want: imageReport([]types.Result{ goSinglePathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{}, - ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln4, vulnerableCodeNotInExecutePath, "testdata/csaf-relationships.json")}, + ModifiedFindings: []types.ModifiedFinding{modifiedFinding(vuln5, vulnerableCodeNotInExecutePath, "testdata/csaf-relationships.json")}, }), }), }, @@ -395,7 +403,7 @@ func TestFilter(t *testing.T) { report: imageReport([]types.Result{ goMultiPathResult(types.Result{ Vulnerabilities: []types.DetectedVulnerability{ - vuln4, + vuln5, }, }), }), @@ -405,7 +413,62 @@ func TestFilter(t *testing.T) { }, want: imageReport([]types.Result{ goMultiPathResult(types.Result{ - Vulnerabilities: []types.DetectedVulnerability{vuln4}, // Will not be filtered because of multi paths + Vulnerabilities: []types.DetectedVulnerability{vuln5}, // Will not be filtered because of multi paths + }), + }), + }, + { + name: "VEX Repository", + setup: func(t *testing.T, tmpDir string) { + // Create repository.yaml + vexDir := filepath.Join(tmpDir, ".trivy", "vex") + require.NoError(t, os.MkdirAll(vexDir, 0755)) + + configPath := filepath.Join(vexDir, "repository.yaml") + configContent := ` +repositories: + - name: default + url: https://example.com/vex/default` + require.NoError(t, os.WriteFile(configPath, []byte(configContent), 0644)) + }, + args: args{ + report: imageReport([]types.Result{ + bashResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{ + vuln3, // filtered by VEX + }, + }), + }), + opts: vex.Options{ + CacheDir: "testdata/single-repo", + }, + }, + want: imageReport([]types.Result{ + bashResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{}, + ModifiedFindings: []types.ModifiedFinding{ + modifiedFinding(vuln3, "vulnerable_code_not_in_execute_path", "VEX Repository: default (https://example.com/vex/default)"), + }, + }), + }), + }, + { + name: "VEX Repository without config", + args: args{ + report: imageReport([]types.Result{ + bashResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{ + vuln3, // not filtered by VEX + }, + }), + }), + opts: vex.Options{ + CacheDir: "testdata/no-repo", + }, + }, + want: imageReport([]types.Result{ + bashResult(types.Result{ + Vulnerabilities: []types.DetectedVulnerability{vuln3}, }), }), }, @@ -423,6 +486,11 @@ func TestFilter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("XDG_DATA_HOME", tmpDir) + if tt.setup != nil { + tt.setup(t, tmpDir) + } err := vex.Filter(context.Background(), tt.args.report, tt.args.opts) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) From 02e173ac705fed188e78d8781b7bee9a2aef6075 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 10 Jul 2024 14:12:07 +0400 Subject: [PATCH 23/55] feat: skip repos that hasn't been downloaded Signed-off-by: knqyf263 --- pkg/vex/repo.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go index ea8e5f6c1139..04c80f4fb387 100644 --- a/pkg/vex/repo.go +++ b/pkg/vex/repo.go @@ -2,6 +2,7 @@ package vex import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -35,7 +36,10 @@ func NewRepositorySet(ctx context.Context, cacheDir string) (*RepositorySet, err var indexes []RepositoryIndex for _, r := range conf.Repositories { index, err := r.Index(ctx) - if err != nil { + if errors.Is(err, os.ErrNotExist) { + log.Warn("VEX repository not found locally, skipping this repository", log.String("repo", r.Name)) + continue + } else if err != nil { return nil, xerrors.Errorf("failed to get VEX repository index: %w", err) } indexes = append(indexes, RepositoryIndex{ From ab19441e4d8147db9fcf148f82456142e7db47c1 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 11 Jul 2024 12:05:08 +0400 Subject: [PATCH 24/55] feat: use repository_url for OCI Signed-off-by: knqyf263 --- pkg/vex/repo.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go index 04c80f4fb387..7c06d5f382b8 100644 --- a/pkg/vex/repo.go +++ b/pkg/vex/repo.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" + "github.com/package-url/packageurl-go" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/log" @@ -63,6 +64,16 @@ func (rs *RepositorySet) NotAffected(vuln types.DetectedVulnerability, product, p.Qualifiers = nil p.Subpath = "" + if p.Type == packageurl.TypeOCI { + // For OCI artifacts, we consider "repository_url" is part of name. + for _, q := range product.PkgIdentifier.PURL.Qualifiers { + if q.Key == "repository_url" { + p.Qualifiers = packageurl.Qualifiers{q} + break + } + } + } + pkgID := p.String() // PURL without version, qualifiers, and subpath for _, index := range rs.indexes { entry, ok := index.Packages[pkgID] From 0b8c2fd818e1c90754744df7f84bdd55dd515661 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 11 Jul 2024 14:53:18 +0400 Subject: [PATCH 25/55] feat: add "--vex repo" Signed-off-by: knqyf263 --- pkg/commands/app.go | 6 +-- pkg/commands/artifact/run.go | 2 +- pkg/commands/operation/operation.go | 21 +++++++---- pkg/flag/options.go | 2 +- pkg/flag/vulnerability_flags.go | 18 +++++---- pkg/result/filter.go | 4 +- pkg/vex/repo/manager.go | 8 ++-- pkg/vex/vex.go | 57 ++++++++++++++++++++++------- 8 files changed, 78 insertions(+), 40 deletions(-) diff --git a/pkg/commands/app.go b/pkg/commands/app.go index ca60bd4c170c..2d1010125c32 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -1285,10 +1285,8 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { err := vexrepo.NewManager(vexOptions.CacheDir).DownloadRepositories(cmd.Context(), args, vexrepo.Options{Insecure: vexOptions.Insecure}) - if errors.Is(err, vexrepo.ErrNoConfig) { - return errors.New("no config found, run 'trivy vex repo init' first") - } else if err != nil { - return xerrors.Errorf("repository manifest update error: %w", err) + if err != nil { + return xerrors.Errorf("repository download error: %w", err) } return nil }, diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go index 3e928fa9c72d..043109cc0fd5 100644 --- a/pkg/commands/artifact/run.go +++ b/pkg/commands/artifact/run.go @@ -120,7 +120,7 @@ func NewRunner(ctx context.Context, cliOptions flag.Options, opts ...RunnerOptio } // Update the VEX repositories if needed - if err := operation.DownloadVEXRepositories(ctx, cliOptions.CacheDir, cliOptions.SkipDBUpdate, cliOptions.Insecure); err != nil { + if err := operation.DownloadVEXRepositories(ctx, cliOptions); err != nil { return nil, xerrors.Errorf("VEX repositories download error: %w", err) } diff --git a/pkg/commands/operation/operation.go b/pkg/commands/operation/operation.go index 152c109f39ff..478b39dbdf2a 100644 --- a/pkg/commands/operation/operation.go +++ b/pkg/commands/operation/operation.go @@ -2,10 +2,10 @@ package operation import ( "context" - "errors" "sync" "github.com/google/go-containerregistry/pkg/name" + "github.com/samber/lo" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/db" @@ -14,6 +14,7 @@ import ( "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/policy" "github.com/aquasecurity/trivy/pkg/types" + "github.com/aquasecurity/trivy/pkg/vex" "github.com/aquasecurity/trivy/pkg/vex/repo" ) @@ -48,17 +49,23 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n return nil } -func DownloadVEXRepositories(ctx context.Context, cacheDir string, skipUpdate, insecure bool) error { +func DownloadVEXRepositories(ctx context.Context, opts flag.Options) error { mu.Lock() defer mu.Unlock() - ctx = log.WithContextPrefix(ctx, "vex") - err := repo.NewManager(cacheDir).DownloadRepositories(ctx, nil, repo.Options{ - Insecure: insecure, + // Download VEX repositories only if `--vex repo` is passed. + _, enabled := lo.Find(opts.VEXSources, func(src vex.Source) bool { + return src.Type == vex.TypeRepository }) - if errors.Is(err, repo.ErrNoConfig) { + if !enabled { return nil - } else if err != nil { + } + + ctx = log.WithContextPrefix(ctx, "vex") + err := repo.NewManager(opts.CacheDir).DownloadRepositories(ctx, nil, repo.Options{ + Insecure: opts.Insecure, + }) + if err != nil { return xerrors.Errorf("failed to get vex repository config: %w", err) } diff --git a/pkg/flag/options.go b/pkg/flag/options.go index c960617ef29a..24ca7aaf4823 100644 --- a/pkg/flag/options.go +++ b/pkg/flag/options.go @@ -446,7 +446,7 @@ func (o *Options) FilterOpts() result.FilterOptions { PolicyFile: o.IgnorePolicy, IgnoreLicenses: o.IgnoredLicenses, CacheDir: o.CacheDir, - VEXPath: o.VEXPath, + VEXSources: o.VEXSources, } } diff --git a/pkg/flag/vulnerability_flags.go b/pkg/flag/vulnerability_flags.go index 0e9d8291d059..f0176c547e76 100644 --- a/pkg/flag/vulnerability_flags.go +++ b/pkg/flag/vulnerability_flags.go @@ -5,6 +5,7 @@ import ( dbTypes "github.com/aquasecurity/trivy-db/pkg/types" "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/vex" ) var ( @@ -19,30 +20,29 @@ var ( Values: dbTypes.Statuses, Usage: "comma-separated list of vulnerability status to ignore", } - VEXFlag = Flag[string]{ + VEXFlag = Flag[[]string]{ Name: "vex", ConfigName: "vulnerability.vex", - Default: "", - Usage: "[EXPERIMENTAL] file path to VEX", + Usage: `[EXPERIMENTAL] VEX sources ("repo" or file path)`, } ) type VulnerabilityFlagGroup struct { IgnoreUnfixed *Flag[bool] IgnoreStatus *Flag[[]string] - VEXPath *Flag[string] + VEX *Flag[[]string] } type VulnerabilityOptions struct { IgnoreStatuses []dbTypes.Status - VEXPath string + VEXSources []vex.Source } func NewVulnerabilityFlagGroup() *VulnerabilityFlagGroup { return &VulnerabilityFlagGroup{ IgnoreUnfixed: IgnoreUnfixedFlag.Clone(), IgnoreStatus: IgnoreStatusFlag.Clone(), - VEXPath: VEXFlag.Clone(), + VEX: VEXFlag.Clone(), } } @@ -54,7 +54,7 @@ func (f *VulnerabilityFlagGroup) Flags() []Flagger { return []Flagger{ f.IgnoreUnfixed, f.IgnoreStatus, - f.VEXPath, + f.VEX, } } @@ -88,6 +88,8 @@ func (f *VulnerabilityFlagGroup) ToOptions() (VulnerabilityOptions, error) { return VulnerabilityOptions{ IgnoreStatuses: ignoreStatuses, - VEXPath: f.VEXPath.Value(), + VEXSources: lo.Map(f.VEX.Value(), func(s string, _ int) vex.Source { + return vex.NewSource(s) + }), }, nil } diff --git a/pkg/result/filter.go b/pkg/result/filter.go index 8c7a52e4527e..e1f0e632197e 100644 --- a/pkg/result/filter.go +++ b/pkg/result/filter.go @@ -30,7 +30,7 @@ type FilterOptions struct { PolicyFile string IgnoreLicenses []string CacheDir string - VEXPath string + VEXSources []vex.Source } // Filter filters out the report @@ -49,7 +49,7 @@ func Filter(ctx context.Context, report types.Report, opts FilterOptions) error // Filter out vulnerabilities based on the given VEX document. if err = vex.Filter(ctx, &report, vex.Options{ CacheDir: opts.CacheDir, - VEXPath: opts.VEXPath, + Sources: opts.VEXSources, }); err != nil { return xerrors.Errorf("VEX error: %w", err) } diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go index 75a4603e5efc..40a89a6576f9 100644 --- a/pkg/vex/repo/manager.go +++ b/pkg/vex/repo/manager.go @@ -81,8 +81,10 @@ func (m *Manager) writeConfig(conf Config) error { func (m *Manager) Config(ctx context.Context) (Config, error) { if !fsutils.FileExists(m.configFile) { - log.DebugContext(ctx, "No config found", log.String("path", m.configFile)) - return Config{}, ErrNoConfig + log.DebugContext(ctx, "No repository config found", log.String("path", m.configFile)) + if err := m.Init(ctx); err != nil { + return Config{}, xerrors.Errorf("unable to initialize the VEX repository config: %w", err) + } } f, err := os.Open(m.configFile) @@ -119,7 +121,7 @@ func (m *Manager) Init(ctx context.Context) error { if err != nil { return xerrors.Errorf("failed to write the default config: %w", err) } - log.InfoContext(ctx, "The default configuration file has been created", log.FilePath(m.configFile)) + log.InfoContext(ctx, "The default repository config has been created", log.FilePath(m.configFile)) return nil } diff --git a/pkg/vex/vex.go b/pkg/vex/vex.go index 4c02242bb643..8641f2cdb760 100644 --- a/pkg/vex/vex.go +++ b/pkg/vex/vex.go @@ -2,7 +2,6 @@ package vex import ( "context" - "errors" "github.com/samber/lo" "golang.org/x/xerrors" @@ -12,7 +11,11 @@ import ( sbomio "github.com/aquasecurity/trivy/pkg/sbom/io" "github.com/aquasecurity/trivy/pkg/types" "github.com/aquasecurity/trivy/pkg/uuid" - vexrepo "github.com/aquasecurity/trivy/pkg/vex/repo" +) + +const ( + TypeFile SourceType = "file" + TypeRepository SourceType = "repo" ) // VEX represents Vulnerability Exploitability eXchange. It abstracts multiple VEX formats. @@ -28,7 +31,26 @@ type Client struct { type Options struct { CacheDir string - VEXPath string + Sources []Source +} + +type SourceType string + +type Source struct { + Type SourceType + FilePath string // Used only for the file type +} + +func NewSource(src string) Source { + switch src { + case "repository", "repo": + return Source{Type: TypeRepository} + default: + return Source{ + Type: TypeFile, + FilePath: src, + } + } } type NotAffected func(vuln types.DetectedVulnerability, product, subComponent *core.Component) (types.ModifiedFinding, bool) @@ -60,20 +82,27 @@ func Filter(ctx context.Context, report *types.Report, opts Options) error { func New(ctx context.Context, report *types.Report, opts Options) (*Client, error) { var vexes []VEX - v, err := NewDocument(opts.VEXPath, report) - if err != nil { - return nil, xerrors.Errorf("unable to load VEX: %w", err) - } else if v != nil { + for _, src := range opts.Sources { + var v VEX + var err error + switch src.Type { + case TypeFile: + v, err = NewDocument(src.FilePath, report) + if err != nil { + return nil, xerrors.Errorf("unable to load VEX: %w", err) + } + case TypeRepository: + v, err = NewRepositorySet(ctx, opts.CacheDir) + if err != nil { + return nil, xerrors.Errorf("failed to create a repository set: %w", err) + } + default: + log.Warn("Unsupported VEX source", log.String("type", string(src.Type))) + continue + } vexes = append(vexes, v) } - rs, err := NewRepositorySet(ctx, opts.CacheDir) - if !errors.Is(err, vexrepo.ErrNoConfig) && err != nil { - return nil, xerrors.Errorf("failed to create a repository set: %w", err) - } else if rs != nil { - vexes = append(vexes, rs) - } - if len(vexes) == 0 { return nil, nil } From 03c707f962d168d81d1c21b0163c203ed13a2ada Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 11 Jul 2024 15:07:03 +0400 Subject: [PATCH 26/55] feat: add "enabled" field Signed-off-by: knqyf263 --- pkg/vex/repo/manager.go | 21 ++++++++++++--------- pkg/vex/repo/repo.go | 5 +++-- pkg/vex/vex.go | 1 + 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go index 40a89a6576f9..9ab6431a297e 100644 --- a/pkg/vex/repo/manager.go +++ b/pkg/vex/repo/manager.go @@ -2,12 +2,12 @@ package repo import ( "context" - "errors" "io" "os" "path/filepath" "slices" + "github.com/samber/lo" "golang.org/x/xerrors" "gopkg.in/yaml.v3" @@ -21,8 +21,6 @@ const ( repoDir = "repositories" ) -var ErrNoConfig = errors.New("no config found") - type ManagerOption func(indexer *Manager) func WithWriter(w io.Writer) ManagerOption { @@ -98,9 +96,12 @@ func (m *Manager) Config(ctx context.Context) (Config, error) { return conf, xerrors.Errorf("unable to decode metadata: %w", err) } - for i, r := range conf.Repositories { - conf.Repositories[i].dir = filepath.Join(m.cacheDir, repoDir, r.Name) - } + // Filter out disabled repositories + conf.Repositories = lo.FilterMap(conf.Repositories, func(r Repository, _ int) (Repository, bool) { + r.dir = filepath.Join(m.cacheDir, repoDir, r.Name) + return r, r.Enabled + }) + return conf, nil } @@ -113,8 +114,9 @@ func (m *Manager) Init(ctx context.Context) error { err := m.writeConfig(Config{ Repositories: []Repository{ { - Name: "default", - URL: defaultVEXHubURL, + Name: "default", + URL: defaultVEXHubURL, + Enabled: true, }, }, }) @@ -130,7 +132,8 @@ func (m *Manager) DownloadRepositories(ctx context.Context, names []string, opts if err != nil { return xerrors.Errorf("unable to read config: %w", err) } else if len(conf.Repositories) == 0 { - return xerrors.Errorf("no repositories found in config: %s", m.configFile) + log.WarnContext(ctx, "No enabled repositories found in config", log.String("path", m.configFile)) + return nil } for _, repo := range conf.Repositories { diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index 22b540c8fa16..4e7093bb1465 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -85,8 +85,9 @@ type rawIndex struct { } type Repository struct { - Name string - URL string + Name string + URL string + Enabled bool dir string // Root directory for this VEX repository, $CACHE_DIR/vex/repositories/$REPO_NAME/ } diff --git a/pkg/vex/vex.go b/pkg/vex/vex.go index 8641f2cdb760..793c96c41830 100644 --- a/pkg/vex/vex.go +++ b/pkg/vex/vex.go @@ -59,6 +59,7 @@ type NotAffected func(vuln types.DetectedVulnerability, product, subComponent *c // If the VEX document is passed and the vulnerability is either not affected or fixed according to the VEX statement, // the vulnerability is filtered out. func Filter(ctx context.Context, report *types.Report, opts Options) error { + ctx = log.WithContextPrefix(ctx, "vex") client, err := New(ctx, report, opts) if err != nil { return xerrors.Errorf("VEX error: %w", err) From 82f9595f04e8ae75401efe23958e63e386490239 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 12 Jul 2024 11:56:23 +0400 Subject: [PATCH 27/55] feat: add authentication Signed-off-by: knqyf263 --- pkg/downloader/download.go | 34 ++++++++++++++++++++++++++-------- pkg/vex/repo/repo.go | 31 +++++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index ba83ca731f6a..ec05be416060 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -1,6 +1,7 @@ package downloader import ( + "cmp" "context" "errors" "maps" @@ -21,9 +22,16 @@ var ErrSkipDownload = errors.New("skip download") type Options struct { Insecure bool + Auth Auth ETag string } +type Auth struct { + Username string + Password string + Token string +} + // DownloadToTempDir downloads the configured source to a temp dir. func DownloadToTempDir(ctx context.Context, url string, opts Options) (string, error) { tempDir, err := os.MkdirTemp("", "trivy-download") @@ -66,7 +74,7 @@ func Download(ctx context.Context, src, dst, pwd string, opts Options) (string, // it cannot enable WithInsecure() afterwards because its state is preserved. // Therefore, we need to create a new "HttpGetter" instance every time. // cf. https://github.com/hashicorp/go-getter/blob/5a63fd9c0d5b8da8a6805e8c283f46f0dacb30b3/get.go#L63-L65 - transport := NewCustomTransport(opts.ETag) + transport := NewCustomTransport(opts.Auth, opts.ETag) httpGetter := &getter.HttpGetter{ Netrc: true, Client: &http.Client{ @@ -96,22 +104,31 @@ func Download(ctx context.Context, src, dst, pwd string, opts Options) (string, } type CustomTransport struct { + auth Auth cachedETag string newETag string } -func NewCustomTransport(etag string) *CustomTransport { - return &CustomTransport{cachedETag: etag} +func NewCustomTransport(auth Auth, etag string) *CustomTransport { + return &CustomTransport{ + auth: auth, + cachedETag: etag, + } } func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) { if t.cachedETag != "" { req.Header.Set("If-None-Match", t.cachedETag) } + if t.auth.Token != "" { + req.Header.Set("Authorization", "Bearer "+t.auth.Token) + } else if t.auth.Username != "" || t.auth.Password != "" { + req.SetBasicAuth(t.auth.Username, t.auth.Password) + } var transport http.RoundTripper if req.URL.Host == "github.com" { - transport = NewGitHubTransport(req.URL) + transport = NewGitHubTransport(req.URL, t.auth.Token) } if transport == nil { transport = http.DefaultTransport @@ -133,8 +150,8 @@ func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) { return res, nil } -func NewGitHubTransport(u *url.URL) http.RoundTripper { - client := newGitHubClient() +func NewGitHubTransport(u *url.URL, token string) http.RoundTripper { + client := newGitHubClient(token) ss := strings.SplitN(u.Path, "/", 4) if len(ss) < 4 || strings.HasPrefix(ss[3], "archive/") { // Use the default transport from go-github for authentication @@ -166,9 +183,10 @@ func (t *GitHubContentTransport) RoundTrip(req *http.Request) (*http.Response, e return res.Response, nil } -func newGitHubClient() *github.Client { +func newGitHubClient(token string) *github.Client { client := github.NewClient(nil) - if token := os.Getenv("GITHUB_TOKEN"); token != "" { + token = cmp.Or(token, os.Getenv("GITHUB_TOKEN")) + if token != "" { client = client.WithAuthToken(token) } return client diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index 4e7093bb1465..eddac59f3832 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -85,9 +85,12 @@ type rawIndex struct { } type Repository struct { - Name string - URL string - Enabled bool + Name string + URL string + Enabled bool + Username string + Password string + Token string // For Bearer dir string // Root directory for this VEX repository, $CACHE_DIR/vex/repositories/$REPO_NAME/ } @@ -158,7 +161,15 @@ func (r *Repository) downloadManifest(ctx context.Context, opts Options) error { } log.DebugContext(ctx, "Downloading the repository metadata...", log.String("url", u.String()), log.String("dst", r.dir)) - if _, err = downloader.Download(ctx, u.String(), r.dir, ".", downloader.Options{Insecure: opts.Insecure}); err != nil { + _, err = downloader.Download(ctx, u.String(), r.dir, ".", downloader.Options{ + Insecure: opts.Insecure, + Auth: downloader.Auth{ + Username: r.Username, + Password: r.Password, + Token: r.Token, + }, + }) + if err != nil { _ = os.RemoveAll(r.dir) return xerrors.Errorf("failed to download the repository: %w", err) } @@ -231,7 +242,12 @@ func (r *Repository) download(ctx context.Context, ver Version, dst string, opts log.String("dir", dst), log.String("etag", etags[loc.URL])) etag, err := downloader.Download(ctx, loc.URL, dst, ".", downloader.Options{ Insecure: opts.Insecure, - ETag: etags[loc.URL], + Auth: downloader.Auth{ + Username: r.Username, + Password: r.Password, + Token: r.Token, + }, + ETag: etags[loc.URL], }) switch { case errors.Is(err, downloader.ErrSkipDownload): @@ -258,7 +274,10 @@ func (r *Repository) download(ctx context.Context, ver Version, dst string, opts log.Time("updated_at", now)) return nil } - return errs + if errs != nil { + return xerrors.Errorf("failed to download the repository: %w", errs) + } + return nil } func (r *Repository) cacheMetadata() (CacheMetadata, error) { From a22ad3641bc998a74fb311c5a5912a4e29a27c1f Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 13 Jul 2024 17:08:02 +0400 Subject: [PATCH 28/55] test(vex): add tests for vex-repo Signed-off-by: knqyf263 --- pkg/vex/repo/repo.go | 14 +- pkg/vex/repo/repo_test.go | 395 ++++++++++++++++++ .../repo/testdata/.trivy/vex/repository.yaml | 4 + pkg/vex/repo/testdata/test-repo/index.json | 9 + .../testdata/test-repo/vex-repository.json | 16 + 5 files changed, 433 insertions(+), 5 deletions(-) create mode 100644 pkg/vex/repo/repo_test.go create mode 100644 pkg/vex/repo/testdata/.trivy/vex/repository.yaml create mode 100644 pkg/vex/repo/testdata/test-repo/index.json create mode 100644 pkg/vex/repo/testdata/test-repo/vex-repository.json diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index eddac59f3832..aa3c048361a0 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -48,6 +48,10 @@ type Duration struct { time.Duration } +func (d Duration) MarshalJSON() ([]byte, error) { + return json.Marshal(d.String()) +} + // UnmarshalJSON implements the json.Unmarshaler interface func (d *Duration) UnmarshalJSON(b []byte) error { var s string @@ -79,7 +83,7 @@ type PackageEntry struct { Format string `json:"format"` } -type rawIndex struct { +type RawIndex struct { UpdatedAt time.Time `json:"updated_at"` Packages []PackageEntry `json:"packages"` } @@ -100,10 +104,10 @@ type CacheMetadata struct { ETags map[string]string // Last ETag for each URL } -func (r *Repository) Manifest(ctx context.Context) (Manifest, error) { +func (r *Repository) Manifest(ctx context.Context, opts Options) (Manifest, error) { filePath := filepath.Join(r.dir, manifestFile) if !fsutils.FileExists(filePath) { - if err := r.downloadManifest(ctx, Options{}); err != nil { + if err := r.downloadManifest(ctx, opts); err != nil { return Manifest{}, xerrors.Errorf("failed to download the repository metadata: %w", err) } } @@ -132,7 +136,7 @@ func (r *Repository) Index(ctx context.Context) (Index, error) { } defer f.Close() - var raw rawIndex + var raw RawIndex if err = json.NewDecoder(f).Decode(&raw); err != nil { return Index{}, xerrors.Errorf("failed to decode the index: %w", err) } @@ -177,7 +181,7 @@ func (r *Repository) downloadManifest(ctx context.Context, opts Options) error { } func (r *Repository) Update(ctx context.Context, opts Options) error { - manifest, err := r.Manifest(ctx) + manifest, err := r.Manifest(ctx, opts) if err != nil { return xerrors.Errorf("failed to get the repository metadata: %w", err) } diff --git a/pkg/vex/repo/repo_test.go b/pkg/vex/repo/repo_test.go new file mode 100644 index 000000000000..e4c1e09635f5 --- /dev/null +++ b/pkg/vex/repo/repo_test.go @@ -0,0 +1,395 @@ +package repo_test + +import ( + "archive/zip" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aquasecurity/trivy/pkg/clock" + "github.com/aquasecurity/trivy/pkg/vex/repo" +) + +var manifest = repo.Manifest{ + Name: "test-repo", + Description: "test repository", + Versions: map[string]repo.Version{ + "v0": { + SpecVersion: "v0.1", + Locations: []repo.Location{ + { + URL: "https://localhost", + }, + }, + UpdateInterval: repo.Duration{Duration: time.Hour * 24}, + }, + }, +} + +func TestRepository_Manifest(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Println(r.URL.Path) + switch r.URL.Path { + case "/.well-known/vex-repository.json": + err := json.NewEncoder(w).Encode(manifest) + assert.NoError(t, err) + } + http.Error(w, "error", http.StatusInternalServerError) + })) + t.Cleanup(ts.Close) + + tests := []struct { + name string + setup func(*testing.T, string, *repo.Repository) + want repo.Manifest + wantErr string + }{ + { + name: "local manifest exists", + setup: func(t *testing.T, dir string, _ *repo.Repository) { + manifestFile := filepath.Join(dir, "vex", "repositories", "test-repo", "vex-repository.json") + mustEncode(t, manifestFile, manifest) + }, + want: manifest, + }, + { + name: "fetch from remote", + setup: func(t *testing.T, dir string, r *repo.Repository) { + r.URL = ts.URL + }, + want: manifest, + }, + { + name: "http error", + setup: func(t *testing.T, dir string, r *repo.Repository) { + r.URL = ts.URL + "/error" + }, + wantErr: "failed to download the repository metadata", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempDir, m := setupManager(t) + conf, err := m.Config(context.Background()) + require.NoError(t, err) + + r := conf.Repositories[0] + tt.setup(t, tempDir, &r) + + got, err := r.Manifest(context.Background(), repo.Options{}) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestRepository_Index(t *testing.T) { + tests := []struct { + name string + setup func(*testing.T, string, *repo.Repository) + want repo.Index + wantErr string + }{ + { + name: "local index exists", + setup: func(t *testing.T, cacheDir string, r *repo.Repository) { + indexData := repo.RawIndex{ + UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + Packages: []repo.PackageEntry{ + { + ID: "pkg1", + Location: "location1", + Format: "format1", + }, + { + ID: "pkg2", + Location: "location2", + Format: "format2", + }, + }, + } + + indexPath := filepath.Join(cacheDir, "vex", "repositories", r.Name, "v0", "index.json") + mustEncode(t, indexPath, indexData) + }, + want: repo.Index{ + UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + Packages: map[string]repo.PackageEntry{ + "pkg1": { + ID: "pkg1", + Location: "location1", + Format: "format1", + }, + "pkg2": { + ID: "pkg2", + Location: "location2", + Format: "format2", + }, + }, + }, + }, + { + name: "index file not found", + setup: func(*testing.T, string, *repo.Repository) {}, + wantErr: "failed to open the file", + }, + { + name: "invalid JSON in index file", + setup: func(t *testing.T, cacheDir string, r *repo.Repository) { + indexPath := filepath.Join(cacheDir, "vex", "repositories", r.Name, "v0", "index.json") + mustWriteFile(t, indexPath, []byte("invalid JSON")) + }, + wantErr: "failed to decode the index", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempDir, m := setupManager(t) + conf, err := m.Config(context.Background()) + require.NoError(t, err) + + r := conf.Repositories[0] + tt.setup(t, tempDir, &r) + + got, err := r.Index(context.Background()) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + tt.want.Path = filepath.Join(tempDir, "vex", "repositories", r.Name, "v0", "index.json") + assert.Equal(t, tt.want, got) + }) + } +} + +func TestRepository_Update(t *testing.T) { + manifestFile := "testdata/test-repo/vex-repository.json" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/vex-repository.json": + http.ServeFile(w, r, manifestFile) + case "/archive.zip": + if r.Header.Get("If-None-Match") == "current-etag" { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("ETag", "new-etag") + zw := zip.NewWriter(w) + assert.NoError(t, zw.AddFS(os.DirFS("testdata/test-repo"))) + assert.NoError(t, zw.Close()) + case "/error": + w.WriteHeader(http.StatusInternalServerError) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + tests := []struct { + name string + setup func(*testing.T, string, *repo.Repository) + clockTime time.Time + wantErr string + wantCache repo.CacheMetadata + }{ + { + name: "successful update", + setup: func(t *testing.T, cacheDir string, r *repo.Repository) { + setUpManifest(t, cacheDir, ts.URL+"/archive.zip") + }, + clockTime: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + wantCache: repo.CacheMetadata{ + UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + ETags: map[string]string{ts.URL + "/archive.zip": "new-etag"}, + }, + }, + { + name: "no update needed (within update interval)", + setup: func(t *testing.T, cacheDir string, r *repo.Repository) { + setUpManifest(t, cacheDir, "") // No location as the test server is not used + + repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) + mustMkdirAll(t, filepath.Join(repoDir, "v0")) + + cacheMetadata := repo.CacheMetadata{ + UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + ETags: map[string]string{ts.URL + "/archive.zip": "current-etag"}, + } + mustEncode(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) + }, + clockTime: time.Date(2023, 1, 1, 1, 30, 0, 0, time.UTC), + wantCache: repo.CacheMetadata{ + UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + ETags: map[string]string{ts.URL + "/archive.zip": "current-etag"}, + }, + }, + { + name: "update needed (update interval passed)", + setup: func(t *testing.T, cacheDir string, r *repo.Repository) { + setUpManifest(t, cacheDir, ts.URL+"/archive.zip") + + repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) + mustMkdirAll(t, filepath.Join(repoDir, "v0")) + + cacheMetadata := repo.CacheMetadata{ + UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + ETags: map[string]string{ts.URL + "/archive.zip": "old-etag"}, + } + mustEncode(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) + }, + clockTime: time.Date(2023, 1, 2, 3, 0, 0, 0, time.UTC), + wantCache: repo.CacheMetadata{ + UpdatedAt: time.Date(2023, 1, 2, 3, 0, 0, 0, time.UTC), + ETags: map[string]string{ts.URL + "/archive.zip": "new-etag"}, + }, + }, + { + name: "no update needed (304 Not Modified)", + setup: func(t *testing.T, cacheDir string, r *repo.Repository) { + setUpManifest(t, cacheDir, ts.URL+"/archive.zip") + + repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) + mustMkdirAll(t, filepath.Join(repoDir, "v0")) + + cacheMetadata := repo.CacheMetadata{ + UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + ETags: map[string]string{ts.URL + "/archive.zip": "current-etag"}, + } + mustEncode(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) + }, + clockTime: time.Date(2023, 1, 2, 3, 0, 0, 0, time.UTC), + wantCache: repo.CacheMetadata{ + UpdatedAt: time.Date(2023, 1, 2, 3, 0, 0, 0, time.UTC), + ETags: map[string]string{ts.URL + "/archive.zip": "current-etag"}, + }, + }, + { + name: "update with no existing cache.json", + setup: func(t *testing.T, cacheDir string, r *repo.Repository) { + setUpManifest(t, cacheDir, ts.URL+"/archive.zip") + + repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) + mustMkdirAll(t, filepath.Join(repoDir, "v0")) + }, + clockTime: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + wantCache: repo.CacheMetadata{ + UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + ETags: map[string]string{ts.URL + "/archive.zip": "new-etag"}, + }, + }, + { + name: "manifest not found", + setup: func(*testing.T, string, *repo.Repository) {}, + clockTime: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + wantErr: "failed to get the repository metadata", + }, + { + name: "download error", + setup: func(t *testing.T, cacheDir string, r *repo.Repository) { + setUpManifest(t, cacheDir, ts.URL+"/error") + + repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) + mustMkdirAll(t, filepath.Join(repoDir, "v0")) + }, + clockTime: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + wantErr: "failed to download the repository", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempDir, m := setupManager(t) + conf, err := m.Config(context.Background()) + require.NoError(t, err) + + r := conf.Repositories[0] + r.URL = ts.URL + "/vex-repository.json" + tt.setup(t, tempDir, &r) + + ctx := clock.With(context.Background(), tt.clockTime) + err = r.Update(ctx, repo.Options{}) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + return + } + + require.NoError(t, err) + + cacheFile := filepath.Join(tempDir, "vex", "repositories", r.Name, "cache.json") + var gotCache repo.CacheMetadata + mustDecode(t, cacheFile, &gotCache) + assert.Equal(t, tt.wantCache, gotCache) + }) + } +} + +func setupManager(t *testing.T) (string, *repo.Manager) { + tempDir := t.TempDir() + t.Setenv("XDG_DATA_HOME", "testdata") + return tempDir, repo.NewManager(tempDir) +} + +func setUpManifest(t *testing.T, dir, url string) { + manifest := repo.Manifest{ + Name: "test-repo", + Description: "test repository", + Versions: map[string]repo.Version{ + "v0": { + SpecVersion: "v0.1", + Locations: []repo.Location{ + { + URL: url, + }, + }, + UpdateInterval: repo.Duration{Duration: time.Hour * 24}, + }, + }, + } + manifestPath := filepath.Join(dir, "vex", "repositories", "test-repo", "vex-repository.json") + mustMkdirAll(t, filepath.Dir(manifestPath)) + mustEncode(t, manifestPath, manifest) +} + +func mustMkdirAll(t *testing.T, dir string) { + err := os.MkdirAll(dir, 0755) + require.NoError(t, err) +} + +func mustDecode(t *testing.T, filePath string, v interface{}) { + b, err := os.ReadFile(filePath) + require.NoError(t, err) + err = json.Unmarshal(b, v) + require.NoError(t, err) +} + +func mustEncode(t *testing.T, filePath string, v interface{}) { + data, err := json.Marshal(v) + require.NoError(t, err) + + mustWriteFile(t, filePath, data) +} + +func mustWriteFile(t *testing.T, filePath string, content []byte) { + dir := filepath.Dir(filePath) + mustMkdirAll(t, dir) + + err := os.WriteFile(filePath, content, 0744) + require.NoError(t, err) +} diff --git a/pkg/vex/repo/testdata/.trivy/vex/repository.yaml b/pkg/vex/repo/testdata/.trivy/vex/repository.yaml new file mode 100644 index 000000000000..92f1c4b7538d --- /dev/null +++ b/pkg/vex/repo/testdata/.trivy/vex/repository.yaml @@ -0,0 +1,4 @@ +repositories: + - name: "test-repo" + url: "https://localhost" + enabled: true \ No newline at end of file diff --git a/pkg/vex/repo/testdata/test-repo/index.json b/pkg/vex/repo/testdata/test-repo/index.json new file mode 100644 index 000000000000..a2b346441c8a --- /dev/null +++ b/pkg/vex/repo/testdata/test-repo/index.json @@ -0,0 +1,9 @@ +{ + "version": 1, + "packages": [ + { + "ID": "pkg:golang/github.com/aquasecurity/trivy", + "Location": "test" + } + ] +} \ No newline at end of file diff --git a/pkg/vex/repo/testdata/test-repo/vex-repository.json b/pkg/vex/repo/testdata/test-repo/vex-repository.json new file mode 100644 index 000000000000..132e960e553d --- /dev/null +++ b/pkg/vex/repo/testdata/test-repo/vex-repository.json @@ -0,0 +1,16 @@ +{ + "name": "Test Repository", + "description": "Test Repository", + "versions": { + "v0": { + "spec_version": "v0.1", + "locations": [ + { + "url": "Must be filled in tests" + } + ], + "update_interval": "24h" + } + }, + "latest_version": "v0" +} \ No newline at end of file From 2609053d76a2dfe239c99a8ce72e163accb1fd96 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 18 Jul 2024 13:12:12 +0400 Subject: [PATCH 29/55] feat: support version array Signed-off-by: knqyf263 --- pkg/vex/repo/repo.go | 35 +++++++++++++++++++++-------------- pkg/vex/repo/repo_test.go | 28 ++++++++++++++-------------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index aa3c048361a0..be0971ece228 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -8,7 +8,6 @@ import ( "os" "path" "path/filepath" - "strings" "time" "github.com/samber/lo" @@ -21,20 +20,18 @@ import ( ) const ( - SchemaVersion = "v0.1" + SchemaVersion = "0.1" manifestFile = "vex-repository.json" indexFile = "index.json" cacheMetadataFile = "cache.json" ) -var majorVersion, _, _ = strings.Cut(SchemaVersion, ".") - type Manifest struct { - Name string `json:"name"` - Description string `json:"description"` - Versions map[string]Version `json:"versions"` - LatestVersion string `json:"latest_version"` + Name string `json:"name"` + Description string `json:"description"` + Versions []Version `json:"versions"` + LatestVersion string `json:"latest_version"` } type Version struct { @@ -127,7 +124,7 @@ func (r *Repository) Manifest(ctx context.Context, opts Options) (Manifest, erro } func (r *Repository) Index(ctx context.Context) (Index, error) { - filePath := filepath.Join(r.dir, majorVersion, indexFile) + filePath := filepath.Join(r.dir, SchemaVersion, indexFile) log.DebugContext(ctx, "Reading the repository index...", log.String("repo", r.Name), log.FilePath(filePath)) f, err := os.Open(filePath) @@ -186,13 +183,12 @@ func (r *Repository) Update(ctx context.Context, opts Options) error { return xerrors.Errorf("failed to get the repository metadata: %w", err) } - ver, ok := manifest.Versions[majorVersion] - if !ok { - // TODO: improve error - return xerrors.Errorf("version %s not found", majorVersion) + ver, err := r.selectSupportedVersion(manifest.Versions) + if err != nil { + return xerrors.Errorf("version %s not found", SchemaVersion) } - versionDir := filepath.Join(r.dir, majorVersion) + versionDir := filepath.Join(r.dir, SchemaVersion) if !r.needUpdate(ctx, ver, versionDir) { log.InfoContext(ctx, "No need to check repository updates", log.String("repo", r.Name)) return nil @@ -302,6 +298,17 @@ func (r *Repository) cacheMetadata() (CacheMetadata, error) { return metadata, nil } +func (r *Repository) selectSupportedVersion(versions []Version) (Version, error) { + for _, ver := range versions { + // Versions should exactly match until the spec version reaches 1.0. + // After reaching 1.0, we can select the latest version that has the same major version. + if ver.SpecVersion == SchemaVersion { + return ver, nil + } + } + return Version{}, xerrors.New("no supported version found") +} + func (r *Repository) updateCacheMetadata(ctx context.Context, metadata CacheMetadata) error { filePath := filepath.Join(r.dir, cacheMetadataFile) log.DebugContext(ctx, "Updating repository cache metadata...", log.FilePath(filePath)) diff --git a/pkg/vex/repo/repo_test.go b/pkg/vex/repo/repo_test.go index e4c1e09635f5..8e0547b584fc 100644 --- a/pkg/vex/repo/repo_test.go +++ b/pkg/vex/repo/repo_test.go @@ -22,9 +22,9 @@ import ( var manifest = repo.Manifest{ Name: "test-repo", Description: "test repository", - Versions: map[string]repo.Version{ - "v0": { - SpecVersion: "v0.1", + Versions: []repo.Version{ + { + SpecVersion: "0.1", Locations: []repo.Location{ { URL: "https://localhost", @@ -123,7 +123,7 @@ func TestRepository_Index(t *testing.T) { }, } - indexPath := filepath.Join(cacheDir, "vex", "repositories", r.Name, "v0", "index.json") + indexPath := filepath.Join(cacheDir, "vex", "repositories", r.Name, "0.1", "index.json") mustEncode(t, indexPath, indexData) }, want: repo.Index{ @@ -150,7 +150,7 @@ func TestRepository_Index(t *testing.T) { { name: "invalid JSON in index file", setup: func(t *testing.T, cacheDir string, r *repo.Repository) { - indexPath := filepath.Join(cacheDir, "vex", "repositories", r.Name, "v0", "index.json") + indexPath := filepath.Join(cacheDir, "vex", "repositories", r.Name, "0.1", "index.json") mustWriteFile(t, indexPath, []byte("invalid JSON")) }, wantErr: "failed to decode the index", @@ -172,7 +172,7 @@ func TestRepository_Index(t *testing.T) { return } require.NoError(t, err) - tt.want.Path = filepath.Join(tempDir, "vex", "repositories", r.Name, "v0", "index.json") + tt.want.Path = filepath.Join(tempDir, "vex", "repositories", r.Name, "0.1", "index.json") assert.Equal(t, tt.want, got) }) } @@ -226,7 +226,7 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, "") // No location as the test server is not used repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "v0")) + mustMkdirAll(t, filepath.Join(repoDir, "0.1")) cacheMetadata := repo.CacheMetadata{ UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), @@ -246,7 +246,7 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, ts.URL+"/archive.zip") repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "v0")) + mustMkdirAll(t, filepath.Join(repoDir, "0.1")) cacheMetadata := repo.CacheMetadata{ UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), @@ -266,7 +266,7 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, ts.URL+"/archive.zip") repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "v0")) + mustMkdirAll(t, filepath.Join(repoDir, "0.1")) cacheMetadata := repo.CacheMetadata{ UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), @@ -286,7 +286,7 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, ts.URL+"/archive.zip") repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "v0")) + mustMkdirAll(t, filepath.Join(repoDir, "0.1")) }, clockTime: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), wantCache: repo.CacheMetadata{ @@ -306,7 +306,7 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, ts.URL+"/error") repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "v0")) + mustMkdirAll(t, filepath.Join(repoDir, "0.1")) }, clockTime: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), wantErr: "failed to download the repository", @@ -350,9 +350,9 @@ func setUpManifest(t *testing.T, dir, url string) { manifest := repo.Manifest{ Name: "test-repo", Description: "test repository", - Versions: map[string]repo.Version{ - "v0": { - SpecVersion: "v0.1", + Versions: []repo.Version{ + { + SpecVersion: "0.1", Locations: []repo.Location{ { URL: url, From 438187dddc93804b1437d68fd8687c7d5a8660c2 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 20 Jul 2024 11:00:23 +0400 Subject: [PATCH 30/55] test(vex): add tests for manager Signed-off-by: knqyf263 --- pkg/vex/repo/manager_test.go | 286 +++++++++++++++++++++++++++++++++++ pkg/vex/repo/repo_test.go | 62 ++++---- 2 files changed, 317 insertions(+), 31 deletions(-) create mode 100644 pkg/vex/repo/manager_test.go diff --git a/pkg/vex/repo/manager_test.go b/pkg/vex/repo/manager_test.go new file mode 100644 index 000000000000..787f89f132eb --- /dev/null +++ b/pkg/vex/repo/manager_test.go @@ -0,0 +1,286 @@ +package repo_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/aquasecurity/trivy/pkg/vex/repo" +) + +func TestManager_Config(t *testing.T) { + tests := []struct { + name string + setup func(*testing.T, string) + want repo.Config + wantErr string + }{ + { + name: "config file exists", + setup: func(t *testing.T, dir string) { + config := repo.Config{ + Repositories: []repo.Repository{ + { + Name: "test-repo", + URL: "https://example.com/repo", + Enabled: true, + }, + { + Name: "test-disabled-repo", + URL: "https://example.com/disabled-repo", + Enabled: false, + }, + }, + } + configPath := filepath.Join(dir, ".trivy", "vex", "repository.yaml") + mustWriteYAML(t, configPath, config) + }, + want: repo.Config{ + Repositories: []repo.Repository{ + { + Name: "test-repo", + URL: "https://example.com/repo", + Enabled: true, + }, + }, + }, + }, + { + name: "config file does not exist", + setup: func(t *testing.T, dir string) {}, + want: repo.Config{ + Repositories: []repo.Repository{ + { + Name: "default", + URL: "https://github.com/aquasecurity/vexhub", + Enabled: true, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempDir := t.TempDir() + t.Setenv("XDG_DATA_HOME", tempDir) + m := repo.NewManager(tempDir) + + tt.setup(t, tempDir) + + got, err := m.Config(context.Background()) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.EqualExportedValues(t, tt.want, got) + }) + } +} + +func TestManager_Init(t *testing.T) { + tests := []struct { + name string + setup func(*testing.T, string) + want repo.Config + wantErr string + }{ + { + name: "successful init", + setup: func(t *testing.T, dir string) {}, + want: repo.Config{ + Repositories: []repo.Repository{ + { + Name: "default", + URL: "https://github.com/aquasecurity/vexhub", + Enabled: true, + }, + }, + }, + }, + { + name: "config already exists", + setup: func(t *testing.T, dir string) { + configPath := filepath.Join(dir, ".trivy", "vex", "repository.yaml") + mustWriteYAML(t, configPath, repo.Config{}) + }, + want: repo.Config{ + Repositories: []repo.Repository{}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempDir := t.TempDir() + t.Setenv("XDG_DATA_HOME", tempDir) + m := repo.NewManager(tempDir) + + tt.setup(t, tempDir) + + err := m.Init(context.Background()) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + + configPath := filepath.Join(tempDir, ".trivy", "vex", "repository.yaml") + assert.FileExists(t, configPath) + + var got repo.Config + mustReadYAML(t, configPath, &got) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestManager_DownloadRepositories(t *testing.T) { + ts := setUpRepository(t) + defer ts.Close() + + tests := []struct { + name string + config repo.Config + location string + names []string + wantErr string + wantDownload bool + }{ + { + name: "successful download", + config: repo.Config{ + Repositories: []repo.Repository{ + { + Name: "test-repo", + URL: ts.URL, + Enabled: true, + }, + }, + }, + location: ts.URL + "/archive.zip", + wantDownload: true, + }, + { + name: "no enabled repositories", + config: repo.Config{ + Repositories: []repo.Repository{ + { + Name: "test-repo", + URL: ts.URL, + Enabled: false, + }, + }, + }, + location: ts.URL + "/archive.zip", + wantDownload: false, + }, + { + name: "download specific repository", + config: repo.Config{ + Repositories: []repo.Repository{ + { + Name: "another-repo", + URL: "https://example.com/repo", + Enabled: true, + }, + { + Name: "test-repo", + URL: ts.URL, + Enabled: true, + }, + }, + }, + location: ts.URL + "/archive.zip", + names: []string{"test-repo"}, + wantDownload: true, + }, + { + name: "download error", + config: repo.Config{ + Repositories: []repo.Repository{ + { + Name: "test-repo", + URL: ts.URL, + Enabled: true, + }, + }, + }, + location: ts.URL + "/error", + wantErr: "failed to download the repository", + wantDownload: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempDir := t.TempDir() + t.Setenv("XDG_DATA_HOME", tempDir) + m := repo.NewManager(tempDir) + + configPath := filepath.Join(tempDir, ".trivy", "vex", "repository.yaml") + mustWriteYAML(t, configPath, tt.config) + + manifestPath := filepath.Join(tempDir, "vex", "repositories", "test-repo", "vex-repository.json") + manifest.Versions[0].Locations[0].URL = tt.location + mustWriteJSON(t, manifestPath, manifest) + + err := m.DownloadRepositories(context.Background(), tt.names, repo.Options{}) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + + // Check if the repository was downloaded + if tt.wantDownload { + repoDir := filepath.Join(tempDir, "vex", "repositories", "test-repo") + assert.DirExists(t, repoDir) + assert.FileExists(t, filepath.Join(repoDir, "vex-repository.json")) + assert.FileExists(t, filepath.Join(repoDir, "0.1", "index.json")) + } + }) + } +} + +func TestManager_Clear(t *testing.T) { + tempDir := t.TempDir() + m := repo.NewManager(tempDir) + + // Create some dummy files + cacheDir := filepath.Join(tempDir, "vex") + require.NoError(t, os.MkdirAll(cacheDir, 0755)) + dummyFile := filepath.Join(cacheDir, "dummy.txt") + require.NoError(t, os.WriteFile(dummyFile, []byte("dummy"), 0644)) + + err := m.Clear() + require.NoError(t, err) + + // Check if the cache directory was removed + _, err = os.Stat(cacheDir) + assert.True(t, os.IsNotExist(err)) +} + +func mustWriteYAML(t *testing.T, path string, data interface{}) { + t.Helper() + dir := filepath.Dir(path) + require.NoError(t, os.MkdirAll(dir, 0755)) + f, err := os.Create(path) + require.NoError(t, err) + defer f.Close() + require.NoError(t, yaml.NewEncoder(f).Encode(data)) +} + +func mustReadYAML(t *testing.T, path string, out interface{}) { + t.Helper() + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + require.NoError(t, yaml.NewDecoder(f).Decode(out)) +} diff --git a/pkg/vex/repo/repo_test.go b/pkg/vex/repo/repo_test.go index 8e0547b584fc..572422ad2683 100644 --- a/pkg/vex/repo/repo_test.go +++ b/pkg/vex/repo/repo_test.go @@ -57,7 +57,7 @@ func TestRepository_Manifest(t *testing.T) { name: "local manifest exists", setup: func(t *testing.T, dir string, _ *repo.Repository) { manifestFile := filepath.Join(dir, "vex", "repositories", "test-repo", "vex-repository.json") - mustEncode(t, manifestFile, manifest) + mustWriteJSON(t, manifestFile, manifest) }, want: manifest, }, @@ -124,7 +124,7 @@ func TestRepository_Index(t *testing.T) { } indexPath := filepath.Join(cacheDir, "vex", "repositories", r.Name, "0.1", "index.json") - mustEncode(t, indexPath, indexData) + mustWriteJSON(t, indexPath, indexData) }, want: repo.Index{ UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), @@ -179,27 +179,7 @@ func TestRepository_Index(t *testing.T) { } func TestRepository_Update(t *testing.T) { - manifestFile := "testdata/test-repo/vex-repository.json" - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/vex-repository.json": - http.ServeFile(w, r, manifestFile) - case "/archive.zip": - if r.Header.Get("If-None-Match") == "current-etag" { - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set("Content-Type", "application/zip") - w.Header().Set("ETag", "new-etag") - zw := zip.NewWriter(w) - assert.NoError(t, zw.AddFS(os.DirFS("testdata/test-repo"))) - assert.NoError(t, zw.Close()) - case "/error": - w.WriteHeader(http.StatusInternalServerError) - default: - w.WriteHeader(http.StatusNotFound) - } - })) + ts := setUpRepository(t) defer ts.Close() tests := []struct { @@ -232,7 +212,7 @@ func TestRepository_Update(t *testing.T) { UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), ETags: map[string]string{ts.URL + "/archive.zip": "current-etag"}, } - mustEncode(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) + mustWriteJSON(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) }, clockTime: time.Date(2023, 1, 1, 1, 30, 0, 0, time.UTC), wantCache: repo.CacheMetadata{ @@ -252,7 +232,7 @@ func TestRepository_Update(t *testing.T) { UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), ETags: map[string]string{ts.URL + "/archive.zip": "old-etag"}, } - mustEncode(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) + mustWriteJSON(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) }, clockTime: time.Date(2023, 1, 2, 3, 0, 0, 0, time.UTC), wantCache: repo.CacheMetadata{ @@ -272,7 +252,7 @@ func TestRepository_Update(t *testing.T) { UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), ETags: map[string]string{ts.URL + "/archive.zip": "current-etag"}, } - mustEncode(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) + mustWriteJSON(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) }, clockTime: time.Date(2023, 1, 2, 3, 0, 0, 0, time.UTC), wantCache: repo.CacheMetadata{ @@ -334,7 +314,7 @@ func TestRepository_Update(t *testing.T) { cacheFile := filepath.Join(tempDir, "vex", "repositories", r.Name, "cache.json") var gotCache repo.CacheMetadata - mustDecode(t, cacheFile, &gotCache) + mustReadJSON(t, cacheFile, &gotCache) assert.Equal(t, tt.wantCache, gotCache) }) } @@ -363,8 +343,28 @@ func setUpManifest(t *testing.T, dir, url string) { }, } manifestPath := filepath.Join(dir, "vex", "repositories", "test-repo", "vex-repository.json") - mustMkdirAll(t, filepath.Dir(manifestPath)) - mustEncode(t, manifestPath, manifest) + mustWriteJSON(t, manifestPath, manifest) +} + +func setUpRepository(t *testing.T) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/archive.zip": + if r.Header.Get("If-None-Match") == "current-etag" { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("ETag", "new-etag") + zw := zip.NewWriter(w) + assert.NoError(t, zw.AddFS(os.DirFS("testdata/test-repo"))) + assert.NoError(t, zw.Close()) + case "/error": + w.WriteHeader(http.StatusInternalServerError) + default: + w.WriteHeader(http.StatusNotFound) + } + })) } func mustMkdirAll(t *testing.T, dir string) { @@ -372,14 +372,14 @@ func mustMkdirAll(t *testing.T, dir string) { require.NoError(t, err) } -func mustDecode(t *testing.T, filePath string, v interface{}) { +func mustReadJSON(t *testing.T, filePath string, v interface{}) { b, err := os.ReadFile(filePath) require.NoError(t, err) err = json.Unmarshal(b, v) require.NoError(t, err) } -func mustEncode(t *testing.T, filePath string, v interface{}) { +func mustWriteJSON(t *testing.T, filePath string, v interface{}) { data, err := json.Marshal(v) require.NoError(t, err) From 74ebecfee20793c8a2c447a7b3a4e278c56c148c Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 20 Jul 2024 11:13:27 +0400 Subject: [PATCH 31/55] feat: add "vex repo list" Signed-off-by: knqyf263 --- pkg/commands/app.go | 14 +++++++ pkg/downloader/download.go | 2 +- pkg/vex/repo.go | 2 +- pkg/vex/repo/manager.go | 50 +++++++++++++++++++---- pkg/vex/repo/manager_test.go | 79 +++++++++++++++++++++++++++++++++--- 5 files changed, 132 insertions(+), 15 deletions(-) diff --git a/pkg/commands/app.go b/pkg/commands/app.go index 2d1010125c32..554d6330529f 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -1276,6 +1276,20 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { return nil }, }, + &cobra.Command{ + Use: "list", + Short: "List VEX repositories", + SilenceErrors: true, + SilenceUsage: true, + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + if err := vexrepo.NewManager(vexOptions.CacheDir).List(cmd.Context()); err != nil { + return xerrors.Errorf("list error: %w", err) + } + return nil + }, + }, &cobra.Command{ Use: "download [REPO_NAMES]", Short: "Download the VEX repositories", diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index ec05be416060..87bf65272eb8 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -97,7 +97,7 @@ func Download(ctx context.Context, src, dst, pwd string, opts Options) (string, } if err := client.Get(); err != nil { - return "", xerrors.Errorf("failed to download: %w", err) + return "", xerrors.Errorf("failed to download %s: %w", src, err) } return transport.newETag, nil diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go index 7c06d5f382b8..f71c24eee97b 100644 --- a/pkg/vex/repo.go +++ b/pkg/vex/repo.go @@ -35,7 +35,7 @@ func NewRepositorySet(ctx context.Context, cacheDir string) (*RepositorySet, err } var indexes []RepositoryIndex - for _, r := range conf.Repositories { + for _, r := range conf.EnabledRepositories() { index, err := r.Index(ctx) if errors.Is(err, os.ErrNotExist) { log.Warn("VEX repository not found locally, skipping this repository", log.String("repo", r.Name)) diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go index 9ab6431a297e..56c1ae7a0cb9 100644 --- a/pkg/vex/repo/manager.go +++ b/pkg/vex/repo/manager.go @@ -2,10 +2,12 @@ package repo import ( "context" + "fmt" "io" "os" "path/filepath" "slices" + "strings" "github.com/samber/lo" "golang.org/x/xerrors" @@ -33,6 +35,12 @@ type Config struct { Repositories []Repository `json:"repositories"` } +func (c *Config) EnabledRepositories() []Repository { + return lo.Filter(c.Repositories, func(r Repository, _ int) bool { + return r.Enabled + }) +} + type Options struct { Insecure bool } @@ -96,11 +104,9 @@ func (m *Manager) Config(ctx context.Context) (Config, error) { return conf, xerrors.Errorf("unable to decode metadata: %w", err) } - // Filter out disabled repositories - conf.Repositories = lo.FilterMap(conf.Repositories, func(r Repository, _ int) (Repository, bool) { - r.dir = filepath.Join(m.cacheDir, repoDir, r.Name) - return r, r.Enabled - }) + for i, repo := range conf.Repositories { + conf.Repositories[i].dir = filepath.Join(m.cacheDir, repoDir, repo.Name) + } return conf, nil } @@ -131,12 +137,12 @@ func (m *Manager) DownloadRepositories(ctx context.Context, names []string, opts conf, err := m.Config(ctx) if err != nil { return xerrors.Errorf("unable to read config: %w", err) - } else if len(conf.Repositories) == 0 { + } else if len(conf.EnabledRepositories()) == 0 { log.WarnContext(ctx, "No enabled repositories found in config", log.String("path", m.configFile)) return nil } - for _, repo := range conf.Repositories { + for _, repo := range conf.EnabledRepositories() { if len(names) > 0 && !slices.Contains(names, repo.Name) { continue } @@ -147,6 +153,36 @@ func (m *Manager) DownloadRepositories(ctx context.Context, names []string, opts return nil } +// List returns a list of all repositories in the configuration +func (m *Manager) List(ctx context.Context) error { + conf, err := m.Config(ctx) + if err != nil { + return xerrors.Errorf("unable to read config: %w", err) + } + + var output strings.Builder + + output.WriteString(fmt.Sprintf("VEX Repositories (config: %s)\n\n", m.configFile)) + + if len(conf.Repositories) == 0 { + output.WriteString("No repositories configured.\n") + } else { + for _, repo := range conf.Repositories { + status := "Enabled" + if !repo.Enabled { + status = "Disabled" + } + output.WriteString(fmt.Sprintf("- Name: %s\n URL: %s\n Status: %s\n\n", repo.Name, repo.URL, status)) + } + } + + if _, err = io.WriteString(m.w, output.String()); err != nil { + return xerrors.Errorf("failed to write output: %w", err) + } + + return nil +} + func (m *Manager) Clear() error { return os.RemoveAll(m.cacheDir) } diff --git a/pkg/vex/repo/manager_test.go b/pkg/vex/repo/manager_test.go index 787f89f132eb..af1a339f4f97 100644 --- a/pkg/vex/repo/manager_test.go +++ b/pkg/vex/repo/manager_test.go @@ -1,7 +1,9 @@ package repo_test import ( + "bytes" "context" + "fmt" "os" "path/filepath" "testing" @@ -30,11 +32,6 @@ func TestManager_Config(t *testing.T) { URL: "https://example.com/repo", Enabled: true, }, - { - Name: "test-disabled-repo", - URL: "https://example.com/disabled-repo", - Enabled: false, - }, }, } configPath := filepath.Join(dir, ".trivy", "vex", "repository.yaml") @@ -173,7 +170,7 @@ func TestManager_DownloadRepositories(t *testing.T) { Repositories: []repo.Repository{ { Name: "test-repo", - URL: ts.URL, + URL: "https://localhost:10000", // Will not be reached Enabled: false, }, }, @@ -249,6 +246,76 @@ func TestManager_DownloadRepositories(t *testing.T) { } } +func TestManager_List(t *testing.T) { + tests := []struct { + name string + config repo.Config + want string + wantErr string + }{ + { + name: "list repositories", + config: repo.Config{ + Repositories: []repo.Repository{ + { + Name: "default", + URL: "https://github.com/aquasecurity/vexhub", + Enabled: true, + }, + { + Name: "custom", + URL: "https://example.com/custom-vex-repo", + Enabled: false, + }, + }, + }, + want: `VEX Repositories (config: %s) + +- Name: default + URL: https://github.com/aquasecurity/vexhub + Status: Enabled + +- Name: custom + URL: https://example.com/custom-vex-repo + Status: Disabled + +`, + }, + { + name: "no repositories", + config: repo.Config{ + Repositories: []repo.Repository{}, + }, + want: `VEX Repositories (config: %s) + +No repositories configured. +`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempDir := t.TempDir() + t.Setenv("XDG_DATA_HOME", tempDir) + configPath := filepath.Join(tempDir, ".trivy", "vex", "repository.yaml") + mustWriteYAML(t, configPath, tt.config) + + var buf bytes.Buffer + m := repo.NewManager(tempDir, repo.WithWriter(&buf)) + + err := m.List(context.Background()) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + return + } + + want := fmt.Sprintf(tt.want, configPath) + require.NoError(t, err) + assert.Equal(t, want, buf.String()) + }) + } +} + func TestManager_Clear(t *testing.T) { tempDir := t.TempDir() m := repo.NewManager(tempDir) From 83d5ae0b4e15159b74cf9c73cc7baa0db851e8cd Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 20 Jul 2024 11:46:22 +0400 Subject: [PATCH 32/55] chore: rename --vex-repos to --vex-repo Signed-off-by: knqyf263 --- pkg/flag/clean_flags.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/flag/clean_flags.go b/pkg/flag/clean_flags.go index 57cf1d3376fc..11e0e5934e90 100644 --- a/pkg/flag/clean_flags.go +++ b/pkg/flag/clean_flags.go @@ -27,9 +27,9 @@ var ( ConfigName: "clean.checks-bundle", Usage: "remove checks bundle", } - CleanVEXRepos = Flag[bool]{ - Name: "vex-repos", - ConfigName: "clean.vex-repos", + CleanVEXRepo = Flag[bool]{ + Name: "vex-repo", + ConfigName: "clean.vex-repo", Usage: "remove VEX repositories", } ) @@ -59,7 +59,7 @@ func NewCleanFlagGroup() *CleanFlagGroup { CleanVulnerabilityDB: CleanVulnerabilityDB.Clone(), CleanJavaDB: CleanJavaDB.Clone(), CleanChecksBundle: CleanChecksBundle.Clone(), - CleanVEXRepositories: CleanVEXRepos.Clone(), + CleanVEXRepositories: CleanVEXRepo.Clone(), } } From db16176d9e9dbb2072112a2a06f8ab525fa4bc87 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 20 Jul 2024 15:35:28 +0400 Subject: [PATCH 33/55] docs: add VEX Repository Signed-off-by: knqyf263 --- docs/docs/configuration/filtering.md | 2 +- .../docs/supply-chain/{vex.md => vex/file.md} | 8 +- docs/docs/supply-chain/vex/index.md | 33 +++ docs/docs/supply-chain/vex/repo.md | 201 ++++++++++++++++++ mkdocs.yml | 11 +- 5 files changed, 246 insertions(+), 9 deletions(-) rename docs/docs/supply-chain/{vex.md => vex/file.md} (97%) create mode 100644 docs/docs/supply-chain/vex/index.md create mode 100644 docs/docs/supply-chain/vex/repo.md diff --git a/docs/docs/configuration/filtering.md b/docs/docs/configuration/filtering.md index e3d38f3cdc15..abe8e84ff7e3 100644 --- a/docs/docs/configuration/filtering.md +++ b/docs/docs/configuration/filtering.md @@ -493,7 +493,7 @@ You can find more example checks [here](https://github.com/aquasecurity/trivy/tr | Secret | | | License | | -Please refer to the [VEX documentation](../supply-chain/vex.md) for the details. +Please refer to the [VEX documentation](../supply-chain/vex/index.md) for the details. [^1]: license name is used as id for `.trivyignore.yaml` files. diff --git a/docs/docs/supply-chain/vex.md b/docs/docs/supply-chain/vex/file.md similarity index 97% rename from docs/docs/supply-chain/vex.md rename to docs/docs/supply-chain/vex/file.md index 0ceaeeeb67b3..7c847ec49e14 100644 --- a/docs/docs/supply-chain/vex.md +++ b/docs/docs/supply-chain/vex/file.md @@ -1,11 +1,11 @@ -# Vulnerability Exploitability Exchange (VEX) +# Local VEX Files !!! warning "EXPERIMENTAL" This feature might change without preserving backwards compatibility. -Trivy supports filtering detected vulnerabilities using [the Vulnerability Exploitability Exchange (VEX)](https://www.ntia.gov/files/ntia/publications/vex_one-page_summary.pdf), a standardized format for sharing and exchanging information about vulnerabilities. -By providing VEX during scanning, it is possible to filter vulnerabilities based on their status. -Currently, Trivy supports the following three formats: +In addition to [VEX repositories](./repo.md), Trivy also supports the use of local VEX files for vulnerability filtering. +This method is useful when you have specific VEX documents that you want to apply to your scans. +Currently, Trivy supports the following formats: - [CycloneDX](https://cyclonedx.org/capabilities/vex/) - [OpenVEX](https://github.com/openvex/spec) diff --git a/docs/docs/supply-chain/vex/index.md b/docs/docs/supply-chain/vex/index.md new file mode 100644 index 000000000000..4f8d59abe126 --- /dev/null +++ b/docs/docs/supply-chain/vex/index.md @@ -0,0 +1,33 @@ +# Vulnerability Exploitability Exchange (VEX) + +!!! warning "EXPERIMENTAL" + This feature might change without preserving backwards compatibility. + +Trivy supports filtering detected vulnerabilities using the [Vulnerability Exploitability eXchange (VEX)](https://www.ntia.gov/files/ntia/publications/vex_one-page_summary.pdf), a standardized format for sharing and exchanging information about vulnerabilities. +By providing VEX during scanning, it is possible to filter vulnerabilities based on their status. + +## VEX Usage Methods + +Trivy currently supports two methods for utilizing VEX: + +1. [VEX Repository](./repo.md) +2. [Local VEX Files](./file.md) + +### Enabling VEX +To enable VEX, use the `--vex` option. +You can specify the method to use: + +- To enable the VEX Repository: `--vex repo` +- To use a local VEX file: `--vex /path/to/vex-document.json` + +```bash +$ trivy image ghcr.io/aquasecurity/trivy:0.52.0 --vex repo +``` + +You can enable both methods simultaneously. +The order of specification determines the priority: + +- `--vex repo --vex /path/to/vex-document.json`: VEX Repository has priority +- `--vex /path/to/vex-document.json --vex repo`: Local file has priority + +For detailed information on each method, please refer to each page. \ No newline at end of file diff --git a/docs/docs/supply-chain/vex/repo.md b/docs/docs/supply-chain/vex/repo.md new file mode 100644 index 000000000000..34fe21980d3b --- /dev/null +++ b/docs/docs/supply-chain/vex/repo.md @@ -0,0 +1,201 @@ +# VEX Repository + +## Using VEX Repository + +Trivy can download and utilize VEX documents from repositories that comply with [the VEX Repository Specification][vex-repo]. +While it's planned to be enabled by default in the future, currently it can be activated by explicitly specifying `--vex repo`. + +``` +$ trivy image ghcr.io/aquasecurity/trivy:0.52.0 --vex repo +2024-07-20T11:22:58+04:00 INFO [vex] The default repository config has been created +file_path="/Users/teppei/.trivy/vex/repository.yaml" +2024-07-20T11:23:23+04:00 INFO [vex] Updating repository... repo="default" url="https://github.com/aquasecurity/vexhub" +``` + +During scanning, Trivy generates PURLs for discovered packages and searches for matching PURLs in the VEX Repository. +If a match is found, the corresponding VEX is utilized. + +### Configuration File + +#### Default Configuration + +When `--vex repo` is specified for the first time, a default configuration file is created at `$HOME/.trivy/vex/repository.yaml`. +The home directory can be configured through environment variable `$XDG_DATA_HOME`. + +You can also create the configuration file in advance using the `trivy vex repo init` command and edit it. + +The default configuration file looks like this: + +```yaml +repositories: + - name: default + url: https://github.com/aquasecurity/vexhub + enabled: true + username: "" + password: "" + token: "" +``` + +By default, [VEX Hub][vexhub] managed by Aqua Security is used. +VEX Hub primarily trusts VEX documents published by the package maintainers. + +#### Show Configuration +You can see the config file path and the configured repositories with `trivy vex repo list`: + +```bash +$ trivy vex repo list +VEX Repositories (config: /home/username/.trivy/vex/repository.yaml) + +- Name: default + URL: https://github.com/aquasecurity/vexhub + Status: Enabled +``` + +#### Custom Repositories + +If you want to trust VEX documents published by other organizations or use your own VEX repository, you can specify a custom repository that complies with [the VEX Repository Specification][vex-repo]. +You can add a custom repository as below: + +```yaml +- name: custom + url: https://example.com/custom-repo + enabled: true +``` + + +#### Authentication + +For private repositories: + +- `username`/`password` can be used for Basic authentication +- `token` can be used for Bearer authentication + +```yaml +- name: custom + url: https://example.com/custom-repo + enabled: true + token: "my-token" +``` + +#### Repository Priority + +The priority of VEX repositories is determined by their order in the configuration file. +You can add repositories with higher priority than the default or even remove the default VEX Hub. + +```yaml +- name: repo1 + url: https://example.com/repo1 +- name: repo2 + url: https://example.com/repo2 +``` + +In this configuration, when Trivy detects a vulnerability in a package, it generates a PURL for that package and searches for matching VEX documents in the configured repositories. +The search process follows this order: + +1. Trivy first looks for a VEX document matching the package's PURL in `repo1`. +2. If no matching VEX document is found in `repo1`, Trivy then searches in `repo2`. +3. This process continues through all configured repositories until a match is found. + +If a matching VEX document is found in any repository (e.g., `repo1`), the search stops, and Trivy uses that VEX document. +Subsequent repositories (e.g., `repo2`) are not checked for that specific vulnerability and package combination. + +It's important to note that the first matching VEX document found determines the final status of the vulnerability. +For example, if `repo1` states that a package is "Affected" by a vulnerability, this status will be used even if `repo2` states that the same package is "Not Affected" for the same vulnerability. +The "Affected" status from the higher-priority repository (`repo1`) takes precedence, and Trivy will consider the package as affected by the vulnerability. + +### Repository Updates + +VEX repositories are automatically updated during scanning. +Updates are performed based on the update frequency specified by the repository. + +To download VEX repositories in advance without scanning, use `trivy vex repo download`. + +The cache can be cleared with `trivy clean --vex-repo`. + +### Displaying Filtered Vulnerabilities + +To see which vulnerabilities were filtered and why, use the `--show-suppressed` option: + +```shell +$ trivy image ghcr.io/aquasecurity/trivy:0.50.0 --vex repo --show-suppressed +... + +Suppressed Vulnerabilities (Total: 4) +===================================== +┌───────────────┬────────────────┬──────────┬──────────────┬───────────────────────────────────────────────────┬──────────────────────────────────────────┐ +│ Library │ Vulnerability │ Severity │ Status │ Statement │ Source │ +├───────────────┼────────────────┼──────────┼──────────────┼───────────────────────────────────────────────────┼──────────────────────────────────────────┤ +│ busybox │ CVE-2023-42364 │ MEDIUM │ not_affected │ vulnerable_code_cannot_be_controlled_by_adversary │ VEX Repository: default │ +│ │ │ │ │ │ (https://github.com/aquasecurity/vexhub) │ +│ ├────────────────┤ │ │ │ │ +│ │ CVE-2023-42365 │ │ │ │ │ +│ │ │ │ │ │ │ +├───────────────┼────────────────┤ │ │ │ │ +│ busybox-binsh │ CVE-2023-42364 │ │ │ │ │ +│ │ │ │ │ │ │ +│ ├────────────────┤ │ │ │ │ +│ │ CVE-2023-42365 │ │ │ │ │ +│ │ │ │ │ │ │ +└───────────────┴────────────────┴──────────┴──────────────┴───────────────────────────────────────────────────┴──────────────────────────────────────────┘ + +``` + +## Publishing VEX Documents + +### For OSS Projects + +As an OSS developer or maintainer, you may encounter vulnerabilities in the packages your project depends on. +These vulnerabilities might be discovered through your own scans or reported by third parties using your OSS project. + +While Trivy strives to minimize false positives, it doesn't perform code graph analysis, which means it can't evaluate exploitability at the code level. +Consequently, Trivy may report vulnerabilities even in cases where: + +1. The vulnerable function in a dependency is never called in your project. +2. The vulnerable code cannot be controlled by an attacker in the context of your project. + +If you're confident that a reported vulnerability in a dependency doesn't affect your OSS project or container image, you can publish a VEX document to reduce noise in Trivy scans. +To assess exploitability, you have several options: + +1. Manual assessment: As a maintainer, you can read the source code and determine if the vulnerability is exploitable in your project's context. +2. Automated assessment: You can use SAST (Static Application Security Testing) tools or similar tools to analyze the code and determine exploitability. + +By publishing VEX documents in the source repository, Trivy can automatically utilize them through VEX Hub. +The main steps are: + +1. Generate a VEX document +2. Commit the VEX document to the `.vex/` directory in the source repository (e.g., [Trivy's VEX][trivy-vex]) +3. Register your project's [PURL][purl] in VEX Hub + +Step 3 is only necessary once. +After that, updating the VEX file in your repository will automatically be fetched by VEX Hub and utilized by Trivy. +See the [VEX Hub repository][vexhub] for more information. + +If you want to issue a VEX for an OSS project that you don't maintain, consider first proposing the VEX publication to the original repository. +Many OSS maintainers are open to contributions that improve the security posture of their projects. +However, if your proposal is not accepted, or if you want to issue a VEX with statements that differ from the maintainer's judgment, you may want to consider creating a [custom repository](#hosting-custom-repositories). + +### For Private Projects + +If you're working on private software or personal projects, you have several options: + +1. [Local VEX files](./file.md): You can create local VEX files and have Trivy read them during scans. This is suitable for individual use or small teams. +2. [.trivyignore](../../configuration/filtering.md#trivyignore): For simpler cases, using a .trivyignore file might be sufficient to suppress specific vulnerabilities. +3. [Custom repositories](#hosting-custom-repositories): For large organizations wanting to share VEX information for internally used software across different departments, setting up a custom VEX repository might be the best approach. + +## Hosting Custom Repositories + +While the principle is to store VEX documents for OSS packages in the source repository, it's possible to create a custom repository if that's difficult. + +There are various use cases for providing custom repositories: + +- A Pull Request to add a VEX document upstream was not merged +- Consolidating VEX documents output by SAST tools +- Publishing vendor-specific VEX documents that differ from OSS maintainer statements +- Creating a private VEX repository to publish common VEX for your company + +In these cases, you can create a repository that complies with [the VEX Repository Specification][vex-repo] to make it available for use with Trivy. + +[vex-repo]: https://github.com/aquasecurity/vex-repo-spec +[vexhub]: https://github.com/aquasecurity/vexhub +[trivy-vex]: https://github.com/aquasecurity/trivy/blob/b76a7250912cfc028cfef743f0f98cd81b39f8aa/.vex/trivy.openvex.json +[purl]: https://github.com/package-url/purl-spec \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 2222a30220fb..6b000541fd16 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -124,10 +124,13 @@ nav: - Supply Chain: - SBOM: docs/supply-chain/sbom.md - Attestation: - - SBOM: docs/supply-chain/attestation/sbom.md - - Cosign Vulnerability Scan Record: docs/supply-chain/attestation/vuln.md - - SBOM Attestation in Rekor: docs/supply-chain/attestation/rekor.md - - VEX: docs/supply-chain/vex.md + - SBOM: docs/supply-chain/attestation/sbom.md + - Cosign Vulnerability Scan Record: docs/supply-chain/attestation/vuln.md + - SBOM Attestation in Rekor: docs/supply-chain/attestation/rekor.md + - VEX: + - Overview: docs/supply-chain/vex/index.md + - VEX Repository: docs/supply-chain/vex/repo.md + - Local VEX Files: docs/supply-chain/vex/file.md - Compliance: - Built-in Compliance: docs/compliance/compliance.md - Custom Compliance: docs/compliance/contrib-compliance.md From 2f04b6e8ee22432537ca98fb664e94fb4c93a00f Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Sat, 20 Jul 2024 22:08:20 +0400 Subject: [PATCH 34/55] feat(vex): add --skip-vex-repo-update Signed-off-by: knqyf263 --- docs/docs/supply-chain/vex/repo.md | 6 ++++++ pkg/commands/operation/operation.go | 7 ++++++- pkg/flag/vulnerability_flags.go | 26 ++++++++++++++++++-------- pkg/vex/repo.go | 14 ++++++++++---- pkg/vex/vex.go | 7 +++++-- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/docs/docs/supply-chain/vex/repo.md b/docs/docs/supply-chain/vex/repo.md index 34fe21980d3b..b4fa7a4ab399 100644 --- a/docs/docs/supply-chain/vex/repo.md +++ b/docs/docs/supply-chain/vex/repo.md @@ -108,6 +108,12 @@ The "Affected" status from the higher-priority repository (`repo1`) takes preced VEX repositories are automatically updated during scanning. Updates are performed based on the update frequency specified by the repository. +To disable auto-update, pass `--skip-vex-repo-update`. + +```shell +$ trivy image ghcr.io/aquasecurity/trivy:0.50.0 --vex repo --skip-vex-repo-update +``` + To download VEX repositories in advance without scanning, use `trivy vex repo download`. The cache can be cleared with `trivy clean --vex-repo`. diff --git a/pkg/commands/operation/operation.go b/pkg/commands/operation/operation.go index 478b39dbdf2a..53820e0aa5fd 100644 --- a/pkg/commands/operation/operation.go +++ b/pkg/commands/operation/operation.go @@ -50,6 +50,12 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n } func DownloadVEXRepositories(ctx context.Context, opts flag.Options) error { + ctx = log.WithContextPrefix(ctx, "vex") + if opts.SkipVEXRepoUpdate { + log.InfoContext(ctx, "Skipping VEX repository update") + return nil + } + mu.Lock() defer mu.Unlock() @@ -61,7 +67,6 @@ func DownloadVEXRepositories(ctx context.Context, opts flag.Options) error { return nil } - ctx = log.WithContextPrefix(ctx, "vex") err := repo.NewManager(opts.CacheDir).DownloadRepositories(ctx, nil, repo.Options{ Insecure: opts.Insecure, }) diff --git a/pkg/flag/vulnerability_flags.go b/pkg/flag/vulnerability_flags.go index f0176c547e76..da306997b5b8 100644 --- a/pkg/flag/vulnerability_flags.go +++ b/pkg/flag/vulnerability_flags.go @@ -25,24 +25,32 @@ var ( ConfigName: "vulnerability.vex", Usage: `[EXPERIMENTAL] VEX sources ("repo" or file path)`, } + SkipVEXRepoUpdateFlag = Flag[bool]{ + Name: "skip-vex-repo-update", + ConfigName: "vulnerability.skip-vex-repo-update", + Usage: `[EXPERIMENTAL] Skip VEX Repository update`, + } ) type VulnerabilityFlagGroup struct { - IgnoreUnfixed *Flag[bool] - IgnoreStatus *Flag[[]string] - VEX *Flag[[]string] + IgnoreUnfixed *Flag[bool] + IgnoreStatus *Flag[[]string] + VEX *Flag[[]string] + SkipVEXRepoUpdate *Flag[bool] } type VulnerabilityOptions struct { - IgnoreStatuses []dbTypes.Status - VEXSources []vex.Source + IgnoreStatuses []dbTypes.Status + VEXSources []vex.Source + SkipVEXRepoUpdate bool } func NewVulnerabilityFlagGroup() *VulnerabilityFlagGroup { return &VulnerabilityFlagGroup{ - IgnoreUnfixed: IgnoreUnfixedFlag.Clone(), - IgnoreStatus: IgnoreStatusFlag.Clone(), - VEX: VEXFlag.Clone(), + IgnoreUnfixed: IgnoreUnfixedFlag.Clone(), + IgnoreStatus: IgnoreStatusFlag.Clone(), + VEX: VEXFlag.Clone(), + SkipVEXRepoUpdate: SkipVEXRepoUpdateFlag.Clone(), } } @@ -55,6 +63,7 @@ func (f *VulnerabilityFlagGroup) Flags() []Flagger { f.IgnoreUnfixed, f.IgnoreStatus, f.VEX, + f.SkipVEXRepoUpdate, } } @@ -91,5 +100,6 @@ func (f *VulnerabilityFlagGroup) ToOptions() (VulnerabilityOptions, error) { VEXSources: lo.Map(f.VEX.Value(), func(s string, _ int) vex.Source { return vex.NewSource(s) }), + SkipVEXRepoUpdate: f.SkipVEXRepoUpdate.Value(), }, nil } diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go index f71c24eee97b..6b1813581b94 100644 --- a/pkg/vex/repo.go +++ b/pkg/vex/repo.go @@ -34,11 +34,12 @@ func NewRepositorySet(ctx context.Context, cacheDir string) (*RepositorySet, err return nil, xerrors.Errorf("failed to get VEX repository config: %w", err) } + logger := log.WithPrefix("vex") var indexes []RepositoryIndex for _, r := range conf.EnabledRepositories() { index, err := r.Index(ctx) if errors.Is(err, os.ErrNotExist) { - log.Warn("VEX repository not found locally, skipping this repository", log.String("repo", r.Name)) + logger.Warn("VEX repository not found locally, skipping this repository", log.String("repo", r.Name)) continue } else if err != nil { return nil, xerrors.Errorf("failed to get VEX repository index: %w", err) @@ -49,9 +50,14 @@ func NewRepositorySet(ctx context.Context, cacheDir string) (*RepositorySet, err Index: index, }) } + if len(indexes) == 0 { + logger.Warn("No available VEX repository found locally") + return nil, nil + } + return &RepositorySet{ indexes: indexes, // In precedence order - logger: log.WithPrefix("vex"), + logger: logger, }, nil } @@ -83,7 +89,7 @@ func (rs *RepositorySet) NotAffected(vuln types.DetectedVulnerability, product, source := fmt.Sprintf("VEX Repository: %s (%s)", index.Name, index.URL) doc, err := rs.OpenDocument(source, filepath.Dir(index.Path), entry) if err != nil { - log.Warn("Failed to open the VEX document", log.String("location", entry.Location), log.Err(err)) + rs.logger.Warn("Failed to open the VEX document", log.String("location", entry.Location), log.Err(err)) return types.ModifiedFinding{}, false } @@ -91,7 +97,7 @@ func (rs *RepositorySet) NotAffected(vuln types.DetectedVulnerability, product, return m, notAffected } - log.Debug("VEX found, but affected", log.String("vulnerability", vuln.VulnerabilityID), + rs.logger.Debug("VEX found, but affected", log.String("vulnerability", vuln.VulnerabilityID), log.String("package", pkgID), log.String("repo", index.Name), log.String("repo_url", index.URL)) break // Stop searching for the next VEX document as this repository has higher precedence. } diff --git a/pkg/vex/vex.go b/pkg/vex/vex.go index 793c96c41830..3e68a996b853 100644 --- a/pkg/vex/vex.go +++ b/pkg/vex/vex.go @@ -95,16 +95,19 @@ func New(ctx context.Context, report *types.Report, opts Options) (*Client, erro case TypeRepository: v, err = NewRepositorySet(ctx, opts.CacheDir) if err != nil { - return nil, xerrors.Errorf("failed to create a repository set: %w", err) + return nil, xerrors.Errorf("failed to create a vex repository set: %w", err) } default: log.Warn("Unsupported VEX source", log.String("type", string(src.Type))) continue } - vexes = append(vexes, v) + if !lo.IsNil(v) { + vexes = append(vexes, v) + } } if len(vexes) == 0 { + log.DebugContext(ctx, "VEX filtering is disabled") return nil, nil } return &Client{VEXes: vexes}, nil From 8fcd8d2abb39acd0af9dce2d182faf169a1b1204 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 22 Jul 2024 15:49:53 +0400 Subject: [PATCH 35/55] test(integration): add VEX tests Signed-off-by: knqyf263 --- integration/integration_test.go | 39 +++++ integration/repo_test.go | 34 +++- .../fixtures/vex/config/repository.yaml | 4 + .../testdata/fixtures/vex/file/openvex.json | 23 +++ .../vex/repositories/default/0.1/index.json | 9 ++ .../repositories/default/vex-repository.json | 15 ++ integration/testdata/gomod-vex.json.golden | 148 ++++++++++++++++++ internal/testutil/fs.go | 58 +++++++ pkg/downloader/downloader_test.go | 4 +- pkg/vex/repo/manager_test.go | 32 +--- pkg/vex/repo/repo_test.go | 54 ++----- 11 files changed, 352 insertions(+), 68 deletions(-) create mode 100644 integration/testdata/fixtures/vex/config/repository.yaml create mode 100644 integration/testdata/fixtures/vex/file/openvex.json create mode 100644 integration/testdata/fixtures/vex/repositories/default/0.1/index.json create mode 100644 integration/testdata/fixtures/vex/repositories/default/vex-repository.json create mode 100644 integration/testdata/gomod-vex.json.golden diff --git a/integration/integration_test.go b/integration/integration_test.go index c7c923af0c33..c263ec2fc51e 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -28,11 +28,13 @@ import ( "github.com/aquasecurity/trivy-db/pkg/metadata" "github.com/aquasecurity/trivy/internal/dbtest" + "github.com/aquasecurity/trivy/internal/testutil" "github.com/aquasecurity/trivy/pkg/clock" "github.com/aquasecurity/trivy/pkg/commands" "github.com/aquasecurity/trivy/pkg/db" "github.com/aquasecurity/trivy/pkg/types" "github.com/aquasecurity/trivy/pkg/uuid" + "github.com/aquasecurity/trivy/pkg/vex/repo" _ "modernc.org/sqlite" ) @@ -68,6 +70,43 @@ func initDB(t *testing.T) string { return cacheDir } +func initVEXRepository(t *testing.T, homeDir, cacheDir string) { + t.Helper() + + // Copy config directory + configSrc := "testdata/fixtures/vex/config/repository.yaml" + configDst := filepath.Join(homeDir, ".trivy", "vex", "repository.yaml") + testutil.CopyFile(t, configSrc, configDst) + + // Copy repository directory + repoSrc := "testdata/fixtures/vex/repositories" + repoDst := filepath.Join(cacheDir, "vex", "repositories") + testutil.CopyDir(t, repoSrc, repoDst) + + // Copy VEX file + vexSrc := "testdata/fixtures/vex/file/openvex.json" + repoDir := filepath.Join(repoDst, "default") + vexDst := filepath.Join(repoDir, "0.1", "openvex.json") + testutil.CopyFile(t, vexSrc, vexDst) + + // Write a dummy cache metadata + testutil.MustWriteJSON(t, filepath.Join(repoDir, "cache.json"), repo.CacheMetadata{ + UpdatedAt: time.Now(), + }) + + // Verify that necessary files exist + requiredFiles := []string{ + configDst, + filepath.Join(repoDir, "vex-repository.json"), + filepath.Join(repoDir, "0.1", "index.json"), + filepath.Join(repoDir, "0.1", "openvex.json"), + } + + for _, file := range requiredFiles { + require.FileExists(t, file) + } +} + func getFreePort() (int, error) { addr, err := net.ResolveTCPAddr("tcp", "localhost:0") if err != nil { diff --git a/integration/repo_test.go b/integration/repo_test.go index b95b4f4f461d..69833bcf2b77 100644 --- a/integration/repo_test.go +++ b/integration/repo_test.go @@ -8,9 +8,10 @@ import ( "strings" "testing" + "github.com/stretchr/testify/require" + "github.com/aquasecurity/trivy/pkg/fanal/artifact" "github.com/aquasecurity/trivy/pkg/types" - "github.com/stretchr/testify/require" ) // TestRepository tests `trivy repo` with the local code repositories @@ -32,6 +33,7 @@ func TestRepository(t *testing.T) { format types.Format includeDevDeps bool parallel int + vex string } tests := []struct { name string @@ -74,6 +76,24 @@ func TestRepository(t *testing.T) { }, golden: "testdata/gomod.json.golden", }, + { + name: "gomod with local VEX file", + args: args{ + scanner: types.VulnerabilityScanner, + input: "testdata/fixtures/repo/gomod", + vex: "testdata/fixtures/vex/file/openvex.json", + }, + golden: "testdata/gomod-vex.json.golden", + }, + { + name: "gomod with VEX repository", + args: args{ + scanner: types.VulnerabilityScanner, + input: "testdata/fixtures/repo/gomod", + vex: "repo", + }, + golden: "testdata/gomod-vex.json.golden", + }, { name: "npm", args: args{ @@ -437,9 +457,15 @@ func TestRepository(t *testing.T) { // Set up testing DB cacheDir := initDB(t) - // Set a temp dir so that modules will not be loaded + // Set up VEX + initVEXRepository(t, cacheDir, cacheDir) + + // Set a temp dir so that the VEX config will be loaded and modules will not be loaded t.Setenv("XDG_DATA_HOME", cacheDir) + // Disable Go license detection + t.Setenv("GOPATH", cacheDir) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { command := "repo" @@ -532,6 +558,10 @@ func TestRepository(t *testing.T) { osArgs = append(osArgs, "--secret-config", tt.args.secretConfig) } + if tt.args.vex != "" { + osArgs = append(osArgs, "--vex", tt.args.vex) + } + runTest(t, osArgs, tt.golden, "", format, runOptions{ fakeUUID: "3ff14136-e09f-4df9-80ea-%012d", override: tt.override, diff --git a/integration/testdata/fixtures/vex/config/repository.yaml b/integration/testdata/fixtures/vex/config/repository.yaml new file mode 100644 index 000000000000..cd1865d9d4a5 --- /dev/null +++ b/integration/testdata/fixtures/vex/config/repository.yaml @@ -0,0 +1,4 @@ +repositories: + - name: default + url: https://localhost + enabled: true \ No newline at end of file diff --git a/integration/testdata/fixtures/vex/file/openvex.json b/integration/testdata/fixtures/vex/file/openvex.json new file mode 100644 index 000000000000..38773ba57694 --- /dev/null +++ b/integration/testdata/fixtures/vex/file/openvex.json @@ -0,0 +1,23 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "aquasecurity/trivy:613fd55abbc2857b5ca28b07a26f3cd4c8b0ddc4c8a97c57497a2d4c4880d7fc", + "author": "Aqua Security", + "timestamp": "2024-07-09T11:38:00.115697+04:00", + "version": 1, + "statements": [ + { + "vulnerability": { "@id": "CVE-2022-23628" }, + "products": [ + { + "@id": "pkg:golang/github.com/testdata/testdata", + "subcomponents": [ + { "@id": "pkg:golang/github.com/open-policy-agent/opa@0.35.0" } + ] + } + ], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path", + "impact_statement": "The vulnerable code isn't called" + } + ] +} diff --git a/integration/testdata/fixtures/vex/repositories/default/0.1/index.json b/integration/testdata/fixtures/vex/repositories/default/0.1/index.json new file mode 100644 index 000000000000..12d7ee936e8f --- /dev/null +++ b/integration/testdata/fixtures/vex/repositories/default/0.1/index.json @@ -0,0 +1,9 @@ +{ + "version": 1, + "packages": [ + { + "ID": "pkg:golang/github.com/testdata/testdata", + "Location": "./openvex.json" + } + ] +} diff --git a/integration/testdata/fixtures/vex/repositories/default/vex-repository.json b/integration/testdata/fixtures/vex/repositories/default/vex-repository.json new file mode 100644 index 000000000000..e064c0e1b3cf --- /dev/null +++ b/integration/testdata/fixtures/vex/repositories/default/vex-repository.json @@ -0,0 +1,15 @@ +{ + "name": "Test VEX Repository", + "description": "VEX Repository for Testing", + "versions": [ + { + "spec_version": "0.1", + "locations": [ + { + "url": "never used" + } + ], + "update_interval": "24h" + } + ] +} \ No newline at end of file diff --git a/integration/testdata/gomod-vex.json.golden b/integration/testdata/gomod-vex.json.golden new file mode 100644 index 000000000000..a2269bd1d0b7 --- /dev/null +++ b/integration/testdata/gomod-vex.json.golden @@ -0,0 +1,148 @@ +{ + "SchemaVersion": 2, + "CreatedAt": "2021-08-25T12:20:30.000000005Z", + "ArtifactName": "testdata/fixtures/repo/gomod", + "ArtifactType": "repository", + "Metadata": { + "ImageConfig": { + "architecture": "", + "created": "0001-01-01T00:00:00Z", + "os": "", + "rootfs": { + "type": "", + "diff_ids": null + }, + "config": {} + } + }, + "Results": [ + { + "Target": "go.mod", + "Class": "lang-pkgs", + "Type": "gomod", + "Vulnerabilities": [ + { + "VulnerabilityID": "GMS-2022-20", + "PkgID": "github.com/docker/distribution@v2.7.1+incompatible", + "PkgName": "github.com/docker/distribution", + "PkgIdentifier": { + "PURL": "pkg:golang/github.com/docker/distribution@2.7.1%2Bincompatible", + "UID": "de19cd663ca047a8" + }, + "InstalledVersion": "2.7.1+incompatible", + "FixedVersion": "v2.8.0", + "Status": "fixed", + "Layer": {}, + "DataSource": { + "ID": "ghsa", + "Name": "GitHub Security Advisory Go", + "URL": "https://github.com/advisories?query=type%3Areviewed+ecosystem%3Ago" + }, + "Title": "OCI Manifest Type Confusion Issue", + "Description": "### Impact\n\nSystems that rely on digest equivalence for image attestations may be vulnerable to type confusion.", + "Severity": "UNKNOWN", + "References": [ + "https://github.com/advisories/GHSA-qq97-vm5h-rrhg", + "https://github.com/distribution/distribution/commit/b59a6f827947f9e0e67df0cfb571046de4733586", + "https://github.com/distribution/distribution/security/advisories/GHSA-qq97-vm5h-rrhg", + "https://github.com/opencontainers/image-spec/pull/411" + ] + }, + { + "VulnerabilityID": "CVE-2021-38561", + "PkgID": "golang.org/x/text@v0.3.6", + "PkgName": "golang.org/x/text", + "PkgIdentifier": { + "PURL": "pkg:golang/golang.org/x/text@0.3.6", + "UID": "825dc613c0f39d45" + }, + "InstalledVersion": "0.3.6", + "FixedVersion": "0.3.7", + "Status": "fixed", + "Layer": {}, + "PrimaryURL": "https://avd.aquasec.com/nvd/cve-2021-38561", + "DataSource": { + "ID": "ghsa", + "Name": "GitHub Security Advisory Go", + "URL": "https://github.com/advisories?query=type%3Areviewed+ecosystem%3Ago" + }, + "Description": "Due to improper index calculation, an incorrectly formatted language tag can cause Parse\nto panic via an out of bounds read. If Parse is used to process untrusted user inputs,\nthis may be used as a vector for a denial of service attack.\n", + "Severity": "UNKNOWN", + "References": [ + "https://go-review.googlesource.com/c/text/+/340830", + "https://go.googlesource.com/text/+/383b2e75a7a4198c42f8f87833eefb772868a56f", + "https://pkg.go.dev/vuln/GO-2021-0113" + ] + } + ] + }, + { + "Target": "submod/go.mod", + "Class": "lang-pkgs", + "Type": "gomod", + "Vulnerabilities": [ + { + "VulnerabilityID": "GMS-2022-20", + "PkgID": "github.com/docker/distribution@v2.7.1+incompatible", + "PkgName": "github.com/docker/distribution", + "PkgIdentifier": { + "PURL": "pkg:golang/github.com/docker/distribution@2.7.1%2Bincompatible", + "UID": "94376dc37054a7e8" + }, + "InstalledVersion": "2.7.1+incompatible", + "FixedVersion": "v2.8.0", + "Status": "fixed", + "Layer": {}, + "DataSource": { + "ID": "ghsa", + "Name": "GitHub Security Advisory Go", + "URL": "https://github.com/advisories?query=type%3Areviewed+ecosystem%3Ago" + }, + "Title": "OCI Manifest Type Confusion Issue", + "Description": "### Impact\n\nSystems that rely on digest equivalence for image attestations may be vulnerable to type confusion.", + "Severity": "UNKNOWN", + "References": [ + "https://github.com/advisories/GHSA-qq97-vm5h-rrhg", + "https://github.com/distribution/distribution/commit/b59a6f827947f9e0e67df0cfb571046de4733586", + "https://github.com/distribution/distribution/security/advisories/GHSA-qq97-vm5h-rrhg", + "https://github.com/opencontainers/image-spec/pull/411" + ] + } + ] + }, + { + "Target": "submod2/go.mod", + "Class": "lang-pkgs", + "Type": "gomod", + "Vulnerabilities": [ + { + "VulnerabilityID": "GMS-2022-20", + "PkgID": "github.com/docker/distribution@v2.7.1+incompatible", + "PkgName": "github.com/docker/distribution", + "PkgIdentifier": { + "PURL": "pkg:golang/github.com/docker/distribution@2.7.1%2Bincompatible", + "UID": "94306cdcf85fb50a" + }, + "InstalledVersion": "2.7.1+incompatible", + "FixedVersion": "v2.8.0", + "Status": "fixed", + "Layer": {}, + "DataSource": { + "ID": "ghsa", + "Name": "GitHub Security Advisory Go", + "URL": "https://github.com/advisories?query=type%3Areviewed+ecosystem%3Ago" + }, + "Title": "OCI Manifest Type Confusion Issue", + "Description": "### Impact\n\nSystems that rely on digest equivalence for image attestations may be vulnerable to type confusion.", + "Severity": "UNKNOWN", + "References": [ + "https://github.com/advisories/GHSA-qq97-vm5h-rrhg", + "https://github.com/distribution/distribution/commit/b59a6f827947f9e0e67df0cfb571046de4733586", + "https://github.com/distribution/distribution/security/advisories/GHSA-qq97-vm5h-rrhg", + "https://github.com/opencontainers/image-spec/pull/411" + ] + } + ] + } + ] +} diff --git a/internal/testutil/fs.go b/internal/testutil/fs.go index 4a1162aa1bab..19f140c9c91a 100644 --- a/internal/testutil/fs.go +++ b/internal/testutil/fs.go @@ -1,15 +1,25 @@ package testutil import ( + "encoding/json" "os" "path/filepath" "testing" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" "github.com/aquasecurity/trivy/pkg/utils/fsutils" ) +func CopyFile(t *testing.T, src, dst string) { + err := os.MkdirAll(filepath.Dir(dst), 0755) + require.NoError(t, err) + + _, err = fsutils.CopyFile(src, dst) + require.NoError(t, err) +} + // CopyDir copies the directory content from src to dst. // It supports only simple cases for testing. func CopyDir(t *testing.T, src, dst string) { @@ -34,3 +44,51 @@ func CopyDir(t *testing.T, src, dst string) { } } } + +func MustWriteYAML(t *testing.T, path string, data interface{}) { + t.Helper() + dir := filepath.Dir(path) + require.NoError(t, os.MkdirAll(dir, 0755)) + + f, err := os.Create(path) + require.NoError(t, err) + defer f.Close() + + require.NoError(t, yaml.NewEncoder(f).Encode(data)) +} + +func MustReadYAML(t *testing.T, path string, out interface{}) { + t.Helper() + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + + require.NoError(t, yaml.NewDecoder(f).Decode(out)) +} + +func MustMkdirAll(t *testing.T, dir string) { + err := os.MkdirAll(dir, 0755) + require.NoError(t, err) +} + +func MustReadJSON(t *testing.T, filePath string, v interface{}) { + b, err := os.ReadFile(filePath) + require.NoError(t, err) + err = json.Unmarshal(b, v) + require.NoError(t, err) +} + +func MustWriteJSON(t *testing.T, filePath string, v interface{}) { + data, err := json.Marshal(v) + require.NoError(t, err) + + MustWriteFile(t, filePath, data) +} + +func MustWriteFile(t *testing.T, filePath string, content []byte) { + dir := filepath.Dir(filePath) + MustMkdirAll(t, dir) + + err := os.WriteFile(filePath, content, 0744) + require.NoError(t, err) +} diff --git a/pkg/downloader/downloader_test.go b/pkg/downloader/downloader_test.go index 0487d7384d41..796f4b96b1d2 100644 --- a/pkg/downloader/downloader_test.go +++ b/pkg/downloader/downloader_test.go @@ -44,7 +44,9 @@ func TestDownload(t *testing.T) { dst := t.TempDir() // Execute the download - err := downloader.Download(context.Background(), server.URL, dst, "", tt.insecure) + _, err := downloader.Download(context.Background(), server.URL, dst, "", downloader.Options{ + Insecure: tt.insecure, + }) if tt.wantErr { assert.Error(t, err) diff --git a/pkg/vex/repo/manager_test.go b/pkg/vex/repo/manager_test.go index af1a339f4f97..362137bd1804 100644 --- a/pkg/vex/repo/manager_test.go +++ b/pkg/vex/repo/manager_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "github.com/aquasecurity/trivy/internal/testutil" "github.com/aquasecurity/trivy/pkg/vex/repo" ) @@ -35,7 +35,7 @@ func TestManager_Config(t *testing.T) { }, } configPath := filepath.Join(dir, ".trivy", "vex", "repository.yaml") - mustWriteYAML(t, configPath, config) + testutil.MustWriteYAML(t, configPath, config) }, want: repo.Config{ Repositories: []repo.Repository{ @@ -105,7 +105,7 @@ func TestManager_Init(t *testing.T) { name: "config already exists", setup: func(t *testing.T, dir string) { configPath := filepath.Join(dir, ".trivy", "vex", "repository.yaml") - mustWriteYAML(t, configPath, repo.Config{}) + testutil.MustWriteYAML(t, configPath, repo.Config{}) }, want: repo.Config{ Repositories: []repo.Repository{}, @@ -132,7 +132,7 @@ func TestManager_Init(t *testing.T) { assert.FileExists(t, configPath) var got repo.Config - mustReadYAML(t, configPath, &got) + testutil.MustReadYAML(t, configPath, &got) assert.Equal(t, tt.want, got) }) } @@ -222,11 +222,11 @@ func TestManager_DownloadRepositories(t *testing.T) { m := repo.NewManager(tempDir) configPath := filepath.Join(tempDir, ".trivy", "vex", "repository.yaml") - mustWriteYAML(t, configPath, tt.config) + testutil.MustWriteYAML(t, configPath, tt.config) manifestPath := filepath.Join(tempDir, "vex", "repositories", "test-repo", "vex-repository.json") manifest.Versions[0].Locations[0].URL = tt.location - mustWriteJSON(t, manifestPath, manifest) + testutil.MustWriteJSON(t, manifestPath, manifest) err := m.DownloadRepositories(context.Background(), tt.names, repo.Options{}) if tt.wantErr != "" { @@ -298,7 +298,7 @@ No repositories configured. tempDir := t.TempDir() t.Setenv("XDG_DATA_HOME", tempDir) configPath := filepath.Join(tempDir, ".trivy", "vex", "repository.yaml") - mustWriteYAML(t, configPath, tt.config) + testutil.MustWriteYAML(t, configPath, tt.config) var buf bytes.Buffer m := repo.NewManager(tempDir, repo.WithWriter(&buf)) @@ -333,21 +333,3 @@ func TestManager_Clear(t *testing.T) { _, err = os.Stat(cacheDir) assert.True(t, os.IsNotExist(err)) } - -func mustWriteYAML(t *testing.T, path string, data interface{}) { - t.Helper() - dir := filepath.Dir(path) - require.NoError(t, os.MkdirAll(dir, 0755)) - f, err := os.Create(path) - require.NoError(t, err) - defer f.Close() - require.NoError(t, yaml.NewEncoder(f).Encode(data)) -} - -func mustReadYAML(t *testing.T, path string, out interface{}) { - t.Helper() - f, err := os.Open(path) - require.NoError(t, err) - defer f.Close() - require.NoError(t, yaml.NewDecoder(f).Decode(out)) -} diff --git a/pkg/vex/repo/repo_test.go b/pkg/vex/repo/repo_test.go index 572422ad2683..086b712229ba 100644 --- a/pkg/vex/repo/repo_test.go +++ b/pkg/vex/repo/repo_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/aquasecurity/trivy/internal/testutil" "github.com/aquasecurity/trivy/pkg/clock" "github.com/aquasecurity/trivy/pkg/vex/repo" ) @@ -57,7 +58,7 @@ func TestRepository_Manifest(t *testing.T) { name: "local manifest exists", setup: func(t *testing.T, dir string, _ *repo.Repository) { manifestFile := filepath.Join(dir, "vex", "repositories", "test-repo", "vex-repository.json") - mustWriteJSON(t, manifestFile, manifest) + testutil.MustWriteJSON(t, manifestFile, manifest) }, want: manifest, }, @@ -124,7 +125,7 @@ func TestRepository_Index(t *testing.T) { } indexPath := filepath.Join(cacheDir, "vex", "repositories", r.Name, "0.1", "index.json") - mustWriteJSON(t, indexPath, indexData) + testutil.MustWriteJSON(t, indexPath, indexData) }, want: repo.Index{ UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), @@ -151,7 +152,7 @@ func TestRepository_Index(t *testing.T) { name: "invalid JSON in index file", setup: func(t *testing.T, cacheDir string, r *repo.Repository) { indexPath := filepath.Join(cacheDir, "vex", "repositories", r.Name, "0.1", "index.json") - mustWriteFile(t, indexPath, []byte("invalid JSON")) + testutil.MustWriteFile(t, indexPath, []byte("invalid JSON")) }, wantErr: "failed to decode the index", }, @@ -206,13 +207,13 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, "") // No location as the test server is not used repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "0.1")) + testutil.MustMkdirAll(t, filepath.Join(repoDir, "0.1")) cacheMetadata := repo.CacheMetadata{ UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), ETags: map[string]string{ts.URL + "/archive.zip": "current-etag"}, } - mustWriteJSON(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) + testutil.MustWriteJSON(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) }, clockTime: time.Date(2023, 1, 1, 1, 30, 0, 0, time.UTC), wantCache: repo.CacheMetadata{ @@ -226,13 +227,13 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, ts.URL+"/archive.zip") repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "0.1")) + testutil.MustMkdirAll(t, filepath.Join(repoDir, "0.1")) cacheMetadata := repo.CacheMetadata{ UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), ETags: map[string]string{ts.URL + "/archive.zip": "old-etag"}, } - mustWriteJSON(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) + testutil.MustWriteJSON(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) }, clockTime: time.Date(2023, 1, 2, 3, 0, 0, 0, time.UTC), wantCache: repo.CacheMetadata{ @@ -246,13 +247,13 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, ts.URL+"/archive.zip") repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "0.1")) + testutil.MustMkdirAll(t, filepath.Join(repoDir, "0.1")) cacheMetadata := repo.CacheMetadata{ UpdatedAt: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), ETags: map[string]string{ts.URL + "/archive.zip": "current-etag"}, } - mustWriteJSON(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) + testutil.MustWriteJSON(t, filepath.Join(repoDir, "cache.json"), cacheMetadata) }, clockTime: time.Date(2023, 1, 2, 3, 0, 0, 0, time.UTC), wantCache: repo.CacheMetadata{ @@ -266,7 +267,7 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, ts.URL+"/archive.zip") repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "0.1")) + testutil.MustMkdirAll(t, filepath.Join(repoDir, "0.1")) }, clockTime: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), wantCache: repo.CacheMetadata{ @@ -286,7 +287,7 @@ func TestRepository_Update(t *testing.T) { setUpManifest(t, cacheDir, ts.URL+"/error") repoDir := filepath.Join(cacheDir, "vex", "repositories", r.Name) - mustMkdirAll(t, filepath.Join(repoDir, "0.1")) + testutil.MustMkdirAll(t, filepath.Join(repoDir, "0.1")) }, clockTime: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), wantErr: "failed to download the repository", @@ -314,7 +315,7 @@ func TestRepository_Update(t *testing.T) { cacheFile := filepath.Join(tempDir, "vex", "repositories", r.Name, "cache.json") var gotCache repo.CacheMetadata - mustReadJSON(t, cacheFile, &gotCache) + testutil.MustReadJSON(t, cacheFile, &gotCache) assert.Equal(t, tt.wantCache, gotCache) }) } @@ -343,7 +344,7 @@ func setUpManifest(t *testing.T, dir, url string) { }, } manifestPath := filepath.Join(dir, "vex", "repositories", "test-repo", "vex-repository.json") - mustWriteJSON(t, manifestPath, manifest) + testutil.MustWriteJSON(t, manifestPath, manifest) } func setUpRepository(t *testing.T) *httptest.Server { @@ -366,30 +367,3 @@ func setUpRepository(t *testing.T) *httptest.Server { } })) } - -func mustMkdirAll(t *testing.T, dir string) { - err := os.MkdirAll(dir, 0755) - require.NoError(t, err) -} - -func mustReadJSON(t *testing.T, filePath string, v interface{}) { - b, err := os.ReadFile(filePath) - require.NoError(t, err) - err = json.Unmarshal(b, v) - require.NoError(t, err) -} - -func mustWriteJSON(t *testing.T, filePath string, v interface{}) { - data, err := json.Marshal(v) - require.NoError(t, err) - - mustWriteFile(t, filePath, data) -} - -func mustWriteFile(t *testing.T, filePath string, content []byte) { - dir := filepath.Dir(filePath) - mustMkdirAll(t, dir) - - err := os.WriteFile(filePath, content, 0744) - require.NoError(t, err) -} From e2ce02f0d7560c957ea0e91febc364be8a293c3f Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 22 Jul 2024 16:25:51 +0400 Subject: [PATCH 36/55] fix(plugin): join plugins dir Signed-off-by: knqyf263 --- pkg/plugin/plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index b427ed12e423..7af8edec0388 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -165,5 +165,5 @@ func (p *Plugin) Dir() string { if p.dir != "" { return p.dir } - return filepath.Join(fsutils.TrivyHomeDir(), p.Name) + return filepath.Join(fsutils.TrivyHomeDir(), pluginsDir, p.Name) } From bfb24cf8edfef9170263870a078de382302bfa8e Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 22 Jul 2024 17:22:17 +0400 Subject: [PATCH 37/55] test(vex): comply with new spec Signed-off-by: knqyf263 --- pkg/result/filter_test.go | 11 +- .../default/{v0 => 0.1}/bash-vex.json | 0 .../default/{v0 => 0.1}/index.json | 0 .../repositories/default/vex-repository.json | 15 +++ pkg/vex/vex_test.go | 108 ++++++++++++------ 5 files changed, 100 insertions(+), 34 deletions(-) rename pkg/vex/testdata/single-repo/vex/repositories/default/{v0 => 0.1}/bash-vex.json (100%) rename pkg/vex/testdata/single-repo/vex/repositories/default/{v0 => 0.1}/index.json (100%) create mode 100644 pkg/vex/testdata/single-repo/vex/repositories/default/vex-repository.json diff --git a/pkg/result/filter_test.go b/pkg/result/filter_test.go index c8eb2e3e3bcf..9ef7e50e0f58 100644 --- a/pkg/result/filter_test.go +++ b/pkg/result/filter_test.go @@ -15,6 +15,7 @@ import ( ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/result" "github.com/aquasecurity/trivy/pkg/types" + "github.com/aquasecurity/trivy/pkg/vex" ) func TestFilter(t *testing.T) { @@ -1007,9 +1008,17 @@ func TestFilter(t *testing.T) { fakeTime := time.Date(2020, 8, 10, 7, 28, 17, 958601, time.UTC) ctx := clock.With(context.Background(), fakeTime) + var vexSources []vex.Source + if tt.args.vexPath != "" { + vexSources = append(vexSources, vex.Source{ + Type: vex.TypeFile, + FilePath: tt.args.vexPath, + }) + } + err := result.Filter(ctx, tt.args.report, result.FilterOptions{ Severities: tt.args.severities, - VEXPath: tt.args.vexPath, + VEXSources: vexSources, IgnoreStatuses: tt.args.ignoreStatuses, IgnoreFile: tt.args.ignoreFile, PolicyFile: tt.args.policyFile, diff --git a/pkg/vex/testdata/single-repo/vex/repositories/default/v0/bash-vex.json b/pkg/vex/testdata/single-repo/vex/repositories/default/0.1/bash-vex.json similarity index 100% rename from pkg/vex/testdata/single-repo/vex/repositories/default/v0/bash-vex.json rename to pkg/vex/testdata/single-repo/vex/repositories/default/0.1/bash-vex.json diff --git a/pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json b/pkg/vex/testdata/single-repo/vex/repositories/default/0.1/index.json similarity index 100% rename from pkg/vex/testdata/single-repo/vex/repositories/default/v0/index.json rename to pkg/vex/testdata/single-repo/vex/repositories/default/0.1/index.json diff --git a/pkg/vex/testdata/single-repo/vex/repositories/default/vex-repository.json b/pkg/vex/testdata/single-repo/vex/repositories/default/vex-repository.json new file mode 100644 index 000000000000..e064c0e1b3cf --- /dev/null +++ b/pkg/vex/testdata/single-repo/vex/repositories/default/vex-repository.json @@ -0,0 +1,15 @@ +{ + "name": "Test VEX Repository", + "description": "VEX Repository for Testing", + "versions": [ + { + "spec_version": "0.1", + "locations": [ + { + "url": "never used" + } + ], + "update_interval": "24h" + } + ] +} \ No newline at end of file diff --git a/pkg/vex/vex_test.go b/pkg/vex/vex_test.go index 3fa30fd79153..c76a9961643c 100644 --- a/pkg/vex/vex_test.go +++ b/pkg/vex/vex_test.go @@ -174,7 +174,12 @@ func TestFilter(t *testing.T) { }), }), opts: vex.Options{ - VEXPath: "testdata/openvex.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/openvex.json", + }, + }, }, }, want: imageReport([]types.Result{ @@ -198,7 +203,12 @@ func TestFilter(t *testing.T) { }), }), opts: vex.Options{ - VEXPath: "testdata/openvex-multiple.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/openvex-multiple.json", + }, + }, }, }, want: imageReport([]types.Result{ @@ -221,7 +231,12 @@ func TestFilter(t *testing.T) { }), }), opts: vex.Options{ - VEXPath: "testdata/openvex-oci.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/openvex-oci.json", + }, + }, }, }, want: imageReport([]types.Result{ @@ -240,7 +255,12 @@ func TestFilter(t *testing.T) { }), }), opts: vex.Options{ - VEXPath: "testdata/openvex-oci-mismatch.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/openvex-oci-mismatch.json", + }, + }, }, }, want: imageReport([]types.Result{ @@ -260,7 +280,12 @@ func TestFilter(t *testing.T) { }), }), opts: vex.Options{ - VEXPath: "testdata/openvex-nested.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/openvex-nested.json", + }, + }, }, }, want: fsReport([]types.Result{ @@ -281,7 +306,12 @@ func TestFilter(t *testing.T) { }), }), opts: vex.Options{ - VEXPath: "testdata/openvex-nested.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/openvex-nested.json", + }, + }, }, }, want: fsReport([]types.Result{ @@ -306,7 +336,12 @@ func TestFilter(t *testing.T) { }, }, opts: vex.Options{ - VEXPath: "testdata/cyclonedx.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/cyclonedx.json", + }, + }, }, }, want: &types.Report{ @@ -339,7 +374,12 @@ func TestFilter(t *testing.T) { }, }, opts: vex.Options{ - VEXPath: "testdata/cyclonedx.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/cyclonedx.json", + }, + }, }, }, want: &types.Report{ @@ -366,7 +406,12 @@ func TestFilter(t *testing.T) { }), }), opts: vex.Options{ - VEXPath: "testdata/csaf.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/csaf.json", + }, + }, }, }, want: imageReport([]types.Result{ @@ -387,7 +432,12 @@ func TestFilter(t *testing.T) { }), }), opts: vex.Options{ - VEXPath: "testdata/csaf-relationships.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/csaf-relationships.json", + }, + }, }, }, want: imageReport([]types.Result{ @@ -408,7 +458,12 @@ func TestFilter(t *testing.T) { }), }), opts: vex.Options{ - VEXPath: "testdata/csaf-relationships.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/csaf-relationships.json", + }, + }, }, }, want: imageReport([]types.Result{ @@ -428,7 +483,8 @@ func TestFilter(t *testing.T) { configContent := ` repositories: - name: default - url: https://example.com/vex/default` + url: https://example.com/vex/default + enabled: true` require.NoError(t, os.WriteFile(configPath, []byte(configContent), 0644)) }, args: args{ @@ -441,6 +497,7 @@ repositories: }), opts: vex.Options{ CacheDir: "testdata/single-repo", + Sources: []vex.Source{{Type: vex.TypeRepository}}, }, }, want: imageReport([]types.Result{ @@ -452,32 +509,17 @@ repositories: }), }), }, - { - name: "VEX Repository without config", - args: args{ - report: imageReport([]types.Result{ - bashResult(types.Result{ - Vulnerabilities: []types.DetectedVulnerability{ - vuln3, // not filtered by VEX - }, - }), - }), - opts: vex.Options{ - CacheDir: "testdata/no-repo", - }, - }, - want: imageReport([]types.Result{ - bashResult(types.Result{ - Vulnerabilities: []types.DetectedVulnerability{vuln3}, - }), - }), - }, { name: "unknown format", args: args{ report: &types.Report{}, opts: vex.Options{ - VEXPath: "testdata/unknown.json", + Sources: []vex.Source{ + { + Type: vex.TypeFile, + FilePath: "testdata/unknown.json", + }, + }, }, }, wantErr: "unable to load VEX", From ba16b8871131b450fdbedcf108c9b2812f3f1a6e Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 22 Jul 2024 17:24:14 +0400 Subject: [PATCH 38/55] test(vex): show a file path Signed-off-by: knqyf263 --- pkg/result/filter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/result/filter_test.go b/pkg/result/filter_test.go index 9ef7e50e0f58..0298cd0d9582 100644 --- a/pkg/result/filter_test.go +++ b/pkg/result/filter_test.go @@ -292,7 +292,7 @@ func TestFilter(t *testing.T) { Type: types.FindingTypeVulnerability, Status: types.FindingStatusNotAffected, Statement: "vulnerable_code_not_in_execute_path", - Source: "OpenVEX", + Source: "testdata/openvex.json", Finding: vuln1, }, }, From 6dff269e79ccf6f5cd7efcc162a3a4ae4d4ef8cf Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 22 Jul 2024 17:31:43 +0400 Subject: [PATCH 39/55] test(vex): add enabled field Signed-off-by: knqyf263 --- pkg/vex/document.go | 2 +- pkg/vex/repo.go | 4 +++- pkg/vex/repo_test.go | 7 ++++++- .../vex/repositories/default/{v0 => 0.1}/bash-vex.json | 0 .../vex/repositories/default/{v0 => 0.1}/index.json | 0 .../repositories/high-priority/{v0 => 0.1}/bash-vex.json | 0 .../repositories/high-priority/{v0 => 0.1}/index.json | 0 pkg/vex/vex.go | 9 +++++---- 8 files changed, 15 insertions(+), 7 deletions(-) rename pkg/vex/testdata/multi-repos/vex/repositories/default/{v0 => 0.1}/bash-vex.json (100%) rename pkg/vex/testdata/multi-repos/vex/repositories/default/{v0 => 0.1}/index.json (100%) rename pkg/vex/testdata/multi-repos/vex/repositories/high-priority/{v0 => 0.1}/bash-vex.json (100%) rename pkg/vex/testdata/multi-repos/vex/repositories/high-priority/{v0 => 0.1}/index.json (100%) diff --git a/pkg/vex/document.go b/pkg/vex/document.go index 450c1676f7ab..7331bc26b93b 100644 --- a/pkg/vex/document.go +++ b/pkg/vex/document.go @@ -19,7 +19,7 @@ import ( func NewDocument(filePath string, report *types.Report) (VEX, error) { if filePath == "" { - return nil, nil + return nil, xerrors.New("VEX file path is empty") } f, err := os.Open(filePath) if err != nil { diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go index 6b1813581b94..3d0382027643 100644 --- a/pkg/vex/repo.go +++ b/pkg/vex/repo.go @@ -16,6 +16,8 @@ import ( "github.com/aquasecurity/trivy/pkg/vex/repo" ) +var errNoRepository = errors.New("no available VEX repository found") + // RepositoryIndex wraps the repository index type RepositoryIndex struct { Name string @@ -52,7 +54,7 @@ func NewRepositorySet(ctx context.Context, cacheDir string) (*RepositorySet, err } if len(indexes) == 0 { logger.Warn("No available VEX repository found locally") - return nil, nil + return nil, errNoRepository } return &RepositorySet{ diff --git a/pkg/vex/repo_test.go b/pkg/vex/repo_test.go index 54682e80cccc..24d8b1334177 100644 --- a/pkg/vex/repo_test.go +++ b/pkg/vex/repo_test.go @@ -37,6 +37,7 @@ func TestRepositorySet_NotAffected(t *testing.T) { repositories: - name: default url: https://example.com/vex/default + enabled: true `, vuln: vuln3, product: bashComponent, @@ -56,8 +57,11 @@ repositories: repositories: - name: high-priority url: https://example.com/vex/high-priority + enabled: true - name: default - url: https://example.com/vex/default`, + url: https://example.com/vex/default + enabled: true +`, vuln: vuln3, product: bashComponent, wantNotAffected: false, @@ -69,6 +73,7 @@ repositories: repositories: - name: default url: https://example.com/vex/default + enabled: true `, vuln: vuln4, product: bashComponent, diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/bash-vex.json b/pkg/vex/testdata/multi-repos/vex/repositories/default/0.1/bash-vex.json similarity index 100% rename from pkg/vex/testdata/multi-repos/vex/repositories/default/v0/bash-vex.json rename to pkg/vex/testdata/multi-repos/vex/repositories/default/0.1/bash-vex.json diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json b/pkg/vex/testdata/multi-repos/vex/repositories/default/0.1/index.json similarity index 100% rename from pkg/vex/testdata/multi-repos/vex/repositories/default/v0/index.json rename to pkg/vex/testdata/multi-repos/vex/repositories/default/0.1/index.json diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/bash-vex.json b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/0.1/bash-vex.json similarity index 100% rename from pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/bash-vex.json rename to pkg/vex/testdata/multi-repos/vex/repositories/high-priority/0.1/bash-vex.json diff --git a/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json b/pkg/vex/testdata/multi-repos/vex/repositories/high-priority/0.1/index.json similarity index 100% rename from pkg/vex/testdata/multi-repos/vex/repositories/high-priority/v0/index.json rename to pkg/vex/testdata/multi-repos/vex/repositories/high-priority/0.1/index.json diff --git a/pkg/vex/vex.go b/pkg/vex/vex.go index 3e68a996b853..10a44c42e116 100644 --- a/pkg/vex/vex.go +++ b/pkg/vex/vex.go @@ -2,6 +2,7 @@ package vex import ( "context" + "errors" "github.com/samber/lo" "golang.org/x/xerrors" @@ -94,16 +95,16 @@ func New(ctx context.Context, report *types.Report, opts Options) (*Client, erro } case TypeRepository: v, err = NewRepositorySet(ctx, opts.CacheDir) - if err != nil { + if errors.Is(err, errNoRepository) { + continue + } else if err != nil { return nil, xerrors.Errorf("failed to create a vex repository set: %w", err) } default: log.Warn("Unsupported VEX source", log.String("type", string(src.Type))) continue } - if !lo.IsNil(v) { - vexes = append(vexes, v) - } + vexes = append(vexes, v) } if len(vexes) == 0 { From 8ee23538ecc1b4b18c588be02b0b78102971de9c Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 22 Jul 2024 20:19:21 +0400 Subject: [PATCH 40/55] feat(download): support client mode Signed-off-by: knqyf263 --- pkg/downloader/download.go | 15 +++++++-------- pkg/vex/repo/repo.go | 4 +++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index 87bf65272eb8..db48745d5c72 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -13,17 +13,17 @@ import ( "github.com/google/go-github/v62/github" getter "github.com/hashicorp/go-getter" + "github.com/samber/lo" "golang.org/x/xerrors" - - "github.com/aquasecurity/trivy/pkg/utils/fsutils" ) var ErrSkipDownload = errors.New("skip download") type Options struct { - Insecure bool - Auth Auth - ETag string + Insecure bool + Auth Auth + ETag string + ClientMode getter.ClientMode } type Auth struct { @@ -54,9 +54,7 @@ func DownloadToTempDir(ctx context.Context, url string, opts Options) (string, e // Download downloads the configured source to the destination. func Download(ctx context.Context, src, dst, pwd string, opts Options) (string, error) { // go-getter doesn't allow the dst directory already exists if the src is directory. - if fsutils.DirExists(src) { - _ = os.RemoveAll(dst) - } + _ = os.RemoveAll(dst) var clientOpts []getter.ClientOption if opts.Insecure { @@ -93,6 +91,7 @@ func Download(ctx context.Context, src, dst, pwd string, opts Options) (string, Pwd: pwd, Getters: getters, Mode: getter.ClientModeAny, + Mode: lo.Ternary(opts.ClientMode == 0, getter.ClientModeAny, opts.ClientMode), Options: clientOpts, } diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index be0971ece228..1049cfea9685 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -10,6 +10,7 @@ import ( "path/filepath" "time" + "github.com/hashicorp/go-getter" "github.com/samber/lo" "golang.org/x/xerrors" @@ -162,13 +163,14 @@ func (r *Repository) downloadManifest(ctx context.Context, opts Options) error { } log.DebugContext(ctx, "Downloading the repository metadata...", log.String("url", u.String()), log.String("dst", r.dir)) - _, err = downloader.Download(ctx, u.String(), r.dir, ".", downloader.Options{ + _, err = downloader.Download(ctx, u.String(), filepath.Join(r.dir, manifestFile), ".", downloader.Options{ Insecure: opts.Insecure, Auth: downloader.Auth{ Username: r.Username, Password: r.Password, Token: r.Token, }, + ClientMode: getter.ClientModeFile, }) if err != nil { _ = os.RemoveAll(r.dir) From 7e24dcab487f749c83024feb8fb1d60b3919a389 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 22 Jul 2024 20:19:38 +0400 Subject: [PATCH 41/55] fix: support insecure in custom transport Signed-off-by: knqyf263 --- pkg/downloader/download.go | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index db48745d5c72..35d369e033f3 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -3,6 +3,7 @@ package downloader import ( "cmp" "context" + "crypto/tls" "errors" "maps" "net/http" @@ -72,7 +73,7 @@ func Download(ctx context.Context, src, dst, pwd string, opts Options) (string, // it cannot enable WithInsecure() afterwards because its state is preserved. // Therefore, we need to create a new "HttpGetter" instance every time. // cf. https://github.com/hashicorp/go-getter/blob/5a63fd9c0d5b8da8a6805e8c283f46f0dacb30b3/get.go#L63-L65 - transport := NewCustomTransport(opts.Auth, opts.ETag) + transport := NewCustomTransport(opts) httpGetter := &getter.HttpGetter{ Netrc: true, Client: &http.Client{ @@ -90,7 +91,6 @@ func Download(ctx context.Context, src, dst, pwd string, opts Options) (string, Dst: dst, Pwd: pwd, Getters: getters, - Mode: getter.ClientModeAny, Mode: lo.Ternary(opts.ClientMode == 0, getter.ClientModeAny, opts.ClientMode), Options: clientOpts, } @@ -106,12 +106,14 @@ type CustomTransport struct { auth Auth cachedETag string newETag string + insecure bool } -func NewCustomTransport(auth Auth, etag string) *CustomTransport { +func NewCustomTransport(opts Options) *CustomTransport { return &CustomTransport{ - auth: auth, - cachedETag: etag, + auth: opts.Auth, + cachedETag: opts.ETag, + insecure: opts.Insecure, } } @@ -127,10 +129,10 @@ func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) { var transport http.RoundTripper if req.URL.Host == "github.com" { - transport = NewGitHubTransport(req.URL, t.auth.Token) + transport = NewGitHubTransport(req.URL, t.insecure, t.auth.Token) } if transport == nil { - transport = http.DefaultTransport + transport = httpTransport(t.insecure) } res, err := transport.RoundTrip(req) @@ -149,8 +151,8 @@ func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) { return res, nil } -func NewGitHubTransport(u *url.URL, token string) http.RoundTripper { - client := newGitHubClient(token) +func NewGitHubTransport(u *url.URL, insecure bool, token string) http.RoundTripper { + client := newGitHubClient(insecure, token) ss := strings.SplitN(u.Path, "/", 4) if len(ss) < 4 || strings.HasPrefix(ss[3], "archive/") { // Use the default transport from go-github for authentication @@ -182,11 +184,17 @@ func (t *GitHubContentTransport) RoundTrip(req *http.Request) (*http.Response, e return res.Response, nil } -func newGitHubClient(token string) *github.Client { - client := github.NewClient(nil) +func newGitHubClient(insecure bool, token string) *github.Client { + client := github.NewClient(&http.Client{Transport: httpTransport(insecure)}) token = cmp.Or(token, os.Getenv("GITHUB_TOKEN")) if token != "" { client = client.WithAuthToken(token) } return client } + +func httpTransport(insecure bool) *http.Transport { + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: insecure} + return tr +} From aec8684fa94e1b890a237c03f692f7e80890df19 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 22 Jul 2024 21:10:56 +0400 Subject: [PATCH 42/55] fix: lint issues Signed-off-by: knqyf263 --- internal/testutil/fs.go | 20 +++++++++----------- pkg/downloader/download.go | 4 ++-- pkg/vex/repo/manager.go | 1 - pkg/vex/repo/repo.go | 2 +- pkg/vex/repo/repo_test.go | 3 +-- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/internal/testutil/fs.go b/internal/testutil/fs.go index 19f140c9c91a..842cf7042c55 100644 --- a/internal/testutil/fs.go +++ b/internal/testutil/fs.go @@ -13,10 +13,9 @@ import ( ) func CopyFile(t *testing.T, src, dst string) { - err := os.MkdirAll(filepath.Dir(dst), 0755) - require.NoError(t, err) + MustMkdirAll(t, filepath.Dir(dst)) - _, err = fsutils.CopyFile(src, dst) + _, err := fsutils.CopyFile(src, dst) require.NoError(t, err) } @@ -45,10 +44,9 @@ func CopyDir(t *testing.T, src, dst string) { } } -func MustWriteYAML(t *testing.T, path string, data interface{}) { +func MustWriteYAML(t *testing.T, path string, data any) { t.Helper() - dir := filepath.Dir(path) - require.NoError(t, os.MkdirAll(dir, 0755)) + MustMkdirAll(t, filepath.Dir(path)) f, err := os.Create(path) require.NoError(t, err) @@ -57,7 +55,7 @@ func MustWriteYAML(t *testing.T, path string, data interface{}) { require.NoError(t, yaml.NewEncoder(f).Encode(data)) } -func MustReadYAML(t *testing.T, path string, out interface{}) { +func MustReadYAML(t *testing.T, path string, out any) { t.Helper() f, err := os.Open(path) require.NoError(t, err) @@ -67,18 +65,18 @@ func MustReadYAML(t *testing.T, path string, out interface{}) { } func MustMkdirAll(t *testing.T, dir string) { - err := os.MkdirAll(dir, 0755) + err := os.MkdirAll(dir, 0750) require.NoError(t, err) } -func MustReadJSON(t *testing.T, filePath string, v interface{}) { +func MustReadJSON(t *testing.T, filePath string, v any) { b, err := os.ReadFile(filePath) require.NoError(t, err) err = json.Unmarshal(b, v) require.NoError(t, err) } -func MustWriteJSON(t *testing.T, filePath string, v interface{}) { +func MustWriteJSON(t *testing.T, filePath string, v any) { data, err := json.Marshal(v) require.NoError(t, err) @@ -89,6 +87,6 @@ func MustWriteFile(t *testing.T, filePath string, content []byte) { dir := filepath.Dir(filePath) MustMkdirAll(t, dir) - err := os.WriteFile(filePath, content, 0744) + err := os.WriteFile(filePath, content, 0600) require.NoError(t, err) } diff --git a/pkg/downloader/download.go b/pkg/downloader/download.go index 35d369e033f3..63b130a667fd 100644 --- a/pkg/downloader/download.go +++ b/pkg/downloader/download.go @@ -34,7 +34,7 @@ type Auth struct { } // DownloadToTempDir downloads the configured source to a temp dir. -func DownloadToTempDir(ctx context.Context, url string, opts Options) (string, error) { +func DownloadToTempDir(ctx context.Context, src string, opts Options) (string, error) { tempDir, err := os.MkdirTemp("", "trivy-download") if err != nil { return "", xerrors.Errorf("failed to create a temp dir: %w", err) @@ -45,7 +45,7 @@ func DownloadToTempDir(ctx context.Context, url string, opts Options) (string, e return "", xerrors.Errorf("unable to get the current dir: %w", err) } - if _, err = Download(ctx, url, tempDir, pwd, opts); err != nil { + if _, err = Download(ctx, src, tempDir, pwd, opts); err != nil { return "", xerrors.Errorf("download error: %w", err) } diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go index 56c1ae7a0cb9..1a861c0d62b6 100644 --- a/pkg/vex/repo/manager.go +++ b/pkg/vex/repo/manager.go @@ -48,7 +48,6 @@ type Options struct { // Manager manages the plugins type Manager struct { w io.Writer - indexURL string configFile string cacheDir string } diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index 1049cfea9685..76c9ee6168a7 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -235,7 +235,7 @@ func (r *Repository) download(ctx context.Context, ver Version, dst string, opts if err != nil { return xerrors.Errorf("failed to get the repository cache metadata: %w", err) } - etags := lo.Ternary(m.ETags == nil, map[string]string{}, m.ETags) + etags := lo.Ternary(m.ETags == nil, make(map[string]string), m.ETags) var errs error for _, loc := range ver.Locations { diff --git a/pkg/vex/repo/repo_test.go b/pkg/vex/repo/repo_test.go index 086b712229ba..67c196da2288 100644 --- a/pkg/vex/repo/repo_test.go +++ b/pkg/vex/repo/repo_test.go @@ -39,8 +39,7 @@ var manifest = repo.Manifest{ func TestRepository_Manifest(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Println(r.URL.Path) - switch r.URL.Path { - case "/.well-known/vex-repository.json": + if r.URL.Path == "/.well-known/vex-repository.json" { err := json.NewEncoder(w).Encode(manifest) assert.NoError(t, err) } From 32bcd210daea324158592819dd84edee40cf2d50 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Tue, 23 Jul 2024 10:03:10 +0400 Subject: [PATCH 43/55] ci: switch to ubuntu-latest ubuntu-latest-m is broken now. TODO: switch it back to ubuntu-latest-m later Signed-off-by: knqyf263 --- .github/workflows/test.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2ff471be1c7d..13f279b519b7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ jobs: runs-on: ${{ matrix.operating-system }} strategy: matrix: - operating-system: [ubuntu-latest-m, windows-latest, macos-latest] + operating-system: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v4.1.6 @@ -31,7 +31,7 @@ jobs: echo "Run 'go mod tidy' and push it" exit 1 fi - if: matrix.operating-system == 'ubuntu-latest-m' + if: matrix.operating-system == 'ubuntu-latest' - name: Lint id: lint @@ -39,7 +39,7 @@ jobs: with: version: v1.59 args: --verbose --out-format=line-number - if: matrix.operating-system == 'ubuntu-latest-m' + if: matrix.operating-system == 'ubuntu-latest' - name: Check if linter failed run: | @@ -60,14 +60,14 @@ jobs: echo "Run 'mage docs:generate' and push it" exit 1 fi - if: matrix.operating-system == 'ubuntu-latest-m' + if: matrix.operating-system == 'ubuntu-latest' - name: Run unit tests run: mage test:unit integration: name: Integration Test - runs-on: ubuntu-latest-m + runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v4.1.6 @@ -87,7 +87,7 @@ jobs: k8s-integration: name: K8s Integration Test - runs-on: ubuntu-latest-m + runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v4.1.6 @@ -129,7 +129,7 @@ jobs: vm-test: name: VM Integration Test - runs-on: ubuntu-latest-m + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4.1.6 @@ -151,7 +151,7 @@ jobs: runs-on: ${{ matrix.operating-system }} strategy: matrix: - operating-system: [ubuntu-latest-m, windows-latest, macos-latest] + operating-system: [ubuntu-latest, windows-latest, macos-latest] env: DOCKER_CLI_EXPERIMENTAL: "enabled" steps: From d5cc59db5e2a1d94c02a319e29b264164aeeaa41 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Tue, 23 Jul 2024 10:23:57 +0400 Subject: [PATCH 44/55] docs: auto-generate Signed-off-by: knqyf263 --- .../references/configuration/cli/trivy.md | 1 + .../configuration/cli/trivy_clean.md | 1 + .../configuration/cli/trivy_filesystem.md | 3 +- .../configuration/cli/trivy_image.md | 3 +- .../configuration/cli/trivy_kubernetes.md | 3 +- .../configuration/cli/trivy_repository.md | 3 +- .../configuration/cli/trivy_rootfs.md | 3 +- .../configuration/cli/trivy_sbom.md | 3 +- .../references/configuration/cli/trivy_vex.md | 28 ++++++++++++++ .../configuration/cli/trivy_vex_repo.md | 38 +++++++++++++++++++ .../cli/trivy_vex_repo_download.md | 31 +++++++++++++++ .../configuration/cli/trivy_vex_repo_init.md | 31 +++++++++++++++ .../configuration/cli/trivy_vex_repo_list.md | 31 +++++++++++++++ .../references/configuration/cli/trivy_vm.md | 3 +- 14 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 docs/docs/references/configuration/cli/trivy_vex.md create mode 100644 docs/docs/references/configuration/cli/trivy_vex_repo.md create mode 100644 docs/docs/references/configuration/cli/trivy_vex_repo_download.md create mode 100644 docs/docs/references/configuration/cli/trivy_vex_repo_init.md create mode 100644 docs/docs/references/configuration/cli/trivy_vex_repo_list.md diff --git a/docs/docs/references/configuration/cli/trivy.md b/docs/docs/references/configuration/cli/trivy.md index 2992bc0faa9b..3d8ece9cd0e8 100644 --- a/docs/docs/references/configuration/cli/trivy.md +++ b/docs/docs/references/configuration/cli/trivy.md @@ -56,5 +56,6 @@ trivy [global flags] command [flags] target * [trivy sbom](trivy_sbom.md) - Scan SBOM for vulnerabilities and licenses * [trivy server](trivy_server.md) - Server mode * [trivy version](trivy_version.md) - Print the version +* [trivy vex](trivy_vex.md) - [EXPERIMENTAL] VEX utilities * [trivy vm](trivy_vm.md) - [EXPERIMENTAL] Scan a virtual machine image diff --git a/docs/docs/references/configuration/cli/trivy_clean.md b/docs/docs/references/configuration/cli/trivy_clean.md index 7a997bf7b581..65b827136f5b 100644 --- a/docs/docs/references/configuration/cli/trivy_clean.md +++ b/docs/docs/references/configuration/cli/trivy_clean.md @@ -28,6 +28,7 @@ trivy clean [flags] -h, --help help for clean --java-db remove Java database --scan-cache remove scan cache (container and VM image analysis results) + --vex-repo remove VEX repositories --vuln-db remove vulnerability database ``` diff --git a/docs/docs/references/configuration/cli/trivy_filesystem.md b/docs/docs/references/configuration/cli/trivy_filesystem.md index 2fb1ad1c984c..86ac2c2f8918 100644 --- a/docs/docs/references/configuration/cli/trivy_filesystem.md +++ b/docs/docs/references/configuration/cli/trivy_filesystem.md @@ -82,6 +82,7 @@ trivy filesystem [flags] PATH --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database + --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files @@ -89,7 +90,7 @@ trivy filesystem [flags] PATH --token-header string specify a header name for token in client/server mode (default "Trivy-Token") --trace enable more verbose trace output for custom queries --username strings username. Comma-separated usernames allowed. - --vex string [EXPERIMENTAL] file path to VEX + --vex strings [EXPERIMENTAL] VEX sources ("repo" or file path) ``` ### Options inherited from parent commands diff --git a/docs/docs/references/configuration/cli/trivy_image.md b/docs/docs/references/configuration/cli/trivy_image.md index 2ae526d9405c..62e731e14126 100644 --- a/docs/docs/references/configuration/cli/trivy_image.md +++ b/docs/docs/references/configuration/cli/trivy_image.md @@ -103,13 +103,14 @@ trivy image [flags] IMAGE_NAME --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database + --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") --trace enable more verbose trace output for custom queries --username strings username. Comma-separated usernames allowed. - --vex string [EXPERIMENTAL] file path to VEX + --vex strings [EXPERIMENTAL] VEX sources ("repo" or file path) ``` ### Options inherited from parent commands diff --git a/docs/docs/references/configuration/cli/trivy_kubernetes.md b/docs/docs/references/configuration/cli/trivy_kubernetes.md index 3f20a33e866d..2f8539dfeb47 100644 --- a/docs/docs/references/configuration/cli/trivy_kubernetes.md +++ b/docs/docs/references/configuration/cli/trivy_kubernetes.md @@ -98,12 +98,13 @@ trivy kubernetes [flags] [CONTEXT] --skip-files strings specify the files or glob patterns to skip --skip-images skip the downloading and scanning of images (vulnerabilities and secrets) in the cluster resources --skip-java-db-update skip updating Java index database + --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tolerations strings specify node-collector job tolerations (example: key1=value1:NoExecute,key2=value2:NoSchedule) --trace enable more verbose trace output for custom queries --username strings username. Comma-separated usernames allowed. - --vex string [EXPERIMENTAL] file path to VEX + --vex strings [EXPERIMENTAL] VEX sources ("repo" or file path) ``` ### Options inherited from parent commands diff --git a/docs/docs/references/configuration/cli/trivy_repository.md b/docs/docs/references/configuration/cli/trivy_repository.md index 4831c2bad4ae..661381b76ebf 100644 --- a/docs/docs/references/configuration/cli/trivy_repository.md +++ b/docs/docs/references/configuration/cli/trivy_repository.md @@ -81,6 +81,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database + --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update --tag string pass the tag name to be scanned -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules @@ -89,7 +90,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --token-header string specify a header name for token in client/server mode (default "Trivy-Token") --trace enable more verbose trace output for custom queries --username strings username. Comma-separated usernames allowed. - --vex string [EXPERIMENTAL] file path to VEX + --vex strings [EXPERIMENTAL] VEX sources ("repo" or file path) ``` ### Options inherited from parent commands diff --git a/docs/docs/references/configuration/cli/trivy_rootfs.md b/docs/docs/references/configuration/cli/trivy_rootfs.md index ca433b327f0d..01ed2ce062c6 100644 --- a/docs/docs/references/configuration/cli/trivy_rootfs.md +++ b/docs/docs/references/configuration/cli/trivy_rootfs.md @@ -83,6 +83,7 @@ trivy rootfs [flags] ROOTDIR --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database + --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files @@ -90,7 +91,7 @@ trivy rootfs [flags] ROOTDIR --token-header string specify a header name for token in client/server mode (default "Trivy-Token") --trace enable more verbose trace output for custom queries --username strings username. Comma-separated usernames allowed. - --vex string [EXPERIMENTAL] file path to VEX + --vex strings [EXPERIMENTAL] VEX sources ("repo" or file path) ``` ### Options inherited from parent commands diff --git a/docs/docs/references/configuration/cli/trivy_sbom.md b/docs/docs/references/configuration/cli/trivy_sbom.md index 3d4d25e47fcb..0c7508c854e1 100644 --- a/docs/docs/references/configuration/cli/trivy_sbom.md +++ b/docs/docs/references/configuration/cli/trivy_sbom.md @@ -58,10 +58,11 @@ trivy sbom [flags] SBOM_PATH --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database + --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update -t, --template string output template --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") - --vex string [EXPERIMENTAL] file path to VEX + --vex strings [EXPERIMENTAL] VEX sources ("repo" or file path) ``` ### Options inherited from parent commands diff --git a/docs/docs/references/configuration/cli/trivy_vex.md b/docs/docs/references/configuration/cli/trivy_vex.md new file mode 100644 index 000000000000..e7b4e31cb994 --- /dev/null +++ b/docs/docs/references/configuration/cli/trivy_vex.md @@ -0,0 +1,28 @@ +## trivy vex + +[EXPERIMENTAL] VEX utilities + +### Options + +``` + -h, --help help for vex +``` + +### Options inherited from parent commands + +``` + --cache-dir string cache directory (default "/path/to/cache") + -c, --config string config path (default "trivy.yaml") + -d, --debug debug mode + --generate-default-config write the default config to trivy-default.yaml + --insecure allow insecure server connections + -q, --quiet suppress progress bar and log output + --timeout duration timeout (default 5m0s) + -v, --version show version +``` + +### SEE ALSO + +* [trivy](trivy.md) - Unified security scanner +* [trivy vex repo](trivy_vex_repo.md) - Manage VEX repositories + diff --git a/docs/docs/references/configuration/cli/trivy_vex_repo.md b/docs/docs/references/configuration/cli/trivy_vex_repo.md new file mode 100644 index 000000000000..d70f226f9275 --- /dev/null +++ b/docs/docs/references/configuration/cli/trivy_vex_repo.md @@ -0,0 +1,38 @@ +## trivy vex repo + +Manage VEX repositories + +### Examples + +``` + # Initialize the configuration file + $ trivy vex repo init + +``` + +### Options + +``` + -h, --help help for repo +``` + +### Options inherited from parent commands + +``` + --cache-dir string cache directory (default "/path/to/cache") + -c, --config string config path (default "trivy.yaml") + -d, --debug debug mode + --generate-default-config write the default config to trivy-default.yaml + --insecure allow insecure server connections + -q, --quiet suppress progress bar and log output + --timeout duration timeout (default 5m0s) + -v, --version show version +``` + +### SEE ALSO + +* [trivy vex](trivy_vex.md) - [EXPERIMENTAL] VEX utilities +* [trivy vex repo download](trivy_vex_repo_download.md) - Download the VEX repositories +* [trivy vex repo init](trivy_vex_repo_init.md) - Initialize a configuration file +* [trivy vex repo list](trivy_vex_repo_list.md) - List VEX repositories + diff --git a/docs/docs/references/configuration/cli/trivy_vex_repo_download.md b/docs/docs/references/configuration/cli/trivy_vex_repo_download.md new file mode 100644 index 000000000000..5df0c48dfe34 --- /dev/null +++ b/docs/docs/references/configuration/cli/trivy_vex_repo_download.md @@ -0,0 +1,31 @@ +## trivy vex repo download + +Download the VEX repositories + +``` +trivy vex repo download [REPO_NAMES] +``` + +### Options + +``` + -h, --help help for download +``` + +### Options inherited from parent commands + +``` + --cache-dir string cache directory (default "/path/to/cache") + -c, --config string config path (default "trivy.yaml") + -d, --debug debug mode + --generate-default-config write the default config to trivy-default.yaml + --insecure allow insecure server connections + -q, --quiet suppress progress bar and log output + --timeout duration timeout (default 5m0s) + -v, --version show version +``` + +### SEE ALSO + +* [trivy vex repo](trivy_vex_repo.md) - Manage VEX repositories + diff --git a/docs/docs/references/configuration/cli/trivy_vex_repo_init.md b/docs/docs/references/configuration/cli/trivy_vex_repo_init.md new file mode 100644 index 000000000000..226c16cf0897 --- /dev/null +++ b/docs/docs/references/configuration/cli/trivy_vex_repo_init.md @@ -0,0 +1,31 @@ +## trivy vex repo init + +Initialize a configuration file + +``` +trivy vex repo init +``` + +### Options + +``` + -h, --help help for init +``` + +### Options inherited from parent commands + +``` + --cache-dir string cache directory (default "/path/to/cache") + -c, --config string config path (default "trivy.yaml") + -d, --debug debug mode + --generate-default-config write the default config to trivy-default.yaml + --insecure allow insecure server connections + -q, --quiet suppress progress bar and log output + --timeout duration timeout (default 5m0s) + -v, --version show version +``` + +### SEE ALSO + +* [trivy vex repo](trivy_vex_repo.md) - Manage VEX repositories + diff --git a/docs/docs/references/configuration/cli/trivy_vex_repo_list.md b/docs/docs/references/configuration/cli/trivy_vex_repo_list.md new file mode 100644 index 000000000000..8698b67d10e6 --- /dev/null +++ b/docs/docs/references/configuration/cli/trivy_vex_repo_list.md @@ -0,0 +1,31 @@ +## trivy vex repo list + +List VEX repositories + +``` +trivy vex repo list +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --cache-dir string cache directory (default "/path/to/cache") + -c, --config string config path (default "trivy.yaml") + -d, --debug debug mode + --generate-default-config write the default config to trivy-default.yaml + --insecure allow insecure server connections + -q, --quiet suppress progress bar and log output + --timeout duration timeout (default 5m0s) + -v, --version show version +``` + +### SEE ALSO + +* [trivy vex repo](trivy_vex_repo.md) - Manage VEX repositories + diff --git a/docs/docs/references/configuration/cli/trivy_vm.md b/docs/docs/references/configuration/cli/trivy_vm.md index dab35eeb93e1..c250b9bcf06b 100644 --- a/docs/docs/references/configuration/cli/trivy_vm.md +++ b/docs/docs/references/configuration/cli/trivy_vm.md @@ -72,11 +72,12 @@ trivy vm [flags] VM_IMAGE --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database + --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") - --vex string [EXPERIMENTAL] file path to VEX + --vex strings [EXPERIMENTAL] VEX sources ("repo" or file path) ``` ### Options inherited from parent commands From 422e03cb578a2d1df4bbb698417c9ec91b0856b7 Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Tue, 23 Jul 2024 15:26:51 +0400 Subject: [PATCH 45/55] Update manager.go Co-authored-by: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> --- pkg/vex/repo/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go index 1a861c0d62b6..65c9833acaa6 100644 --- a/pkg/vex/repo/manager.go +++ b/pkg/vex/repo/manager.go @@ -45,7 +45,7 @@ type Options struct { Insecure bool } -// Manager manages the plugins +// Manager manages the repositories type Manager struct { w io.Writer configFile string From fca16cb6ddb64f28c37793176f59d6017bfa8fb8 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 25 Jul 2024 10:54:56 +0400 Subject: [PATCH 46/55] fix: cli usage Signed-off-by: knqyf263 --- pkg/commands/app.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pkg/commands/app.go b/pkg/commands/app.go index 554d6330529f..ed1bc83ff4d2 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -1235,7 +1235,6 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { var vexOptions flag.Options cmd := &cobra.Command{ Use: "vex subcommand", - Aliases: []string{"p"}, GroupID: groupManagement, Short: "[EXPERIMENTAL] VEX utilities", SilenceErrors: true, @@ -1258,17 +1257,22 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { SilenceUsage: true, Example: ` # Initialize the configuration file $ trivy vex repo init + + # List VEX repositories + $ trivy vex repo list + + # Download the VEX repositories + $ trivy vex repo download `, } repoCmd.AddCommand( &cobra.Command{ - Use: "init", - Short: "Initialize a configuration file", - SilenceErrors: true, - SilenceUsage: true, - DisableFlagsInUseLine: true, - Args: cobra.ExactArgs(0), + Use: "init", + Short: "Initialize a configuration file", + SilenceErrors: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { if err := vexrepo.NewManager(vexOptions.CacheDir).Init(cmd.Context()); err != nil { return xerrors.Errorf("config init error: %w", err) @@ -1277,12 +1281,11 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { }, }, &cobra.Command{ - Use: "list", - Short: "List VEX repositories", - SilenceErrors: true, - SilenceUsage: true, - DisableFlagsInUseLine: true, - Args: cobra.ExactArgs(0), + Use: "list", + Short: "List VEX repositories", + SilenceErrors: true, + SilenceUsage: true, + Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { if err := vexrepo.NewManager(vexOptions.CacheDir).List(cmd.Context()); err != nil { return xerrors.Errorf("list error: %w", err) From 71c9bf334c50f7b2d930ef3a277d102998095fc3 Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Thu, 25 Jul 2024 10:57:13 +0400 Subject: [PATCH 47/55] Update pkg/commands/operation/operation.go [skip ci] Co-authored-by: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> --- pkg/commands/operation/operation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/commands/operation/operation.go b/pkg/commands/operation/operation.go index 53820e0aa5fd..16aa72085949 100644 --- a/pkg/commands/operation/operation.go +++ b/pkg/commands/operation/operation.go @@ -71,7 +71,7 @@ func DownloadVEXRepositories(ctx context.Context, opts flag.Options) error { Insecure: opts.Insecure, }) if err != nil { - return xerrors.Errorf("failed to get vex repository config: %w", err) + return xerrors.Errorf("failed to download vex repositories: %w", err) } return nil From 642f2d9c751070a089ed8227cf4254bd7cfab655 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 25 Jul 2024 11:00:05 +0400 Subject: [PATCH 48/55] docs(cache): add a link to vex repo Signed-off-by: knqyf263 --- docs/docs/configuration/cache.md | 3 ++- docs/docs/supply-chain/vex/repo.md | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/docs/configuration/cache.md b/docs/docs/configuration/cache.md index 8817a2adb3ea..2ad5086d0e87 100644 --- a/docs/docs/configuration/cache.md +++ b/docs/docs/configuration/cache.md @@ -1,10 +1,11 @@ # Cache The cache directory includes +- Cache of previous scans (Scan cache). - [Vulnerability Database][trivy-db][^1] - [Java Index Database][trivy-java-db][^2] - [Misconfiguration Checks][misconf-checks][^3] -- Cache of previous scans. +- [VEX Repositories](../supply-chain/vex/repo.md) The cache option is common to all scanners. diff --git a/docs/docs/supply-chain/vex/repo.md b/docs/docs/supply-chain/vex/repo.md index b4fa7a4ab399..b6c641aee9da 100644 --- a/docs/docs/supply-chain/vex/repo.md +++ b/docs/docs/supply-chain/vex/repo.md @@ -1,5 +1,8 @@ # VEX Repository +!!! warning "EXPERIMENTAL" + This feature might change without preserving backwards compatibility. + ## Using VEX Repository Trivy can download and utilize VEX documents from repositories that comply with [the VEX Repository Specification][vex-repo]. From 9881b0fd3b43455f1c2cc22458aa95eb0420817c Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 25 Jul 2024 11:11:57 +0400 Subject: [PATCH 49/55] revert: add parents flag [skip ci] Signed-off-by: knqyf263 --- pkg/sbom/core/bom.go | 5 ++++- pkg/vex/vex.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/sbom/core/bom.go b/pkg/sbom/core/bom.go index cd238b535a11..51875bff8738 100644 --- a/pkg/sbom/core/bom.go +++ b/pkg/sbom/core/bom.go @@ -203,6 +203,7 @@ type Vulnerability struct { type Options struct { GenerateBOMRef bool // Generate BOMRef for CycloneDX + Parents bool // Hold parent maps } func NewBOM(opts Options) *BOM { @@ -262,7 +263,9 @@ func (b *BOM) AddRelationship(parent, child *Component, relationshipType Relatio Dependency: child.id, }) - b.parents[child.id] = append(b.parents[child.id], parent.id) + if b.opts.Parents { + b.parents[child.id] = append(b.parents[child.id], parent.id) + } } func (b *BOM) AddVulnerabilities(c *Component, vulns []Vulnerability) { diff --git a/pkg/vex/vex.go b/pkg/vex/vex.go index 10a44c42e116..de93f542ca00 100644 --- a/pkg/vex/vex.go +++ b/pkg/vex/vex.go @@ -68,7 +68,7 @@ func Filter(ctx context.Context, report *types.Report, opts Options) error { return nil } - bom, err := sbomio.NewEncoder(core.Options{}).Encode(*report) + bom, err := sbomio.NewEncoder(core.Options{Parents: true}).Encode(*report) if err != nil { return xerrors.Errorf("unable to encode the SBOM: %w", err) } From 19e60ba844060396e7f0f2bad942d44899d4c1ab Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 25 Jul 2024 11:37:25 +0400 Subject: [PATCH 50/55] chore: add a comment [skip ci] Signed-off-by: knqyf263 --- pkg/vex/repo.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go index 3d0382027643..d98aceba91b5 100644 --- a/pkg/vex/repo.go +++ b/pkg/vex/repo.go @@ -68,6 +68,9 @@ func (rs *RepositorySet) NotAffected(vuln types.DetectedVulnerability, product, return types.ModifiedFinding{}, false } p := *product.PkgIdentifier.PURL + + // Exclude version, qualifiers, and subpath from the package URL except for OCI + // cf. https://github.com/aquasecurity/vex-repo-spec?tab=readme-ov-file#32-indexjson p.Version = "" p.Qualifiers = nil p.Subpath = "" From e02af76134f093186328e0af42088d8b761e8460 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 25 Jul 2024 11:52:39 +0400 Subject: [PATCH 51/55] chore: add a long description Signed-off-by: knqyf263 --- pkg/commands/app.go | 10 +++++----- pkg/vex/repo/manager.go | 12 +++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/commands/app.go b/pkg/commands/app.go index ed1bc83ff4d2..22af74bdb987 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -1294,11 +1294,11 @@ func NewVEXCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { }, }, &cobra.Command{ - Use: "download [REPO_NAMES]", - Short: "Download the VEX repositories", - DisableFlagsInUseLine: true, - SilenceErrors: true, - SilenceUsage: true, + Use: "download [REPO_NAMES]", + Short: "Download the VEX repositories", + Long: `Downloads enabled VEX repositories. If specific repository names are provided as arguments, only those repositories will be downloaded. Otherwise, all enabled repositories are downloaded.`, + SilenceErrors: true, + SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { err := vexrepo.NewManager(vexOptions.CacheDir).DownloadRepositories(cmd.Context(), args, vexrepo.Options{Insecure: vexOptions.Insecure}) diff --git a/pkg/vex/repo/manager.go b/pkg/vex/repo/manager.go index 65c9833acaa6..b157156bdf74 100644 --- a/pkg/vex/repo/manager.go +++ b/pkg/vex/repo/manager.go @@ -136,15 +136,17 @@ func (m *Manager) DownloadRepositories(ctx context.Context, names []string, opts conf, err := m.Config(ctx) if err != nil { return xerrors.Errorf("unable to read config: %w", err) - } else if len(conf.EnabledRepositories()) == 0 { + } + + repos := lo.Filter(conf.EnabledRepositories(), func(r Repository, _ int) bool { + return len(names) == 0 || slices.Contains(names, r.Name) + }) + if len(repos) == 0 { log.WarnContext(ctx, "No enabled repositories found in config", log.String("path", m.configFile)) return nil } - for _, repo := range conf.EnabledRepositories() { - if len(names) > 0 && !slices.Contains(names, repo.Name) { - continue - } + for _, repo := range repos { if err = repo.Update(ctx, opts); err != nil { return xerrors.Errorf("failed to update the repository: %w", err) } From 424759ed76d159674b319282498f02c59fac3c81 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 25 Jul 2024 11:55:07 +0400 Subject: [PATCH 52/55] fix: delete unused field [skip ci] Signed-off-by: knqyf263 --- pkg/vex/repo/repo.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/vex/repo/repo.go b/pkg/vex/repo/repo.go index 76c9ee6168a7..433b8224c20e 100644 --- a/pkg/vex/repo/repo.go +++ b/pkg/vex/repo/repo.go @@ -29,10 +29,9 @@ const ( ) type Manifest struct { - Name string `json:"name"` - Description string `json:"description"` - Versions []Version `json:"versions"` - LatestVersion string `json:"latest_version"` + Name string `json:"name"` + Description string `json:"description"` + Versions []Version `json:"versions"` } type Version struct { From 07ba807deee3ea5c323ec02b51536406d13e9e84 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 25 Jul 2024 11:56:03 +0400 Subject: [PATCH 53/55] fix: delete debug code Signed-off-by: knqyf263 --- pkg/vex/repo/repo_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/vex/repo/repo_test.go b/pkg/vex/repo/repo_test.go index 67c196da2288..0118b7391523 100644 --- a/pkg/vex/repo/repo_test.go +++ b/pkg/vex/repo/repo_test.go @@ -4,7 +4,6 @@ import ( "archive/zip" "context" "encoding/json" - "fmt" "net/http" "net/http/httptest" "os" @@ -38,7 +37,6 @@ var manifest = repo.Manifest{ func TestRepository_Manifest(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Println(r.URL.Path) if r.URL.Path == "/.well-known/vex-repository.json" { err := json.NewEncoder(w).Encode(manifest) assert.NoError(t, err) From ae853ce4c87e8026860b0b72b25ec20b5ca98f24 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 25 Jul 2024 12:05:38 +0400 Subject: [PATCH 54/55] docs: generate references Signed-off-by: knqyf263 --- docs/docs/references/configuration/cli/trivy_vex_repo.md | 6 ++++++ .../references/configuration/cli/trivy_vex_repo_download.md | 6 +++++- .../references/configuration/cli/trivy_vex_repo_init.md | 2 +- .../references/configuration/cli/trivy_vex_repo_list.md | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/docs/references/configuration/cli/trivy_vex_repo.md b/docs/docs/references/configuration/cli/trivy_vex_repo.md index d70f226f9275..32777ba4bab8 100644 --- a/docs/docs/references/configuration/cli/trivy_vex_repo.md +++ b/docs/docs/references/configuration/cli/trivy_vex_repo.md @@ -8,6 +8,12 @@ Manage VEX repositories # Initialize the configuration file $ trivy vex repo init + # List VEX repositories + $ trivy vex repo list + + # Download the VEX repositories + $ trivy vex repo download + ``` ### Options diff --git a/docs/docs/references/configuration/cli/trivy_vex_repo_download.md b/docs/docs/references/configuration/cli/trivy_vex_repo_download.md index 5df0c48dfe34..eebf63f81187 100644 --- a/docs/docs/references/configuration/cli/trivy_vex_repo_download.md +++ b/docs/docs/references/configuration/cli/trivy_vex_repo_download.md @@ -2,8 +2,12 @@ Download the VEX repositories +### Synopsis + +Downloads enabled VEX repositories. If specific repository names are provided as arguments, only those repositories will be downloaded. Otherwise, all enabled repositories are downloaded. + ``` -trivy vex repo download [REPO_NAMES] +trivy vex repo download [REPO_NAMES] [flags] ``` ### Options diff --git a/docs/docs/references/configuration/cli/trivy_vex_repo_init.md b/docs/docs/references/configuration/cli/trivy_vex_repo_init.md index 226c16cf0897..6e9a9b0f9523 100644 --- a/docs/docs/references/configuration/cli/trivy_vex_repo_init.md +++ b/docs/docs/references/configuration/cli/trivy_vex_repo_init.md @@ -3,7 +3,7 @@ Initialize a configuration file ``` -trivy vex repo init +trivy vex repo init [flags] ``` ### Options diff --git a/docs/docs/references/configuration/cli/trivy_vex_repo_list.md b/docs/docs/references/configuration/cli/trivy_vex_repo_list.md index 8698b67d10e6..5f1c77c23f93 100644 --- a/docs/docs/references/configuration/cli/trivy_vex_repo_list.md +++ b/docs/docs/references/configuration/cli/trivy_vex_repo_list.md @@ -3,7 +3,7 @@ List VEX repositories ``` -trivy vex repo list +trivy vex repo list [flags] ``` ### Options From 1479468b79c3847527537d08f30db2e4c58a45cc Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 25 Jul 2024 15:55:17 +0400 Subject: [PATCH 55/55] feat: show debug message once per package Showing debug messages per vulnerability may be too much. Signed-off-by: knqyf263 --- pkg/vex/repo.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/vex/repo.go b/pkg/vex/repo.go index d98aceba91b5..d7c6717b8e52 100644 --- a/pkg/vex/repo.go +++ b/pkg/vex/repo.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "sync" "github.com/package-url/packageurl-go" "golang.org/x/xerrors" @@ -14,6 +15,7 @@ import ( "github.com/aquasecurity/trivy/pkg/sbom/core" "github.com/aquasecurity/trivy/pkg/types" "github.com/aquasecurity/trivy/pkg/vex/repo" + xsync "github.com/aquasecurity/trivy/pkg/x/sync" ) var errNoRepository = errors.New("no available VEX repository found") @@ -27,6 +29,7 @@ type RepositoryIndex struct { type RepositorySet struct { indexes []RepositoryIndex + logOnce *xsync.Map[string, *sync.Once] logger *log.Logger } @@ -59,6 +62,7 @@ func NewRepositorySet(ctx context.Context, cacheDir string) (*RepositorySet, err return &RepositorySet{ indexes: indexes, // In precedence order + logOnce: new(xsync.Map[string, *sync.Once]), logger: logger, }, nil } @@ -91,6 +95,8 @@ func (rs *RepositorySet) NotAffected(vuln types.DetectedVulnerability, product, if !ok { continue } + rs.logVEXFound(pkgID, index.Name, index.URL, entry.Location) + source := fmt.Sprintf("VEX Repository: %s (%s)", index.Name, index.URL) doc, err := rs.OpenDocument(source, filepath.Dir(index.Path), entry) if err != nil { @@ -102,8 +108,6 @@ func (rs *RepositorySet) NotAffected(vuln types.DetectedVulnerability, product, return m, notAffected } - rs.logger.Debug("VEX found, but affected", log.String("vulnerability", vuln.VulnerabilityID), - log.String("package", pkgID), log.String("repo", index.Name), log.String("repo_url", index.URL)) break // Stop searching for the next VEX document as this repository has higher precedence. } return types.ModifiedFinding{}, false @@ -125,3 +129,15 @@ func (rs *RepositorySet) OpenDocument(source, dir string, entry repo.PackageEntr return nil, xerrors.Errorf("unsupported VEX format: %s", entry.Format) } } + +func (rs *RepositorySet) logVEXFound(pkgID, repoName, repoURL, filePath string) { + once, _ := rs.logOnce.LoadOrStore(pkgID, &sync.Once{}) + once.Do(func() { + rs.logger.Debug("VEX found in the repository", + log.String("package", pkgID), + log.String("repo", repoName), + log.String("repo_url", repoURL), + log.FilePath(filePath), + ) + }) +}