diff --git a/docs/docs/references/configuration/cli/trivy_filesystem.md b/docs/docs/references/configuration/cli/trivy_filesystem.md index e907c5d4f9d5..9944a2d618ff 100644 --- a/docs/docs/references/configuration/cli/trivy_filesystem.md +++ b/docs/docs/references/configuration/cli/trivy_filesystem.md @@ -29,7 +29,7 @@ trivy filesystem [flags] PATH --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -56,7 +56,7 @@ trivy filesystem [flags] PATH --include-deprecated-checks include deprecated checks (default true) --include-dev-deps include development dependencies in the report (supported: npm, yarn) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_image.md b/docs/docs/references/configuration/cli/trivy_image.md index 156582f047ad..b63daffbf4ad 100644 --- a/docs/docs/references/configuration/cli/trivy_image.md +++ b/docs/docs/references/configuration/cli/trivy_image.md @@ -43,7 +43,7 @@ trivy image [flags] IMAGE_NAME --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -74,7 +74,7 @@ trivy image [flags] IMAGE_NAME --include-deprecated-checks include deprecated checks (default true) --include-non-failures include successes and exceptions, available with '--scanners misconfig' --input string input file path instead of image name - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_kubernetes.md b/docs/docs/references/configuration/cli/trivy_kubernetes.md index b7c626a7d476..35190b1b82b5 100644 --- a/docs/docs/references/configuration/cli/trivy_kubernetes.md +++ b/docs/docs/references/configuration/cli/trivy_kubernetes.md @@ -38,7 +38,7 @@ trivy kubernetes [flags] [CONTEXT] --config-check strings specify the paths to the Rego check files or to the directories containing them, applying config files --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -70,7 +70,7 @@ trivy kubernetes [flags] [CONTEXT] --include-kinds strings indicate the kinds included in scanning (example: node) --include-namespaces strings indicate the namespaces included in scanning (example: kube-system) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --k8s-version string specify k8s version to validate outdated api by it (example: 1.21.0) --kubeconfig string specify the kubeconfig file path to use --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_repository.md b/docs/docs/references/configuration/cli/trivy_repository.md index 04f3f26e919a..59e4f2c08cb0 100644 --- a/docs/docs/references/configuration/cli/trivy_repository.md +++ b/docs/docs/references/configuration/cli/trivy_repository.md @@ -29,7 +29,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -56,7 +56,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --include-deprecated-checks include deprecated checks (default true) --include-dev-deps include development dependencies in the report (supported: npm, yarn) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_rootfs.md b/docs/docs/references/configuration/cli/trivy_rootfs.md index 0e0dfa54d448..6022f4589446 100644 --- a/docs/docs/references/configuration/cli/trivy_rootfs.md +++ b/docs/docs/references/configuration/cli/trivy_rootfs.md @@ -31,7 +31,7 @@ trivy rootfs [flags] ROOTDIR --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -58,7 +58,7 @@ trivy rootfs [flags] ROOTDIR --ignorefile string specify .trivyignore file (default ".trivyignore") --include-deprecated-checks include deprecated checks (default true) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_sbom.md b/docs/docs/references/configuration/cli/trivy_sbom.md index 1b0df922c7fc..f09453845518 100644 --- a/docs/docs/references/configuration/cli/trivy_sbom.md +++ b/docs/docs/references/configuration/cli/trivy_sbom.md @@ -20,54 +20,54 @@ trivy sbom [flags] SBOM_PATH ### Options ``` - --cache-backend string [EXPERIMENTAL] cache backend (e.g. redis://localhost:6379) (default "memory") - --cache-ttl duration cache TTL when using redis as cache backend - --compliance string compliance report to generate - --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") - --detection-priority string specify the detection priority: - - "precise": Prioritizes precise by minimizing false positives. - - "comprehensive": Aims to detect more security findings at the cost of potential false positives. - (precise,comprehensive) (default "precise") - --download-db-only download/update vulnerability database but don't run a scan - --download-java-db-only download/update Java index database but don't run a scan - --exit-code int specify exit code when any security issues are found - --exit-on-eol int exit with the specified code when the OS reaches end of service/life - --file-patterns strings specify config file patterns - -f, --format string format (table,json,template,sarif,cyclonedx,spdx,spdx-json,github,cosign-vuln) (default "table") - -h, --help help for sbom - --ignore-policy string specify the Rego file path to evaluate each vulnerability - --ignore-status strings comma-separated list of vulnerability status to ignore (unknown,not_affected,affected,fixed,under_investigation,will_not_fix,fix_deferred,end_of_life) - --ignore-unfixed display only fixed vulnerabilities - --ignored-licenses strings specify a list of license to ignore - --ignorefile string specify .trivyignore file (default ".trivyignore") - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") - --list-all-pkgs output all packages in the JSON report regardless of vulnerability - --no-progress suppress progress bar - --offline-scan do not issue API requests to identify dependencies - -o, --output string output file name - --output-plugin-arg string [EXPERIMENTAL] output plugin arguments - --pkg-relationships strings list of package relationships (unknown,root,direct,indirect) (default [unknown,root,direct,indirect]) - --pkg-types strings list of package types (os,library) (default [os,library]) - --redis-ca string redis ca file location, if using redis as cache backend - --redis-cert string redis certificate file location, if using redis as cache backend - --redis-key string redis key file location, if using redis as cache backend - --redis-tls enable redis TLS with public certificates, if using redis as cache backend - --rekor-url string [EXPERIMENTAL] address of rekor STL server (default "https://rekor.sigstore.dev") - --sbom-sources strings [EXPERIMENTAL] try to retrieve SBOM from the specified sources (oci,rekor) - --scanners strings comma-separated list of what security issues to detect (vuln,license) (default [vuln]) - --server string server address in client mode - -s, --severity strings severities of security issues to be displayed (UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL) (default [UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL]) - --show-suppressed [EXPERIMENTAL] show suppressed vulnerabilities - --skip-db-update skip updating vulnerability database - --skip-dirs strings specify the directories or glob patterns to skip - --skip-files strings specify the files or glob patterns to skip - --skip-java-db-update skip updating Java index database - --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update - -t, --template string output template - --token string for authentication in client/server mode - --token-header string specify a header name for token in client/server mode (default "Trivy-Token") - --vex strings [EXPERIMENTAL] VEX sources ("repo", "oci" or file path) + --cache-backend string [EXPERIMENTAL] cache backend (e.g. redis://localhost:6379) (default "memory") + --cache-ttl duration cache TTL when using redis as cache backend + --compliance string compliance report to generate + --custom-headers strings custom headers in client mode + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) + --detection-priority string specify the detection priority: + - "precise": Prioritizes precise by minimizing false positives. + - "comprehensive": Aims to detect more security findings at the cost of potential false positives. + (precise,comprehensive) (default "precise") + --download-db-only download/update vulnerability database but don't run a scan + --download-java-db-only download/update Java index database but don't run a scan + --exit-code int specify exit code when any security issues are found + --exit-on-eol int exit with the specified code when the OS reaches end of service/life + --file-patterns strings specify config file patterns + -f, --format string format (table,json,template,sarif,cyclonedx,spdx,spdx-json,github,cosign-vuln) (default "table") + -h, --help help for sbom + --ignore-policy string specify the Rego file path to evaluate each vulnerability + --ignore-status strings comma-separated list of vulnerability status to ignore (unknown,not_affected,affected,fixed,under_investigation,will_not_fix,fix_deferred,end_of_life) + --ignore-unfixed display only fixed vulnerabilities + --ignored-licenses strings specify a list of license to ignore + --ignorefile string specify .trivyignore file (default ".trivyignore") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) + --list-all-pkgs output all packages in the JSON report regardless of vulnerability + --no-progress suppress progress bar + --offline-scan do not issue API requests to identify dependencies + -o, --output string output file name + --output-plugin-arg string [EXPERIMENTAL] output plugin arguments + --pkg-relationships strings list of package relationships (unknown,root,direct,indirect) (default [unknown,root,direct,indirect]) + --pkg-types strings list of package types (os,library) (default [os,library]) + --redis-ca string redis ca file location, if using redis as cache backend + --redis-cert string redis certificate file location, if using redis as cache backend + --redis-key string redis key file location, if using redis as cache backend + --redis-tls enable redis TLS with public certificates, if using redis as cache backend + --rekor-url string [EXPERIMENTAL] address of rekor STL server (default "https://rekor.sigstore.dev") + --sbom-sources strings [EXPERIMENTAL] try to retrieve SBOM from the specified sources (oci,rekor) + --scanners strings comma-separated list of what security issues to detect (vuln,license) (default [vuln]) + --server string server address in client mode + -s, --severity strings severities of security issues to be displayed (UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL) (default [UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL]) + --show-suppressed [EXPERIMENTAL] show suppressed vulnerabilities + --skip-db-update skip updating vulnerability database + --skip-dirs strings specify the directories or glob patterns to skip + --skip-files strings specify the files or glob patterns to skip + --skip-java-db-update skip updating Java index database + --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update + -t, --template string output template + --token string for authentication in client/server mode + --token-header string specify a header name for token in client/server mode (default "Trivy-Token") + --vex strings [EXPERIMENTAL] VEX sources ("repo", "oci" or file path) ``` ### Options inherited from parent commands diff --git a/docs/docs/references/configuration/cli/trivy_server.md b/docs/docs/references/configuration/cli/trivy_server.md index 4291496e34f1..b7ada8f2c19b 100644 --- a/docs/docs/references/configuration/cli/trivy_server.md +++ b/docs/docs/references/configuration/cli/trivy_server.md @@ -22,7 +22,7 @@ trivy server [flags] ``` --cache-backend string [EXPERIMENTAL] cache backend (e.g. redis://localhost:6379) (default "fs") --cache-ttl duration cache TTL when using redis as cache backend - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --download-db-only download/update vulnerability database but don't run a scan --enable-modules strings [EXPERIMENTAL] module names to enable -h, --help help for server diff --git a/docs/docs/references/configuration/cli/trivy_vm.md b/docs/docs/references/configuration/cli/trivy_vm.md index fef17624222d..9b7f6a6fef43 100644 --- a/docs/docs/references/configuration/cli/trivy_vm.md +++ b/docs/docs/references/configuration/cli/trivy_vm.md @@ -27,7 +27,7 @@ trivy vm [flags] VM_IMAGE --compliance string compliance report to generate --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -52,7 +52,7 @@ trivy vm [flags] VM_IMAGE --ignore-unfixed display only fixed vulnerabilities --ignorefile string specify .trivyignore file (default ".trivyignore") --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --list-all-pkgs output all packages in the JSON report regardless of vulnerability --misconfig-scanners strings comma-separated list of misconfig scanners to use for misconfiguration scanning (default [azure-arm,cloudformation,dockerfile,helm,kubernetes,terraform,terraformplan-json,terraformplan-snapshot]) --module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules") diff --git a/docs/docs/references/configuration/config-file.md b/docs/docs/references/configuration/config-file.md index b2b25e47689e..ae3a017fda05 100644 --- a/docs/docs/references/configuration/config-file.md +++ b/docs/docs/references/configuration/config-file.md @@ -104,7 +104,8 @@ db: download-only: false # Same as '--java-db-repository' - java-repository: "ghcr.io/aquasecurity/trivy-java-db:1" + java-repository: + - ghcr.io/aquasecurity/trivy-java-db:1 # Same as '--skip-java-db-update' java-skip-update: false @@ -113,7 +114,8 @@ db: no-progress: false # Same as '--db-repository' - repository: "ghcr.io/aquasecurity/trivy-db:2" + repository: + - ghcr.io/aquasecurity/trivy-db:2 # Same as '--skip-db-update' skip-update: false diff --git a/internal/dbtest/fake.go b/internal/dbtest/fake.go index 7943c72273e2..528c549bda7f 100644 --- a/internal/dbtest/fake.go +++ b/internal/dbtest/fake.go @@ -62,7 +62,7 @@ func NewFakeDB(t *testing.T, dbPath string, opts FakeDBOptions) *oci.Artifact { opt := ftypes.RegistryOptions{ Insecure: false, } - return oci.NewArtifact("dummy", true, opt, oci.WithImage(img)) + return oci.NewArtifact("dummy", opt, oci.WithImage(img)) } func ArchiveDir(t *testing.T, dir string) string { diff --git a/pkg/commands/app.go b/pkg/commands/app.go index a3fb6df353d9..23b44fdec553 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -624,7 +624,7 @@ func NewServerCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { // java-db only works on client side. serverFlags.DBFlagGroup.DownloadJavaDBOnly = nil // disable '--download-java-db-only' serverFlags.DBFlagGroup.SkipJavaDBUpdate = nil // disable '--skip-java-db-update' - serverFlags.DBFlagGroup.JavaDBRepository = nil // disable '--java-db-repository' + serverFlags.DBFlagGroup.JavaDBRepositories = nil // disable '--java-db-repository' cmd := &cobra.Command{ Use: "server [flags]", diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go index fa454c0cd276..11c6ef6a3090 100644 --- a/pkg/commands/artifact/run.go +++ b/pkg/commands/artifact/run.go @@ -291,7 +291,7 @@ func (r *runner) initDB(ctx context.Context, opts flag.Options) error { // download the database file noProgress := opts.Quiet || opts.NoProgress - if err := operation.DownloadDB(ctx, opts.AppVersion, opts.CacheDir, opts.DBRepository, noProgress, opts.SkipDBUpdate, opts.RegistryOpts()); err != nil { + if err := operation.DownloadDB(ctx, opts.AppVersion, opts.CacheDir, opts.DBRepositories, noProgress, opts.SkipDBUpdate, opts.RegistryOpts()); err != nil { return err } @@ -321,7 +321,7 @@ func (r *runner) initJavaDB(opts flag.Options) error { // Update the Java DB noProgress := opts.Quiet || opts.NoProgress - javadb.Init(opts.CacheDir, opts.JavaDBRepository, opts.SkipJavaDBUpdate, noProgress, opts.RegistryOpts()) + javadb.Init(opts.CacheDir, opts.JavaDBRepositories, opts.SkipJavaDBUpdate, noProgress, opts.RegistryOpts()) if opts.DownloadJavaDBOnly { if err := javadb.Update(); err != nil { return xerrors.Errorf("Java DB error: %w", err) diff --git a/pkg/commands/operation/operation.go b/pkg/commands/operation/operation.go index 16aa72085949..7570c4393b69 100644 --- a/pkg/commands/operation/operation.go +++ b/pkg/commands/operation/operation.go @@ -21,14 +21,14 @@ import ( var mu sync.Mutex // DownloadDB downloads the DB -func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository name.Reference, quiet, skipUpdate bool, +func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepositories []name.Reference, quiet, skipUpdate bool, opt ftypes.RegistryOptions) error { mu.Lock() defer mu.Unlock() ctx = log.WithContextPrefix(ctx, "db") dbDir := db.Dir(cacheDir) - client := db.NewClient(dbDir, quiet, db.WithDBRepository(dbRepository)) + client := db.NewClient(dbDir, quiet, db.WithDBRepository(dbRepositories)) needsUpdate, err := client.NeedsUpdate(ctx, appVersion, skipUpdate) if err != nil { return xerrors.Errorf("database error: %w", err) @@ -36,7 +36,6 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n if needsUpdate { log.InfoContext(ctx, "Need to update DB") - log.InfoContext(ctx, "Downloading DB...", log.String("repository", dbRepository.String())) if err = client.Download(ctx, dbDir, opt); err != nil { return xerrors.Errorf("failed to download vulnerability DB: %w", err) } diff --git a/pkg/commands/server/run.go b/pkg/commands/server/run.go index 6c5316d7465a..e9187b3442f5 100644 --- a/pkg/commands/server/run.go +++ b/pkg/commands/server/run.go @@ -26,7 +26,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) { defer cleanup() // download the database file - if err = operation.DownloadDB(ctx, opts.AppVersion, opts.CacheDir, opts.DBRepository, + if err = operation.DownloadDB(ctx, opts.AppVersion, opts.CacheDir, opts.DBRepositories, true, opts.SkipDBUpdate, opts.RegistryOpts()); err != nil { return err } @@ -50,6 +50,6 @@ func Run(ctx context.Context, opts flag.Options) (err error) { m.Register() server := rpcServer.NewServer(opts.AppVersion, opts.Listen, opts.CacheDir, opts.Token, opts.TokenHeader, - opts.PathPrefix, opts.DBRepository, opts.RegistryOpts()) + opts.PathPrefix, opts.DBRepositories, opts.RegistryOpts()) return server.ListenAndServe(ctx, cacheClient, opts.SkipDBUpdate) } diff --git a/pkg/db/db.go b/pkg/db/db.go index 344b21b86e44..f7ceb09af4c9 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -2,14 +2,13 @@ package db import ( "context" - "errors" "fmt" "os" "path/filepath" "time" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/samber/lo" "golang.org/x/xerrors" "github.com/aquasecurity/trivy-db/pkg/db" @@ -18,7 +17,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/oci" - "github.com/aquasecurity/trivy/pkg/version/doc" ) const ( @@ -27,8 +25,9 @@ const ( ) var ( - DefaultRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-db", db.SchemaVersion) - defaultRepository, _ = name.NewTag(DefaultRepository) + // GitHub Container Registry + DefaultGHCRRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-db", db.SchemaVersion) + defaultGHCRRepository = lo.Must(name.NewTag(DefaultGHCRRepository)) Init = db.Init Close = db.Close @@ -36,8 +35,8 @@ var ( ) type options struct { - artifact *oci.Artifact - dbRepository name.Reference + artifact *oci.Artifact + dbRepositories []name.Reference } // Option is a functional option @@ -51,9 +50,9 @@ func WithOCIArtifact(art *oci.Artifact) Option { } // WithDBRepository takes a dbRepository -func WithDBRepository(dbRepository name.Reference) Option { +func WithDBRepository(dbRepository []name.Reference) Option { return func(opts *options) { - opts.dbRepository = dbRepository + opts.dbRepositories = dbRepository } } @@ -73,7 +72,9 @@ func Dir(cacheDir string) string { // NewClient is the factory method for DB client func NewClient(dbDir string, quiet bool, opts ...Option) *Client { o := &options{ - dbRepository: defaultRepository, + dbRepositories: []name.Reference{ + defaultGHCRRepository, + }, } for _, opt := range opts { @@ -153,20 +154,8 @@ func (c *Client) Download(ctx context.Context, dst string, opt types.RegistryOpt log.Debug("No metadata file") } - art := c.initOCIArtifact(opt) - if err := art.Download(ctx, dst, oci.DownloadOption{MediaType: dbMediaType}); err != nil { - var terr *transport.Error - if errors.As(err, &terr) { - for _, diagnostic := range terr.Errors { - // For better user experience - if diagnostic.Code == transport.DeniedErrorCode || diagnostic.Code == transport.UnauthorizedErrorCode { - // e.g. https://aquasecurity.github.io/trivy/latest/docs/references/troubleshooting/#db - log.Warnf("See %s", doc.URL("/docs/references/troubleshooting/", "db")) - break - } - } - } - return xerrors.Errorf("database download error: %w", err) + if err := c.downloadDB(ctx, opt, dst); err != nil { + return xerrors.Errorf("OCI artifact error: %w", err) } if err := c.updateDownloadedAt(ctx, dst); err != nil { @@ -201,11 +190,20 @@ func (c *Client) updateDownloadedAt(ctx context.Context, dbDir string) error { return nil } -func (c *Client) initOCIArtifact(opt types.RegistryOptions) *oci.Artifact { +func (c *Client) initArtifacts(opt types.RegistryOptions) oci.Artifacts { if c.artifact != nil { - return c.artifact + return oci.Artifacts{c.artifact} } - return oci.NewArtifact(c.dbRepository.String(), c.quiet, opt) + return oci.NewArtifacts(c.dbRepositories, opt) +} + +func (c *Client) downloadDB(ctx context.Context, opt types.RegistryOptions, dst string) error { + log.Info("Downloading vulnerability DB...") + downloadOpt := oci.DownloadOption{MediaType: dbMediaType, Quiet: c.quiet} + if err := c.initArtifacts(opt).Download(ctx, dst, downloadOpt); err != nil { + return xerrors.Errorf("failed to download vulnerability DB: %w", err) + } + return nil } func (c *Client) ShowInfo() error { diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go index ff5e36e2b194..f64c0dc4a5c6 100644 --- a/pkg/db/db_test.go +++ b/pkg/db/db_test.go @@ -159,7 +159,7 @@ func TestClient_Download(t *testing.T) { { name: "invalid gzip", input: "testdata/trivy.db", - wantErr: "unexpected EOF", + wantErr: "OCI artifact error: failed to download vulnerability DB", }, } diff --git a/pkg/fanal/analyzer/analyzer_test.go b/pkg/fanal/analyzer/analyzer_test.go index 313fccda7e94..d8f93c0aa420 100644 --- a/pkg/fanal/analyzer/analyzer_test.go +++ b/pkg/fanal/analyzer/analyzer_test.go @@ -623,9 +623,9 @@ func TestAnalyzerGroup_PostAnalyze(t *testing.T) { if tt.analyzerType == analyzer.TypeJar { // init java-trivy-db with skip update - repo, err := name.NewTag(javadb.DefaultRepository) + repo, err := name.NewTag(javadb.DefaultGHCRRepository) require.NoError(t, err) - javadb.Init("./language/java/jar/testdata", repo, true, false, types.RegistryOptions{Insecure: false}) + javadb.Init("./language/java/jar/testdata", []name.Reference{repo}, true, false, types.RegistryOptions{Insecure: false}) } ctx := context.Background() diff --git a/pkg/fanal/analyzer/language/java/jar/jar_test.go b/pkg/fanal/analyzer/language/java/jar/jar_test.go index 225737166c81..146b60c1bbd6 100644 --- a/pkg/fanal/analyzer/language/java/jar/jar_test.go +++ b/pkg/fanal/analyzer/language/java/jar/jar_test.go @@ -132,9 +132,9 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // init java-trivy-db with skip update - repo, err := name.NewTag(javadb.DefaultRepository) + repo, err := name.NewTag(javadb.DefaultGHCRRepository) require.NoError(t, err) - javadb.Init("testdata", repo, true, false, types.RegistryOptions{Insecure: false}) + javadb.Init("testdata", []name.Reference{repo}, true, false, types.RegistryOptions{Insecure: false}) a := javaLibraryAnalyzer{} ctx := context.Background() diff --git a/pkg/fanal/artifact/image/remote_sbom.go b/pkg/fanal/artifact/image/remote_sbom.go index e95fef23140c..d07d9cafe3ac 100644 --- a/pkg/fanal/artifact/image/remote_sbom.go +++ b/pkg/fanal/artifact/image/remote_sbom.go @@ -95,10 +95,11 @@ func (a Artifact) parseReferrer(ctx context.Context, repo string, desc v1.Descri defer os.RemoveAll(tmpDir) // Download SBOM to local filesystem - referrer := oci.NewArtifact(repoName, true, a.artifactOption.ImageOption.RegistryOptions) + referrer := oci.NewArtifact(repoName, a.artifactOption.ImageOption.RegistryOptions) if err = referrer.Download(ctx, tmpDir, oci.DownloadOption{ MediaType: desc.ArtifactType, Filename: fileName, + Quiet: true, }); err != nil { return artifact.Reference{}, xerrors.Errorf("SBOM download error: %w", err) } diff --git a/pkg/flag/db_flags.go b/pkg/flag/db_flags.go index 352a0094be7c..df0d6c6f5194 100644 --- a/pkg/flag/db_flags.go +++ b/pkg/flag/db_flags.go @@ -50,17 +50,17 @@ var ( ConfigName: "db.no-progress", Usage: "suppress progress bar", } - DBRepositoryFlag = Flag[string]{ + DBRepositoryFlag = Flag[[]string]{ Name: "db-repository", ConfigName: "db.repository", - Default: db.DefaultRepository, - Usage: "OCI repository to retrieve trivy-db from", + Default: []string{db.DefaultGHCRRepository}, + Usage: "OCI repository(ies) to retrieve trivy-db in order of priority", } - JavaDBRepositoryFlag = Flag[string]{ + JavaDBRepositoryFlag = Flag[[]string]{ Name: "java-db-repository", ConfigName: "db.java-repository", - Default: javadb.DefaultRepository, - Usage: "OCI repository to retrieve trivy-java-db from", + Default: []string{javadb.DefaultGHCRRepository}, + Usage: "OCI repository(ies) to retrieve trivy-java-db in order of priority", } LightFlag = Flag[bool]{ Name: "light", @@ -78,8 +78,8 @@ type DBFlagGroup struct { DownloadJavaDBOnly *Flag[bool] SkipJavaDBUpdate *Flag[bool] NoProgress *Flag[bool] - DBRepository *Flag[string] - JavaDBRepository *Flag[string] + DBRepositories *Flag[[]string] + JavaDBRepositories *Flag[[]string] Light *Flag[bool] // deprecated } @@ -90,8 +90,8 @@ type DBOptions struct { DownloadJavaDBOnly bool SkipJavaDBUpdate bool NoProgress bool - DBRepository name.Reference - JavaDBRepository name.Reference + DBRepositories []name.Reference + JavaDBRepositories []name.Reference } // NewDBFlagGroup returns a default DBFlagGroup @@ -104,8 +104,8 @@ func NewDBFlagGroup() *DBFlagGroup { SkipJavaDBUpdate: SkipJavaDBUpdateFlag.Clone(), Light: LightFlag.Clone(), NoProgress: NoProgressFlag.Clone(), - DBRepository: DBRepositoryFlag.Clone(), - JavaDBRepository: JavaDBRepositoryFlag.Clone(), + DBRepositories: DBRepositoryFlag.Clone(), + JavaDBRepositories: JavaDBRepositoryFlag.Clone(), } } @@ -121,8 +121,8 @@ func (f *DBFlagGroup) Flags() []Flagger { f.DownloadJavaDBOnly, f.SkipJavaDBUpdate, f.NoProgress, - f.DBRepository, - f.JavaDBRepository, + f.DBRepositories, + f.JavaDBRepositories, f.Light, } } @@ -147,30 +147,21 @@ func (f *DBFlagGroup) ToOptions() (DBOptions, error) { return DBOptions{}, xerrors.New("--skip-java-db-update and --download-java-db-only options can not be specified both") } - var dbRepository, javaDBRepository name.Reference - var err error - if f.DBRepository != nil { - if dbRepository, err = name.ParseReference(f.DBRepository.Value(), name.WithDefaultTag("")); err != nil { - return DBOptions{}, xerrors.Errorf("invalid db repository: %w", err) - } - // Add the schema version if the tag is not specified for backward compatibility. - if t, ok := dbRepository.(name.Tag); ok && t.TagStr() == "" { - dbRepository = t.Tag(fmt.Sprint(db.SchemaVersion)) - log.Info("Adding schema version to the DB repository for backward compatibility", - log.String("repository", dbRepository.String())) + var dbRepositories, javaDBRepositories []name.Reference + for _, repo := range f.DBRepositories.Value() { + ref, err := parseRepository(repo, db.SchemaVersion) + if err != nil { + return DBOptions{}, xerrors.Errorf("invalid DB repository: %w", err) } + dbRepositories = append(dbRepositories, ref) } - if f.JavaDBRepository != nil { - if javaDBRepository, err = name.ParseReference(f.JavaDBRepository.Value(), name.WithDefaultTag("")); err != nil { + for _, repo := range f.JavaDBRepositories.Value() { + ref, err := parseRepository(repo, javadb.SchemaVersion) + if err != nil { return DBOptions{}, xerrors.Errorf("invalid javadb repository: %w", err) } - // Add the schema version if the tag is not specified for backward compatibility. - if t, ok := javaDBRepository.(name.Tag); ok && t.TagStr() == "" { - javaDBRepository = t.Tag(fmt.Sprint(javadb.SchemaVersion)) - log.Info("Adding schema version to the Java DB repository for backward compatibility", - log.String("repository", javaDBRepository.String())) - } + javaDBRepositories = append(javaDBRepositories, ref) } return DBOptions{ @@ -180,7 +171,26 @@ func (f *DBFlagGroup) ToOptions() (DBOptions, error) { DownloadJavaDBOnly: downloadJavaDBOnly, SkipJavaDBUpdate: skipJavaDBUpdate, NoProgress: f.NoProgress.Value(), - DBRepository: dbRepository, - JavaDBRepository: javaDBRepository, + DBRepositories: dbRepositories, + JavaDBRepositories: javaDBRepositories, }, nil } + +func parseRepository(repo string, dbSchemaVersion int) (name.Reference, error) { + dbRepository, err := name.ParseReference(repo, name.WithDefaultTag("")) + if err != nil { + return nil, err + } + + // Add the schema version if the tag is not specified for backward compatibility. + t, ok := dbRepository.(name.Tag) + if !ok || t.TagStr() != "" { + return dbRepository, nil + } + + dbRepository = t.Tag(fmt.Sprint(dbSchemaVersion)) + log.Info("Adding schema version to the DB repository for backward compatibility", + log.String("repository", dbRepository.String())) + + return dbRepository, nil +} diff --git a/pkg/flag/db_flags_test.go b/pkg/flag/db_flags_test.go index 2bbd5b00198c..4f742e74ed68 100644 --- a/pkg/flag/db_flags_test.go +++ b/pkg/flag/db_flags_test.go @@ -17,31 +17,34 @@ func TestDBFlagGroup_ToOptions(t *testing.T) { SkipDBUpdate bool DownloadDBOnly bool Light bool - DBRepository string - JavaDBRepository string + DBRepository []string + JavaDBRepository []string } tests := []struct { - name string - fields fields - want flag.DBOptions - wantLogs []string - assertion require.ErrorAssertionFunc + name string + fields fields + want flag.DBOptions + wantLogs []string + wantErr string }{ { name: "happy", fields: fields{ SkipDBUpdate: true, DownloadDBOnly: false, - DBRepository: "ghcr.io/aquasecurity/trivy-db", - JavaDBRepository: "ghcr.io/aquasecurity/trivy-java-db", + DBRepository: []string{"ghcr.io/aquasecurity/trivy-db"}, + JavaDBRepository: []string{"ghcr.io/aquasecurity/trivy-java-db"}, }, want: flag.DBOptions{ - SkipDBUpdate: true, - DownloadDBOnly: false, - DBRepository: name.Tag{}, // All fields are unexported - JavaDBRepository: name.Tag{}, // All fields are unexported + SkipDBUpdate: true, + DownloadDBOnly: false, + DBRepositories: []name.Reference{name.Tag{}}, // All fields are unexported + JavaDBRepositories: []name.Reference{name.Tag{}}, // All fields are unexported + }, + wantLogs: []string{ + `Adding schema version to the DB repository for backward compatibility repository="ghcr.io/aquasecurity/trivy-db:2"`, + `Adding schema version to the DB repository for backward compatibility repository="ghcr.io/aquasecurity/trivy-java-db:1"`, }, - assertion: require.NoError, }, { name: "sad", @@ -49,25 +52,36 @@ func TestDBFlagGroup_ToOptions(t *testing.T) { SkipDBUpdate: true, DownloadDBOnly: true, }, - assertion: func(t require.TestingT, err error, msgs ...any) { - require.ErrorContains(t, err, "--skip-db-update and --download-db-only options can not be specified both") - }, + wantErr: "--skip-db-update and --download-db-only options can not be specified both", }, { name: "invalid repo", fields: fields{ SkipDBUpdate: true, DownloadDBOnly: false, - DBRepository: "foo:bar:baz", + DBRepository: []string{"foo:bar:baz"}, }, - assertion: func(t require.TestingT, err error, msgs ...any) { - require.ErrorContains(t, err, "invalid db repository") + wantErr: "invalid DB repository", + }, + { + name: "multiple repos", + fields: fields{ + SkipDBUpdate: true, + DownloadDBOnly: false, + DBRepository: []string{"ghcr.io/aquasecurity/trivy-db:2", "gallery.ecr.aws/aquasecurity/trivy-db:2"}, + JavaDBRepository: []string{"ghcr.io/aquasecurity/trivy-java-db:1", "gallery.ecr.aws/aquasecurity/trivy-java-db:1"}, + }, + want: flag.DBOptions{ + SkipDBUpdate: true, + DownloadDBOnly: false, + DBRepositories: []name.Reference{name.Tag{}, name.Tag{}}, // All fields are unexported + JavaDBRepositories: []name.Reference{name.Tag{}, name.Tag{}}, // All fields are unexported }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - out := newLogger(log.LevelWarn) + out := newLogger(log.LevelInfo) viper.Set(flag.SkipDBUpdateFlag.ConfigName, tt.fields.SkipDBUpdate) viper.Set(flag.DownloadDBOnlyFlag.ConfigName, tt.fields.DownloadDBOnly) @@ -76,13 +90,19 @@ func TestDBFlagGroup_ToOptions(t *testing.T) { // Assert options f := &flag.DBFlagGroup{ - DownloadDBOnly: flag.DownloadDBOnlyFlag.Clone(), - SkipDBUpdate: flag.SkipDBUpdateFlag.Clone(), - DBRepository: flag.DBRepositoryFlag.Clone(), - JavaDBRepository: flag.JavaDBRepositoryFlag.Clone(), + DownloadDBOnly: flag.DownloadDBOnlyFlag.Clone(), + SkipDBUpdate: flag.SkipDBUpdateFlag.Clone(), + DBRepositories: flag.DBRepositoryFlag.Clone(), + JavaDBRepositories: flag.JavaDBRepositoryFlag.Clone(), } got, err := f.ToOptions() - tt.assertion(t, err) + if tt.wantErr != "" { + require.Error(t, err) + assert.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.EqualExportedValues(t, tt.want, got) // Assert log messages diff --git a/pkg/javadb/client.go b/pkg/javadb/client.go index 9f90a37ba231..8c7aa81df9a3 100644 --- a/pkg/javadb/client.go +++ b/pkg/javadb/client.go @@ -26,12 +26,15 @@ const ( mediaType = "application/vnd.aquasec.trivy.javadb.layer.v1.tar+gzip" ) -var DefaultRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-java-db", SchemaVersion) +var ( + // GitHub Container Registry + DefaultGHCRRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-java-db", SchemaVersion) +) var updater *Updater type Updater struct { - repo name.Reference + repos []name.Reference dbDir string skip bool quiet bool @@ -40,8 +43,7 @@ type Updater struct { } func (u *Updater) Update() error { - dbDir := u.dbDir - metac := db.NewMetadata(dbDir) + metac := db.NewMetadata(u.dbDir) meta, err := metac.Get() if err != nil { @@ -55,13 +57,9 @@ func (u *Updater) Update() error { if (meta.Version != SchemaVersion || !u.isNewDB(meta)) && !u.skip { // Download DB - log.Info("Java DB Repository", log.Any("repository", u.repo)) - log.Info("Downloading the Java DB...") - // TODO: support remote options - art := oci.NewArtifact(u.repo.String(), u.quiet, u.registryOption) - if err = art.Download(context.Background(), dbDir, oci.DownloadOption{MediaType: mediaType}); err != nil { - return xerrors.Errorf("DB download error: %w", err) + if err := u.downloadDB(); err != nil { + return xerrors.Errorf("OCI artifact error: %w", err) } // Parse the newly downloaded metadata.json @@ -96,9 +94,21 @@ func (u *Updater) isNewDB(meta db.Metadata) bool { return false } -func Init(cacheDir string, javaDBRepository name.Reference, skip, quiet bool, registryOption ftypes.RegistryOptions) { +func (u *Updater) downloadDB() error { + log.Info("Downloading Java DB...") + + artifacts := oci.NewArtifacts(u.repos, u.registryOption) + downloadOpt := oci.DownloadOption{MediaType: mediaType, Quiet: u.quiet} + if err := artifacts.Download(context.Background(), u.dbDir, downloadOpt); err != nil { + return xerrors.Errorf("failed to download vulnerability DB: %w", err) + } + + return xerrors.New("failed to download Java DB from any source") +} + +func Init(cacheDir string, javaDBRepositories []name.Reference, skip, quiet bool, registryOption ftypes.RegistryOptions) { updater = &Updater{ - repo: javaDBRepository, + repos: javaDBRepositories, dbDir: dbDir(cacheDir), skip: skip, quiet: quiet, diff --git a/pkg/module/command.go b/pkg/module/command.go index 2dd955aa0e4b..a74da8384c3f 100644 --- a/pkg/module/command.go +++ b/pkg/module/command.go @@ -23,11 +23,11 @@ func Install(ctx context.Context, dir, repo string, quiet bool, opt types.Regist } log.Info("Installing the module from the repository...", log.String("repo", repo)) - art := oci.NewArtifact(repo, quiet, opt) + art := oci.NewArtifact(repo, opt) dst := filepath.Join(dir, ref.Context().Name()) log.Debug("Installing the module...", log.String("dst", dst)) - if err = art.Download(ctx, dst, oci.DownloadOption{MediaType: mediaType}); err != nil { + if err = art.Download(ctx, dst, oci.DownloadOption{MediaType: mediaType, Quiet: quiet}); err != nil { return xerrors.Errorf("module download error: %w", err) } diff --git a/pkg/oci/artifact.go b/pkg/oci/artifact.go index d8474941b3f5..345149892a4e 100644 --- a/pkg/oci/artifact.go +++ b/pkg/oci/artifact.go @@ -2,6 +2,7 @@ package oci import ( "context" + "errors" "io" "os" "path/filepath" @@ -10,11 +11,15 @@ import ( "github.com/cheggaaa/pb/v3" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/samber/lo" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/downloader" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/remote" + "github.com/aquasecurity/trivy/pkg/version/doc" ) const ( @@ -48,7 +53,6 @@ func WithImage(img v1.Image) Option { type Artifact struct { m sync.Mutex repository string - quiet bool // For OCI registries types.RegistryOptions @@ -57,10 +61,9 @@ type Artifact struct { } // NewArtifact returns a new artifact -func NewArtifact(repo string, quiet bool, registryOpt types.RegistryOptions, opts ...Option) *Artifact { +func NewArtifact(repo string, registryOpt types.RegistryOptions, opts ...Option) *Artifact { art := &Artifact{ repository: repo, - quiet: quiet, RegistryOptions: registryOpt, } @@ -98,6 +101,7 @@ func (a *Artifact) populate(ctx context.Context, opt types.RegistryOptions) erro type DownloadOption struct { MediaType string // Accept any media type if not specified Filename string // Use the annotation if not specified + Quiet bool } func (a *Artifact) Download(ctx context.Context, dir string, opt DownloadOption) error { @@ -140,14 +144,14 @@ func (a *Artifact) Download(ctx context.Context, dir string, opt DownloadOption) return xerrors.Errorf("unacceptable media type: %s", string(layerMediaType)) } - if err = a.download(ctx, layer, fileName, dir); err != nil { + if err = a.download(ctx, layer, fileName, dir, opt.Quiet); err != nil { return xerrors.Errorf("oci download error: %w", err) } return nil } -func (a *Artifact) download(ctx context.Context, layer v1.Layer, fileName, dir string) error { +func (a *Artifact) download(ctx context.Context, layer v1.Layer, fileName, dir string, quiet bool) error { size, err := layer.Size() if err != nil { return xerrors.Errorf("size error: %w", err) @@ -161,7 +165,7 @@ func (a *Artifact) download(ctx context.Context, layer v1.Layer, fileName, dir s // Show progress bar bar := pb.Full.Start64(size) - if a.quiet { + if quiet { bar.SetWriter(io.Discard) } pr := bar.NewProxyReader(rc) @@ -207,3 +211,54 @@ func (a *Artifact) Digest(ctx context.Context) (string, error) { } return digest.String(), nil } + +type Artifacts []*Artifact + +// NewArtifacts returns a slice of artifacts. +func NewArtifacts(repos []name.Reference, opt types.RegistryOptions, opts ...Option) Artifacts { + return lo.Map(repos, func(r name.Reference, _ int) *Artifact { + return NewArtifact(r.String(), opt, opts...) + }) +} + +// Download downloads artifacts until one of them succeeds. +// Attempts to download next artifact if the first one fails due to a temporary error. +func (a Artifacts) Download(ctx context.Context, dst string, opt DownloadOption) error { + for i, art := range a { + log.Info("Downloading artifact...", log.String("repo", art.repository)) + err := art.Download(ctx, dst, opt) + if err == nil { + log.Info("Artifact successfully downloaded", log.String("repo", art.repository)) + return nil + } + + if !shouldTryOtherRepo(err) { + return xerrors.Errorf("failed to download artifact from %s: %w", art.repository, err) + } + log.Error("Failed to download artifact", log.String("repo", art.repository), log.Err(err)) + if i < len(a)-1 { + log.Info("Trying to download artifact from other repository...") + } + } + + return xerrors.New("failed to download artifact from any source") +} + +func shouldTryOtherRepo(err error) bool { + var terr *transport.Error + if !errors.As(err, &terr) { + return false + } + + for _, diagnostic := range terr.Errors { + // For better user experience + if diagnostic.Code == transport.DeniedErrorCode || diagnostic.Code == transport.UnauthorizedErrorCode { + // e.g. https://aquasecurity.github.io/trivy/latest/docs/references/troubleshooting/#db + log.Warnf("See %s", doc.URL("/docs/references/troubleshooting/", "db")) + break + } + } + + // try the following artifact only if a temporary error occurs + return terr.Temporary() +} diff --git a/pkg/oci/artifact_test.go b/pkg/oci/artifact_test.go index 7cc8cd19d3c1..a8ce6e542641 100644 --- a/pkg/oci/artifact_test.go +++ b/pkg/oci/artifact_test.go @@ -116,9 +116,10 @@ func TestArtifact_Download(t *testing.T) { }, }, nil) - artifact := oci.NewArtifact("repo", true, ftypes.RegistryOptions{}, oci.WithImage(img)) + artifact := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img)) err = artifact.Download(context.Background(), tempDir, oci.DownloadOption{ MediaType: tt.mediaType, + Quiet: true, }) if tt.wantErr != "" { assert.ErrorContains(t, err, tt.wantErr) diff --git a/pkg/policy/policy.go b/pkg/policy/policy.go index c8c05cf067c7..2dbe2fc1b89a 100644 --- a/pkg/policy/policy.go +++ b/pkg/policy/policy.go @@ -92,7 +92,7 @@ func NewClient(cacheDir string, quiet bool, checkBundleRepo string, opts ...Opti func (c *Client) populateOCIArtifact(registryOpts types.RegistryOptions) { if c.artifact == nil { log.Debug("Loading check bundle", log.String("repository", c.checkBundleRepo)) - c.artifact = oci.NewArtifact(c.checkBundleRepo, c.quiet, registryOpts) + c.artifact = oci.NewArtifact(c.checkBundleRepo, registryOpts) } } @@ -101,7 +101,9 @@ func (c *Client) DownloadBuiltinPolicies(ctx context.Context, registryOpts types c.populateOCIArtifact(registryOpts) dst := c.contentDir() - if err := c.artifact.Download(ctx, dst, oci.DownloadOption{MediaType: policyMediaType}); err != nil { + if err := c.artifact.Download(ctx, dst, + oci.DownloadOption{MediaType: policyMediaType, Quiet: c.quiet}, + ); err != nil { return xerrors.Errorf("download error: %w", err) } diff --git a/pkg/policy/policy_test.go b/pkg/policy/policy_test.go index 4dbd6b9a7558..f6875eb1c9cb 100644 --- a/pkg/policy/policy_test.go +++ b/pkg/policy/policy_test.go @@ -116,7 +116,7 @@ func TestClient_LoadBuiltinPolicies(t *testing.T) { }, nil) // Mock OCI artifact - art := oci.NewArtifact("repo", true, ftypes.RegistryOptions{}, oci.WithImage(img)) + art := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img)) c, err := policy.NewClient(tt.cacheDir, true, "", policy.WithOCIArtifact(art)) require.NoError(t, err) @@ -255,7 +255,7 @@ func TestClient_NeedsUpdate(t *testing.T) { require.NoError(t, err) } - art := oci.NewArtifact("repo", true, ftypes.RegistryOptions{}, oci.WithImage(img)) + art := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img)) c, err := policy.NewClient(tmpDir, true, "", policy.WithOCIArtifact(art), policy.WithClock(tt.clock)) require.NoError(t, err) @@ -357,7 +357,7 @@ func TestClient_DownloadBuiltinPolicies(t *testing.T) { }, nil) // Mock OCI artifact - art := oci.NewArtifact("repo", true, ftypes.RegistryOptions{}, oci.WithImage(img)) + art := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img)) c, err := policy.NewClient(tempDir, true, "", policy.WithClock(tt.clock), policy.WithOCIArtifact(art)) require.NoError(t, err) diff --git a/pkg/rpc/server/listen.go b/pkg/rpc/server/listen.go index f1e39fa1b2f0..c1bc5033530c 100644 --- a/pkg/rpc/server/listen.go +++ b/pkg/rpc/server/listen.go @@ -29,21 +29,21 @@ const updateInterval = 1 * time.Hour // Server represents Trivy server type Server struct { - appVersion string - addr string - cacheDir string - dbDir string - token string - tokenHeader string - pathPrefix string - dbRepository name.Reference + appVersion string + addr string + cacheDir string + dbDir string + token string + tokenHeader string + pathPrefix string + dbRepositories []name.Reference // For OCI registries types.RegistryOptions } // NewServer returns an instance of Server -func NewServer(appVersion, addr, cacheDir, token, tokenHeader, pathPrefix string, dbRepository name.Reference, opt types.RegistryOptions) Server { +func NewServer(appVersion, addr, cacheDir, token, tokenHeader, pathPrefix string, dbRepositories []name.Reference, opt types.RegistryOptions) Server { return Server{ appVersion: appVersion, addr: addr, @@ -52,7 +52,7 @@ func NewServer(appVersion, addr, cacheDir, token, tokenHeader, pathPrefix string token: token, tokenHeader: tokenHeader, pathPrefix: pathPrefix, - dbRepository: dbRepository, + dbRepositories: dbRepositories, RegistryOptions: opt, } } @@ -63,7 +63,7 @@ func (s Server) ListenAndServe(ctx context.Context, serverCache cache.Cache, ski dbUpdateWg := &sync.WaitGroup{} go func() { - worker := newDBWorker(db.NewClient(s.dbDir, true, db.WithDBRepository(s.dbRepository))) + worker := newDBWorker(db.NewClient(s.dbDir, true, db.WithDBRepository(s.dbRepositories))) for { time.Sleep(updateInterval) if err := worker.update(ctx, s.appVersion, s.dbDir, skipDBUpdate, dbUpdateWg, requestWg, s.RegistryOptions); err != nil {