From 7b19d5d8e16f5e4464d59ecf73c11a6bbd10e768 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Thu, 26 Sep 2024 17:31:48 +0600 Subject: [PATCH 1/9] feat: support multiple DB repositories for vulnerability and Java DB Signed-off-by: nikpivkin --- .../configuration/cli/trivy_filesystem.md | 4 +- .../configuration/cli/trivy_image.md | 4 +- .../configuration/cli/trivy_kubernetes.md | 4 +- .../configuration/cli/trivy_repository.md | 4 +- .../configuration/cli/trivy_rootfs.md | 4 +- .../configuration/cli/trivy_sbom.md | 96 +++++++++---------- .../configuration/cli/trivy_server.md | 2 +- .../references/configuration/cli/trivy_vm.md | 4 +- .../references/configuration/config-file.md | 8 +- pkg/commands/app.go | 2 +- pkg/commands/artifact/run.go | 4 +- pkg/commands/operation/operation.go | 6 +- pkg/commands/server/run.go | 4 +- pkg/db/db.go | 62 ++++++++---- pkg/fanal/analyzer/analyzer_test.go | 4 +- .../analyzer/language/java/jar/jar_test.go | 4 +- pkg/flag/db_flags.go | 80 +++++++++------- pkg/flag/db_flags_test.go | 72 +++++++++----- pkg/javadb/client.go | 48 +++++++--- pkg/rpc/server/listen.go | 22 ++--- 20 files changed, 261 insertions(+), 177 deletions(-) diff --git a/docs/docs/references/configuration/cli/trivy_filesystem.md b/docs/docs/references/configuration/cli/trivy_filesystem.md index e907c5d4f9d5..0d5fed4b7738 100644 --- a/docs/docs/references/configuration/cli/trivy_filesystem.md +++ b/docs/docs/references/configuration/cli/trivy_filesystem.md @@ -29,7 +29,7 @@ trivy filesystem [flags] PATH --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -56,7 +56,7 @@ trivy filesystem [flags] PATH --include-deprecated-checks include deprecated checks (default true) --include-dev-deps include development dependencies in the report (supported: npm, yarn) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_image.md b/docs/docs/references/configuration/cli/trivy_image.md index 156582f047ad..5ee0c8ffbbc8 100644 --- a/docs/docs/references/configuration/cli/trivy_image.md +++ b/docs/docs/references/configuration/cli/trivy_image.md @@ -43,7 +43,7 @@ trivy image [flags] IMAGE_NAME --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -74,7 +74,7 @@ trivy image [flags] IMAGE_NAME --include-deprecated-checks include deprecated checks (default true) --include-non-failures include successes and exceptions, available with '--scanners misconfig' --input string input file path instead of image name - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_kubernetes.md b/docs/docs/references/configuration/cli/trivy_kubernetes.md index b7c626a7d476..4490abccfd33 100644 --- a/docs/docs/references/configuration/cli/trivy_kubernetes.md +++ b/docs/docs/references/configuration/cli/trivy_kubernetes.md @@ -38,7 +38,7 @@ trivy kubernetes [flags] [CONTEXT] --config-check strings specify the paths to the Rego check files or to the directories containing them, applying config files --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -70,7 +70,7 @@ trivy kubernetes [flags] [CONTEXT] --include-kinds strings indicate the kinds included in scanning (example: node) --include-namespaces strings indicate the namespaces included in scanning (example: kube-system) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --k8s-version string specify k8s version to validate outdated api by it (example: 1.21.0) --kubeconfig string specify the kubeconfig file path to use --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_repository.md b/docs/docs/references/configuration/cli/trivy_repository.md index 04f3f26e919a..ae8151aec905 100644 --- a/docs/docs/references/configuration/cli/trivy_repository.md +++ b/docs/docs/references/configuration/cli/trivy_repository.md @@ -29,7 +29,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -56,7 +56,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --include-deprecated-checks include deprecated checks (default true) --include-dev-deps include development dependencies in the report (supported: npm, yarn) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_rootfs.md b/docs/docs/references/configuration/cli/trivy_rootfs.md index 0e0dfa54d448..9ee89f549f4d 100644 --- a/docs/docs/references/configuration/cli/trivy_rootfs.md +++ b/docs/docs/references/configuration/cli/trivy_rootfs.md @@ -31,7 +31,7 @@ trivy rootfs [flags] ROOTDIR --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -58,7 +58,7 @@ trivy rootfs [flags] ROOTDIR --ignorefile string specify .trivyignore file (default ".trivyignore") --include-deprecated-checks include deprecated checks (default true) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_sbom.md b/docs/docs/references/configuration/cli/trivy_sbom.md index 1b0df922c7fc..a850591e1c93 100644 --- a/docs/docs/references/configuration/cli/trivy_sbom.md +++ b/docs/docs/references/configuration/cli/trivy_sbom.md @@ -20,54 +20,54 @@ trivy sbom [flags] SBOM_PATH ### Options ``` - --cache-backend string [EXPERIMENTAL] cache backend (e.g. redis://localhost:6379) (default "memory") - --cache-ttl duration cache TTL when using redis as cache backend - --compliance string compliance report to generate - --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") - --detection-priority string specify the detection priority: - - "precise": Prioritizes precise by minimizing false positives. - - "comprehensive": Aims to detect more security findings at the cost of potential false positives. - (precise,comprehensive) (default "precise") - --download-db-only download/update vulnerability database but don't run a scan - --download-java-db-only download/update Java index database but don't run a scan - --exit-code int specify exit code when any security issues are found - --exit-on-eol int exit with the specified code when the OS reaches end of service/life - --file-patterns strings specify config file patterns - -f, --format string format (table,json,template,sarif,cyclonedx,spdx,spdx-json,github,cosign-vuln) (default "table") - -h, --help help for sbom - --ignore-policy string specify the Rego file path to evaluate each vulnerability - --ignore-status strings comma-separated list of vulnerability status to ignore (unknown,not_affected,affected,fixed,under_investigation,will_not_fix,fix_deferred,end_of_life) - --ignore-unfixed display only fixed vulnerabilities - --ignored-licenses strings specify a list of license to ignore - --ignorefile string specify .trivyignore file (default ".trivyignore") - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") - --list-all-pkgs output all packages in the JSON report regardless of vulnerability - --no-progress suppress progress bar - --offline-scan do not issue API requests to identify dependencies - -o, --output string output file name - --output-plugin-arg string [EXPERIMENTAL] output plugin arguments - --pkg-relationships strings list of package relationships (unknown,root,direct,indirect) (default [unknown,root,direct,indirect]) - --pkg-types strings list of package types (os,library) (default [os,library]) - --redis-ca string redis ca file location, if using redis as cache backend - --redis-cert string redis certificate file location, if using redis as cache backend - --redis-key string redis key file location, if using redis as cache backend - --redis-tls enable redis TLS with public certificates, if using redis as cache backend - --rekor-url string [EXPERIMENTAL] address of rekor STL server (default "https://rekor.sigstore.dev") - --sbom-sources strings [EXPERIMENTAL] try to retrieve SBOM from the specified sources (oci,rekor) - --scanners strings comma-separated list of what security issues to detect (vuln,license) (default [vuln]) - --server string server address in client mode - -s, --severity strings severities of security issues to be displayed (UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL) (default [UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL]) - --show-suppressed [EXPERIMENTAL] show suppressed vulnerabilities - --skip-db-update skip updating vulnerability database - --skip-dirs strings specify the directories or glob patterns to skip - --skip-files strings specify the files or glob patterns to skip - --skip-java-db-update skip updating Java index database - --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update - -t, --template string output template - --token string for authentication in client/server mode - --token-header string specify a header name for token in client/server mode (default "Trivy-Token") - --vex strings [EXPERIMENTAL] VEX sources ("repo", "oci" or file path) + --cache-backend string [EXPERIMENTAL] cache backend (e.g. redis://localhost:6379) (default "memory") + --cache-ttl duration cache TTL when using redis as cache backend + --compliance string compliance report to generate + --custom-headers strings custom headers in client mode + --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --detection-priority string specify the detection priority: + - "precise": Prioritizes precise by minimizing false positives. + - "comprehensive": Aims to detect more security findings at the cost of potential false positives. + (precise,comprehensive) (default "precise") + --download-db-only download/update vulnerability database but don't run a scan + --download-java-db-only download/update Java index database but don't run a scan + --exit-code int specify exit code when any security issues are found + --exit-on-eol int exit with the specified code when the OS reaches end of service/life + --file-patterns strings specify config file patterns + -f, --format string format (table,json,template,sarif,cyclonedx,spdx,spdx-json,github,cosign-vuln) (default "table") + -h, --help help for sbom + --ignore-policy string specify the Rego file path to evaluate each vulnerability + --ignore-status strings comma-separated list of vulnerability status to ignore (unknown,not_affected,affected,fixed,under_investigation,will_not_fix,fix_deferred,end_of_life) + --ignore-unfixed display only fixed vulnerabilities + --ignored-licenses strings specify a list of license to ignore + --ignorefile string specify .trivyignore file (default ".trivyignore") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --list-all-pkgs output all packages in the JSON report regardless of vulnerability + --no-progress suppress progress bar + --offline-scan do not issue API requests to identify dependencies + -o, --output string output file name + --output-plugin-arg string [EXPERIMENTAL] output plugin arguments + --pkg-relationships strings list of package relationships (unknown,root,direct,indirect) (default [unknown,root,direct,indirect]) + --pkg-types strings list of package types (os,library) (default [os,library]) + --redis-ca string redis ca file location, if using redis as cache backend + --redis-cert string redis certificate file location, if using redis as cache backend + --redis-key string redis key file location, if using redis as cache backend + --redis-tls enable redis TLS with public certificates, if using redis as cache backend + --rekor-url string [EXPERIMENTAL] address of rekor STL server (default "https://rekor.sigstore.dev") + --sbom-sources strings [EXPERIMENTAL] try to retrieve SBOM from the specified sources (oci,rekor) + --scanners strings comma-separated list of what security issues to detect (vuln,license) (default [vuln]) + --server string server address in client mode + -s, --severity strings severities of security issues to be displayed (UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL) (default [UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL]) + --show-suppressed [EXPERIMENTAL] show suppressed vulnerabilities + --skip-db-update skip updating vulnerability database + --skip-dirs strings specify the directories or glob patterns to skip + --skip-files strings specify the files or glob patterns to skip + --skip-java-db-update skip updating Java index database + --skip-vex-repo-update [EXPERIMENTAL] Skip VEX Repository update + -t, --template string output template + --token string for authentication in client/server mode + --token-header string specify a header name for token in client/server mode (default "Trivy-Token") + --vex strings [EXPERIMENTAL] VEX sources ("repo", "oci" or file path) ``` ### Options inherited from parent commands diff --git a/docs/docs/references/configuration/cli/trivy_server.md b/docs/docs/references/configuration/cli/trivy_server.md index 4291496e34f1..9edb2fe931f7 100644 --- a/docs/docs/references/configuration/cli/trivy_server.md +++ b/docs/docs/references/configuration/cli/trivy_server.md @@ -22,7 +22,7 @@ trivy server [flags] ``` --cache-backend string [EXPERIMENTAL] cache backend (e.g. redis://localhost:6379) (default "fs") --cache-ttl duration cache TTL when using redis as cache backend - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --download-db-only download/update vulnerability database but don't run a scan --enable-modules strings [EXPERIMENTAL] module names to enable -h, --help help for server diff --git a/docs/docs/references/configuration/cli/trivy_vm.md b/docs/docs/references/configuration/cli/trivy_vm.md index fef17624222d..27af05423b65 100644 --- a/docs/docs/references/configuration/cli/trivy_vm.md +++ b/docs/docs/references/configuration/cli/trivy_vm.md @@ -27,7 +27,7 @@ trivy vm [flags] VM_IMAGE --compliance string compliance report to generate --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository string OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2") + --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -52,7 +52,7 @@ trivy vm [flags] VM_IMAGE --ignore-unfixed display only fixed vulnerabilities --ignorefile string specify .trivyignore file (default ".trivyignore") --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository string OCI repository to retrieve trivy-java-db from (default "ghcr.io/aquasecurity/trivy-java-db:1") + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --list-all-pkgs output all packages in the JSON report regardless of vulnerability --misconfig-scanners strings comma-separated list of misconfig scanners to use for misconfiguration scanning (default [azure-arm,cloudformation,dockerfile,helm,kubernetes,terraform,terraformplan-json,terraformplan-snapshot]) --module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules") diff --git a/docs/docs/references/configuration/config-file.md b/docs/docs/references/configuration/config-file.md index b2b25e47689e..5bb27e589057 100644 --- a/docs/docs/references/configuration/config-file.md +++ b/docs/docs/references/configuration/config-file.md @@ -104,7 +104,9 @@ db: download-only: false # Same as '--java-db-repository' - java-repository: "ghcr.io/aquasecurity/trivy-java-db:1" + java-repository: + - ghcr.io/aquasecurity/trivy-java-db:1 + - public.ecr.aws/aquasecurity/trivy-java-db:1 # Same as '--skip-java-db-update' java-skip-update: false @@ -113,7 +115,9 @@ db: no-progress: false # Same as '--db-repository' - repository: "ghcr.io/aquasecurity/trivy-db:2" + repository: + - ghcr.io/aquasecurity/trivy-db:2 + - public.ecr.aws/aquasecurity/trivy-db:2 # Same as '--skip-db-update' skip-update: false diff --git a/pkg/commands/app.go b/pkg/commands/app.go index a3fb6df353d9..23b44fdec553 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -624,7 +624,7 @@ func NewServerCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { // java-db only works on client side. serverFlags.DBFlagGroup.DownloadJavaDBOnly = nil // disable '--download-java-db-only' serverFlags.DBFlagGroup.SkipJavaDBUpdate = nil // disable '--skip-java-db-update' - serverFlags.DBFlagGroup.JavaDBRepository = nil // disable '--java-db-repository' + serverFlags.DBFlagGroup.JavaDBRepositories = nil // disable '--java-db-repository' cmd := &cobra.Command{ Use: "server [flags]", diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go index fa454c0cd276..11c6ef6a3090 100644 --- a/pkg/commands/artifact/run.go +++ b/pkg/commands/artifact/run.go @@ -291,7 +291,7 @@ func (r *runner) initDB(ctx context.Context, opts flag.Options) error { // download the database file noProgress := opts.Quiet || opts.NoProgress - if err := operation.DownloadDB(ctx, opts.AppVersion, opts.CacheDir, opts.DBRepository, noProgress, opts.SkipDBUpdate, opts.RegistryOpts()); err != nil { + if err := operation.DownloadDB(ctx, opts.AppVersion, opts.CacheDir, opts.DBRepositories, noProgress, opts.SkipDBUpdate, opts.RegistryOpts()); err != nil { return err } @@ -321,7 +321,7 @@ func (r *runner) initJavaDB(opts flag.Options) error { // Update the Java DB noProgress := opts.Quiet || opts.NoProgress - javadb.Init(opts.CacheDir, opts.JavaDBRepository, opts.SkipJavaDBUpdate, noProgress, opts.RegistryOpts()) + javadb.Init(opts.CacheDir, opts.JavaDBRepositories, opts.SkipJavaDBUpdate, noProgress, opts.RegistryOpts()) if opts.DownloadJavaDBOnly { if err := javadb.Update(); err != nil { return xerrors.Errorf("Java DB error: %w", err) diff --git a/pkg/commands/operation/operation.go b/pkg/commands/operation/operation.go index 16aa72085949..7ca3425cdcc5 100644 --- a/pkg/commands/operation/operation.go +++ b/pkg/commands/operation/operation.go @@ -21,14 +21,14 @@ import ( var mu sync.Mutex // DownloadDB downloads the DB -func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository name.Reference, quiet, skipUpdate bool, +func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepositories []name.Reference, quiet, skipUpdate bool, opt ftypes.RegistryOptions) error { mu.Lock() defer mu.Unlock() ctx = log.WithContextPrefix(ctx, "db") dbDir := db.Dir(cacheDir) - client := db.NewClient(dbDir, quiet, db.WithDBRepository(dbRepository)) + client := db.NewClient(dbDir, quiet, db.WithDBRepository(dbRepositories)) needsUpdate, err := client.NeedsUpdate(ctx, appVersion, skipUpdate) if err != nil { return xerrors.Errorf("database error: %w", err) @@ -36,7 +36,7 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n if needsUpdate { log.InfoContext(ctx, "Need to update DB") - log.InfoContext(ctx, "Downloading DB...", log.String("repository", dbRepository.String())) + log.InfoContext(ctx, "Downloading DB...", log.Any("repositories", dbRepositories)) if err = client.Download(ctx, dbDir, opt); err != nil { return xerrors.Errorf("failed to download vulnerability DB: %w", err) } diff --git a/pkg/commands/server/run.go b/pkg/commands/server/run.go index 6c5316d7465a..e9187b3442f5 100644 --- a/pkg/commands/server/run.go +++ b/pkg/commands/server/run.go @@ -26,7 +26,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) { defer cleanup() // download the database file - if err = operation.DownloadDB(ctx, opts.AppVersion, opts.CacheDir, opts.DBRepository, + if err = operation.DownloadDB(ctx, opts.AppVersion, opts.CacheDir, opts.DBRepositories, true, opts.SkipDBUpdate, opts.RegistryOpts()); err != nil { return err } @@ -50,6 +50,6 @@ func Run(ctx context.Context, opts flag.Options) (err error) { m.Register() server := rpcServer.NewServer(opts.AppVersion, opts.Listen, opts.CacheDir, opts.Token, opts.TokenHeader, - opts.PathPrefix, opts.DBRepository, opts.RegistryOpts()) + opts.PathPrefix, opts.DBRepositories, opts.RegistryOpts()) return server.ListenAndServe(ctx, cacheClient, opts.SkipDBUpdate) } diff --git a/pkg/db/db.go b/pkg/db/db.go index e4af60d092c6..e271d872eab4 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -10,6 +10,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/samber/lo" "golang.org/x/xerrors" "github.com/aquasecurity/trivy-db/pkg/db" @@ -27,8 +28,13 @@ const ( ) var ( - DefaultRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-db", db.SchemaVersion) - defaultRepository, _ = name.NewTag(DefaultRepository) + // GitHub Container Registry + DefaultGHCRRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-db", db.SchemaVersion) + defaultGHCRRepository = lo.Must(name.NewTag(DefaultGHCRRepository)) + + // Amazon Elastic Container Registry + DefaultECRRepository = fmt.Sprintf("%s:%d", "public.ecr.aws/aquasecurity/trivy-db", db.SchemaVersion) + defaultECRRepository = lo.Must(name.NewTag(DefaultECRRepository)) Init = db.Init Close = db.Close @@ -36,8 +42,8 @@ var ( ) type options struct { - artifact *oci.Artifact - dbRepository name.Reference + artifact *oci.Artifact + dbRepositories []name.Reference } // Option is a functional option @@ -51,9 +57,9 @@ func WithOCIArtifact(art *oci.Artifact) Option { } // WithDBRepository takes a dbRepository -func WithDBRepository(dbRepository name.Reference) Option { +func WithDBRepository(dbRepository []name.Reference) Option { return func(opts *options) { - opts.dbRepository = dbRepository + opts.dbRepositories = dbRepository } } @@ -73,7 +79,10 @@ func Dir(cacheDir string) string { // NewClient is the factory method for DB client func NewClient(dbDir string, quiet bool, opts ...Option) *Client { o := &options{ - dbRepository: defaultRepository, + dbRepositories: []name.Reference{ + defaultGHCRRepository, + defaultECRRepository, + }, } for _, opt := range opts { @@ -153,16 +162,11 @@ func (c *Client) Download(ctx context.Context, dst string, opt types.RegistryOpt log.Debug("No metadata file") } - art, err := c.initOCIArtifact(opt) - if err != nil { + if err := c.downloadDB(ctx, opt, dst); err != nil { return xerrors.Errorf("OCI artifact error: %w", err) } - if err = art.Download(ctx, dst, oci.DownloadOption{MediaType: dbMediaType}); err != nil { - return xerrors.Errorf("database download error: %w", err) - } - - if err = c.updateDownloadedAt(ctx, dst); err != nil { + if err := c.updateDownloadedAt(ctx, dst); err != nil { return xerrors.Errorf("failed to update downloaded_at: %w", err) } return nil @@ -194,12 +198,12 @@ func (c *Client) updateDownloadedAt(ctx context.Context, dbDir string) error { return nil } -func (c *Client) initOCIArtifact(opt types.RegistryOptions) (*oci.Artifact, error) { +func (c *Client) initOCIArtifact(repository name.Reference, opt types.RegistryOptions) (*oci.Artifact, error) { if c.artifact != nil { return c.artifact, nil } - art, err := oci.NewArtifact(c.dbRepository.String(), c.quiet, opt) + art, err := oci.NewArtifact(repository.String(), c.quiet, opt) if err != nil { var terr *transport.Error if errors.As(err, &terr) { @@ -217,6 +221,32 @@ func (c *Client) initOCIArtifact(opt types.RegistryOptions) (*oci.Artifact, erro return art, nil } +func (c *Client) downloadDB(ctx context.Context, opt types.RegistryOptions, dst string) error { + downloadOpt := oci.DownloadOption{MediaType: dbMediaType} + if c.artifact != nil { + return c.artifact.Download(ctx, dst, downloadOpt) + } + + for i, repo := range c.dbRepositories { + a, err := c.initOCIArtifact(repo, opt) + if err != nil { + return err + } + + log.Info("Downloading vulnerability DB...", log.String("repo", repo.String())) + if err := a.Download(ctx, dst, downloadOpt); err != nil { + log.Error("Failed to download DB", log.String("repo", repo.String()), log.Err(err)) + if i < len(c.dbRepositories)-1 { + log.Info("Trying to download DB from other repository...") + } + continue + } + return nil + } + + return xerrors.New("failed to download vulnerability DB from any source") +} + func (c *Client) ShowInfo() error { meta, err := c.metadata.Get() if err != nil { diff --git a/pkg/fanal/analyzer/analyzer_test.go b/pkg/fanal/analyzer/analyzer_test.go index 313fccda7e94..d8f93c0aa420 100644 --- a/pkg/fanal/analyzer/analyzer_test.go +++ b/pkg/fanal/analyzer/analyzer_test.go @@ -623,9 +623,9 @@ func TestAnalyzerGroup_PostAnalyze(t *testing.T) { if tt.analyzerType == analyzer.TypeJar { // init java-trivy-db with skip update - repo, err := name.NewTag(javadb.DefaultRepository) + repo, err := name.NewTag(javadb.DefaultGHCRRepository) require.NoError(t, err) - javadb.Init("./language/java/jar/testdata", repo, true, false, types.RegistryOptions{Insecure: false}) + javadb.Init("./language/java/jar/testdata", []name.Reference{repo}, true, false, types.RegistryOptions{Insecure: false}) } ctx := context.Background() diff --git a/pkg/fanal/analyzer/language/java/jar/jar_test.go b/pkg/fanal/analyzer/language/java/jar/jar_test.go index 225737166c81..146b60c1bbd6 100644 --- a/pkg/fanal/analyzer/language/java/jar/jar_test.go +++ b/pkg/fanal/analyzer/language/java/jar/jar_test.go @@ -132,9 +132,9 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // init java-trivy-db with skip update - repo, err := name.NewTag(javadb.DefaultRepository) + repo, err := name.NewTag(javadb.DefaultGHCRRepository) require.NoError(t, err) - javadb.Init("testdata", repo, true, false, types.RegistryOptions{Insecure: false}) + javadb.Init("testdata", []name.Reference{repo}, true, false, types.RegistryOptions{Insecure: false}) a := javaLibraryAnalyzer{} ctx := context.Background() diff --git a/pkg/flag/db_flags.go b/pkg/flag/db_flags.go index 352a0094be7c..f6ff24450305 100644 --- a/pkg/flag/db_flags.go +++ b/pkg/flag/db_flags.go @@ -50,17 +50,17 @@ var ( ConfigName: "db.no-progress", Usage: "suppress progress bar", } - DBRepositoryFlag = Flag[string]{ + DBRepositoryFlag = Flag[[]string]{ Name: "db-repository", ConfigName: "db.repository", - Default: db.DefaultRepository, - Usage: "OCI repository to retrieve trivy-db from", + Default: []string{db.DefaultGHCRRepository, db.DefaultECRRepository}, + Usage: "OCI repository(ies) to retrieve trivy-db from", } - JavaDBRepositoryFlag = Flag[string]{ + JavaDBRepositoryFlag = Flag[[]string]{ Name: "java-db-repository", ConfigName: "db.java-repository", - Default: javadb.DefaultRepository, - Usage: "OCI repository to retrieve trivy-java-db from", + Default: []string{javadb.DefaultGHCRRepository, javadb.DefaultECRRepository}, + Usage: "OCI repository(ies) to retrieve trivy-java-db from", } LightFlag = Flag[bool]{ Name: "light", @@ -78,8 +78,8 @@ type DBFlagGroup struct { DownloadJavaDBOnly *Flag[bool] SkipJavaDBUpdate *Flag[bool] NoProgress *Flag[bool] - DBRepository *Flag[string] - JavaDBRepository *Flag[string] + DBRepositories *Flag[[]string] + JavaDBRepositories *Flag[[]string] Light *Flag[bool] // deprecated } @@ -90,8 +90,8 @@ type DBOptions struct { DownloadJavaDBOnly bool SkipJavaDBUpdate bool NoProgress bool - DBRepository name.Reference - JavaDBRepository name.Reference + DBRepositories []name.Reference + JavaDBRepositories []name.Reference } // NewDBFlagGroup returns a default DBFlagGroup @@ -104,8 +104,8 @@ func NewDBFlagGroup() *DBFlagGroup { SkipJavaDBUpdate: SkipJavaDBUpdateFlag.Clone(), Light: LightFlag.Clone(), NoProgress: NoProgressFlag.Clone(), - DBRepository: DBRepositoryFlag.Clone(), - JavaDBRepository: JavaDBRepositoryFlag.Clone(), + DBRepositories: DBRepositoryFlag.Clone(), + JavaDBRepositories: JavaDBRepositoryFlag.Clone(), } } @@ -121,8 +121,8 @@ func (f *DBFlagGroup) Flags() []Flagger { f.DownloadJavaDBOnly, f.SkipJavaDBUpdate, f.NoProgress, - f.DBRepository, - f.JavaDBRepository, + f.DBRepositories, + f.JavaDBRepositories, f.Light, } } @@ -147,30 +147,21 @@ func (f *DBFlagGroup) ToOptions() (DBOptions, error) { return DBOptions{}, xerrors.New("--skip-java-db-update and --download-java-db-only options can not be specified both") } - var dbRepository, javaDBRepository name.Reference - var err error - if f.DBRepository != nil { - if dbRepository, err = name.ParseReference(f.DBRepository.Value(), name.WithDefaultTag("")); err != nil { - return DBOptions{}, xerrors.Errorf("invalid db repository: %w", err) - } - // Add the schema version if the tag is not specified for backward compatibility. - if t, ok := dbRepository.(name.Tag); ok && t.TagStr() == "" { - dbRepository = t.Tag(fmt.Sprint(db.SchemaVersion)) - log.Info("Adding schema version to the DB repository for backward compatibility", - log.String("repository", dbRepository.String())) + var dbRepositories, javaDBRepositories []name.Reference + for _, repo := range f.DBRepositories.Value() { + ref, err := parseRepository(repo, db.SchemaVersion) + if err != nil { + return DBOptions{}, xerrors.Errorf("invalid DB repository: %w", err) } + dbRepositories = append(dbRepositories, ref) } - if f.JavaDBRepository != nil { - if javaDBRepository, err = name.ParseReference(f.JavaDBRepository.Value(), name.WithDefaultTag("")); err != nil { + for _, repo := range f.JavaDBRepositories.Value() { + ref, err := parseRepository(repo, javadb.SchemaVersion) + if err != nil { return DBOptions{}, xerrors.Errorf("invalid javadb repository: %w", err) } - // Add the schema version if the tag is not specified for backward compatibility. - if t, ok := javaDBRepository.(name.Tag); ok && t.TagStr() == "" { - javaDBRepository = t.Tag(fmt.Sprint(javadb.SchemaVersion)) - log.Info("Adding schema version to the Java DB repository for backward compatibility", - log.String("repository", javaDBRepository.String())) - } + javaDBRepositories = append(javaDBRepositories, ref) } return DBOptions{ @@ -180,7 +171,26 @@ func (f *DBFlagGroup) ToOptions() (DBOptions, error) { DownloadJavaDBOnly: downloadJavaDBOnly, SkipJavaDBUpdate: skipJavaDBUpdate, NoProgress: f.NoProgress.Value(), - DBRepository: dbRepository, - JavaDBRepository: javaDBRepository, + DBRepositories: dbRepositories, + JavaDBRepositories: javaDBRepositories, }, nil } + +func parseRepository(repo string, dbSchemaVersion int) (name.Reference, error) { + dbRepository, err := name.ParseReference(repo, name.WithDefaultTag("")) + if err != nil { + return nil, err + } + + // Add the schema version if the tag is not specified for backward compatibility. + t, ok := dbRepository.(name.Tag) + if !ok || t.TagStr() != "" { + return dbRepository, nil + } + + dbRepository = t.Tag(fmt.Sprint(dbSchemaVersion)) + log.Info("Adding schema version to the DB repository for backward compatibility", + log.String("repository", dbRepository.String())) + + return dbRepository, nil +} diff --git a/pkg/flag/db_flags_test.go b/pkg/flag/db_flags_test.go index 2bbd5b00198c..4f742e74ed68 100644 --- a/pkg/flag/db_flags_test.go +++ b/pkg/flag/db_flags_test.go @@ -17,31 +17,34 @@ func TestDBFlagGroup_ToOptions(t *testing.T) { SkipDBUpdate bool DownloadDBOnly bool Light bool - DBRepository string - JavaDBRepository string + DBRepository []string + JavaDBRepository []string } tests := []struct { - name string - fields fields - want flag.DBOptions - wantLogs []string - assertion require.ErrorAssertionFunc + name string + fields fields + want flag.DBOptions + wantLogs []string + wantErr string }{ { name: "happy", fields: fields{ SkipDBUpdate: true, DownloadDBOnly: false, - DBRepository: "ghcr.io/aquasecurity/trivy-db", - JavaDBRepository: "ghcr.io/aquasecurity/trivy-java-db", + DBRepository: []string{"ghcr.io/aquasecurity/trivy-db"}, + JavaDBRepository: []string{"ghcr.io/aquasecurity/trivy-java-db"}, }, want: flag.DBOptions{ - SkipDBUpdate: true, - DownloadDBOnly: false, - DBRepository: name.Tag{}, // All fields are unexported - JavaDBRepository: name.Tag{}, // All fields are unexported + SkipDBUpdate: true, + DownloadDBOnly: false, + DBRepositories: []name.Reference{name.Tag{}}, // All fields are unexported + JavaDBRepositories: []name.Reference{name.Tag{}}, // All fields are unexported + }, + wantLogs: []string{ + `Adding schema version to the DB repository for backward compatibility repository="ghcr.io/aquasecurity/trivy-db:2"`, + `Adding schema version to the DB repository for backward compatibility repository="ghcr.io/aquasecurity/trivy-java-db:1"`, }, - assertion: require.NoError, }, { name: "sad", @@ -49,25 +52,36 @@ func TestDBFlagGroup_ToOptions(t *testing.T) { SkipDBUpdate: true, DownloadDBOnly: true, }, - assertion: func(t require.TestingT, err error, msgs ...any) { - require.ErrorContains(t, err, "--skip-db-update and --download-db-only options can not be specified both") - }, + wantErr: "--skip-db-update and --download-db-only options can not be specified both", }, { name: "invalid repo", fields: fields{ SkipDBUpdate: true, DownloadDBOnly: false, - DBRepository: "foo:bar:baz", + DBRepository: []string{"foo:bar:baz"}, }, - assertion: func(t require.TestingT, err error, msgs ...any) { - require.ErrorContains(t, err, "invalid db repository") + wantErr: "invalid DB repository", + }, + { + name: "multiple repos", + fields: fields{ + SkipDBUpdate: true, + DownloadDBOnly: false, + DBRepository: []string{"ghcr.io/aquasecurity/trivy-db:2", "gallery.ecr.aws/aquasecurity/trivy-db:2"}, + JavaDBRepository: []string{"ghcr.io/aquasecurity/trivy-java-db:1", "gallery.ecr.aws/aquasecurity/trivy-java-db:1"}, + }, + want: flag.DBOptions{ + SkipDBUpdate: true, + DownloadDBOnly: false, + DBRepositories: []name.Reference{name.Tag{}, name.Tag{}}, // All fields are unexported + JavaDBRepositories: []name.Reference{name.Tag{}, name.Tag{}}, // All fields are unexported }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - out := newLogger(log.LevelWarn) + out := newLogger(log.LevelInfo) viper.Set(flag.SkipDBUpdateFlag.ConfigName, tt.fields.SkipDBUpdate) viper.Set(flag.DownloadDBOnlyFlag.ConfigName, tt.fields.DownloadDBOnly) @@ -76,13 +90,19 @@ func TestDBFlagGroup_ToOptions(t *testing.T) { // Assert options f := &flag.DBFlagGroup{ - DownloadDBOnly: flag.DownloadDBOnlyFlag.Clone(), - SkipDBUpdate: flag.SkipDBUpdateFlag.Clone(), - DBRepository: flag.DBRepositoryFlag.Clone(), - JavaDBRepository: flag.JavaDBRepositoryFlag.Clone(), + DownloadDBOnly: flag.DownloadDBOnlyFlag.Clone(), + SkipDBUpdate: flag.SkipDBUpdateFlag.Clone(), + DBRepositories: flag.DBRepositoryFlag.Clone(), + JavaDBRepositories: flag.JavaDBRepositoryFlag.Clone(), } got, err := f.ToOptions() - tt.assertion(t, err) + if tt.wantErr != "" { + require.Error(t, err) + assert.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.EqualExportedValues(t, tt.want, got) // Assert log messages diff --git a/pkg/javadb/client.go b/pkg/javadb/client.go index 936c2d40c028..cb1cc419e6ba 100644 --- a/pkg/javadb/client.go +++ b/pkg/javadb/client.go @@ -26,12 +26,17 @@ const ( mediaType = "application/vnd.aquasec.trivy.javadb.layer.v1.tar+gzip" ) -var DefaultRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-java-db", SchemaVersion) +var ( + // GitHub Container Registry + DefaultGHCRRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-java-db", SchemaVersion) + // Amazon Elastic Container Registry + DefaultECRRepository = fmt.Sprintf("%s:%d", "public.ecr.aws/aquasecurity/trivy-java-db", SchemaVersion) +) var updater *Updater type Updater struct { - repo name.Reference + repos []name.Reference dbDir string skip bool quiet bool @@ -40,8 +45,7 @@ type Updater struct { } func (u *Updater) Update() error { - dbDir := u.dbDir - metac := db.NewMetadata(dbDir) + metac := db.NewMetadata(u.dbDir) meta, err := metac.Get() if err != nil { @@ -55,16 +59,11 @@ func (u *Updater) Update() error { if (meta.Version != SchemaVersion || !u.isNewDB(meta)) && !u.skip { // Download DB - log.Info("Java DB Repository", log.Any("repository", u.repo)) - log.Info("Downloading the Java DB...") + log.Info("Java DB Repositories", log.Any("repositories", u.repos)) // TODO: support remote options - var a *oci.Artifact - if a, err = oci.NewArtifact(u.repo.String(), u.quiet, u.registryOption); err != nil { - return xerrors.Errorf("oci error: %w", err) - } - if err = a.Download(context.Background(), dbDir, oci.DownloadOption{MediaType: mediaType}); err != nil { - return xerrors.Errorf("DB download error: %w", err) + if err := u.downloadDB(); err != nil { + return xerrors.Errorf("OCI artifact error: %w", err) } // Parse the newly downloaded metadata.json @@ -99,9 +98,30 @@ func (u *Updater) isNewDB(meta db.Metadata) bool { return false } -func Init(cacheDir string, javaDBRepository name.Reference, skip, quiet bool, registryOption ftypes.RegistryOptions) { +func (u *Updater) downloadDB() error { + for i, repo := range u.repos { + a, err := oci.NewArtifact(repo.String(), u.quiet, u.registryOption) + if err != nil { + return xerrors.Errorf("OCI error: %w", err) + } + + log.Info("Downloading the Java DB...", log.String("repository", repo.String())) + if err := a.Download(context.Background(), u.dbDir, oci.DownloadOption{MediaType: mediaType}); err != nil { + log.Error("Failed to download DB", log.String("repo", repo.String()), log.Err(err)) + if i < len(u.repos)-1 { + log.Info("Trying to download DB from other repository...") + } + continue + } + return nil + } + + return xerrors.New("failed to download Java DB from any source") +} + +func Init(cacheDir string, javaDBRepositories []name.Reference, skip, quiet bool, registryOption ftypes.RegistryOptions) { updater = &Updater{ - repo: javaDBRepository, + repos: javaDBRepositories, dbDir: dbDir(cacheDir), skip: skip, quiet: quiet, diff --git a/pkg/rpc/server/listen.go b/pkg/rpc/server/listen.go index f1e39fa1b2f0..c1bc5033530c 100644 --- a/pkg/rpc/server/listen.go +++ b/pkg/rpc/server/listen.go @@ -29,21 +29,21 @@ const updateInterval = 1 * time.Hour // Server represents Trivy server type Server struct { - appVersion string - addr string - cacheDir string - dbDir string - token string - tokenHeader string - pathPrefix string - dbRepository name.Reference + appVersion string + addr string + cacheDir string + dbDir string + token string + tokenHeader string + pathPrefix string + dbRepositories []name.Reference // For OCI registries types.RegistryOptions } // NewServer returns an instance of Server -func NewServer(appVersion, addr, cacheDir, token, tokenHeader, pathPrefix string, dbRepository name.Reference, opt types.RegistryOptions) Server { +func NewServer(appVersion, addr, cacheDir, token, tokenHeader, pathPrefix string, dbRepositories []name.Reference, opt types.RegistryOptions) Server { return Server{ appVersion: appVersion, addr: addr, @@ -52,7 +52,7 @@ func NewServer(appVersion, addr, cacheDir, token, tokenHeader, pathPrefix string token: token, tokenHeader: tokenHeader, pathPrefix: pathPrefix, - dbRepository: dbRepository, + dbRepositories: dbRepositories, RegistryOptions: opt, } } @@ -63,7 +63,7 @@ func (s Server) ListenAndServe(ctx context.Context, serverCache cache.Cache, ski dbUpdateWg := &sync.WaitGroup{} go func() { - worker := newDBWorker(db.NewClient(s.dbDir, true, db.WithDBRepository(s.dbRepository))) + worker := newDBWorker(db.NewClient(s.dbDir, true, db.WithDBRepository(s.dbRepositories))) for { time.Sleep(updateInterval) if err := worker.update(ctx, s.appVersion, s.dbDir, skipDBUpdate, dbUpdateWg, requestWg, s.RegistryOptions); err != nil { From aea7eaccee69f9e0069dcf4ae25a287413d189e6 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Fri, 27 Sep 2024 18:11:25 +0600 Subject: [PATCH 2/9] chore: remove unnecessary logging Signed-off-by: nikpivkin --- pkg/commands/operation/operation.go | 1 - pkg/javadb/client.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/pkg/commands/operation/operation.go b/pkg/commands/operation/operation.go index 7ca3425cdcc5..7570c4393b69 100644 --- a/pkg/commands/operation/operation.go +++ b/pkg/commands/operation/operation.go @@ -36,7 +36,6 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepositories if needsUpdate { log.InfoContext(ctx, "Need to update DB") - log.InfoContext(ctx, "Downloading DB...", log.Any("repositories", dbRepositories)) if err = client.Download(ctx, dbDir, opt); err != nil { return xerrors.Errorf("failed to download vulnerability DB: %w", err) } diff --git a/pkg/javadb/client.go b/pkg/javadb/client.go index cb1cc419e6ba..a1485c125681 100644 --- a/pkg/javadb/client.go +++ b/pkg/javadb/client.go @@ -59,8 +59,6 @@ func (u *Updater) Update() error { if (meta.Version != SchemaVersion || !u.isNewDB(meta)) && !u.skip { // Download DB - log.Info("Java DB Repositories", log.Any("repositories", u.repos)) - // TODO: support remote options if err := u.downloadDB(); err != nil { return xerrors.Errorf("OCI artifact error: %w", err) From 50ad0d6c282f01d7670d772738a0da9a6cceab98 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Fri, 27 Sep 2024 18:13:16 +0600 Subject: [PATCH 3/9] cli: update description of db flags Signed-off-by: nikpivkin --- docs/docs/references/configuration/cli/trivy_filesystem.md | 4 ++-- docs/docs/references/configuration/cli/trivy_image.md | 4 ++-- docs/docs/references/configuration/cli/trivy_kubernetes.md | 4 ++-- docs/docs/references/configuration/cli/trivy_repository.md | 4 ++-- docs/docs/references/configuration/cli/trivy_rootfs.md | 4 ++-- docs/docs/references/configuration/cli/trivy_sbom.md | 4 ++-- docs/docs/references/configuration/cli/trivy_server.md | 2 +- docs/docs/references/configuration/cli/trivy_vm.md | 4 ++-- pkg/flag/db_flags.go | 4 ++-- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/docs/references/configuration/cli/trivy_filesystem.md b/docs/docs/references/configuration/cli/trivy_filesystem.md index 0d5fed4b7738..f907b74fce18 100644 --- a/docs/docs/references/configuration/cli/trivy_filesystem.md +++ b/docs/docs/references/configuration/cli/trivy_filesystem.md @@ -29,7 +29,7 @@ trivy filesystem [flags] PATH --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -56,7 +56,7 @@ trivy filesystem [flags] PATH --include-deprecated-checks include deprecated checks (default true) --include-dev-deps include development dependencies in the report (supported: npm, yarn) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_image.md b/docs/docs/references/configuration/cli/trivy_image.md index 5ee0c8ffbbc8..919570031ae0 100644 --- a/docs/docs/references/configuration/cli/trivy_image.md +++ b/docs/docs/references/configuration/cli/trivy_image.md @@ -43,7 +43,7 @@ trivy image [flags] IMAGE_NAME --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -74,7 +74,7 @@ trivy image [flags] IMAGE_NAME --include-deprecated-checks include deprecated checks (default true) --include-non-failures include successes and exceptions, available with '--scanners misconfig' --input string input file path instead of image name - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_kubernetes.md b/docs/docs/references/configuration/cli/trivy_kubernetes.md index 4490abccfd33..9dae1722bbe2 100644 --- a/docs/docs/references/configuration/cli/trivy_kubernetes.md +++ b/docs/docs/references/configuration/cli/trivy_kubernetes.md @@ -38,7 +38,7 @@ trivy kubernetes [flags] [CONTEXT] --config-check strings specify the paths to the Rego check files or to the directories containing them, applying config files --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking - --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -70,7 +70,7 @@ trivy kubernetes [flags] [CONTEXT] --include-kinds strings indicate the kinds included in scanning (example: node) --include-namespaces strings indicate the namespaces included in scanning (example: kube-system) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --k8s-version string specify k8s version to validate outdated api by it (example: 1.21.0) --kubeconfig string specify the kubeconfig file path to use --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_repository.md b/docs/docs/references/configuration/cli/trivy_repository.md index ae8151aec905..8a4bbd35f56e 100644 --- a/docs/docs/references/configuration/cli/trivy_repository.md +++ b/docs/docs/references/configuration/cli/trivy_repository.md @@ -29,7 +29,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -56,7 +56,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --include-deprecated-checks include deprecated checks (default true) --include-dev-deps include development dependencies in the report (supported: npm, yarn) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_rootfs.md b/docs/docs/references/configuration/cli/trivy_rootfs.md index 9ee89f549f4d..b7ce12fc32fd 100644 --- a/docs/docs/references/configuration/cli/trivy_rootfs.md +++ b/docs/docs/references/configuration/cli/trivy_rootfs.md @@ -31,7 +31,7 @@ trivy rootfs [flags] ROOTDIR --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -58,7 +58,7 @@ trivy rootfs [flags] ROOTDIR --ignorefile string specify .trivyignore file (default ".trivyignore") --include-deprecated-checks include deprecated checks (default true) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_sbom.md b/docs/docs/references/configuration/cli/trivy_sbom.md index a850591e1c93..133f8d8eaebe 100644 --- a/docs/docs/references/configuration/cli/trivy_sbom.md +++ b/docs/docs/references/configuration/cli/trivy_sbom.md @@ -24,7 +24,7 @@ trivy sbom [flags] SBOM_PATH --cache-ttl duration cache TTL when using redis as cache backend --compliance string compliance report to generate --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. - "comprehensive": Aims to detect more security findings at the cost of potential false positives. @@ -41,7 +41,7 @@ trivy sbom [flags] SBOM_PATH --ignore-unfixed display only fixed vulnerabilities --ignored-licenses strings specify a list of license to ignore --ignorefile string specify .trivyignore file (default ".trivyignore") - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --list-all-pkgs output all packages in the JSON report regardless of vulnerability --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies diff --git a/docs/docs/references/configuration/cli/trivy_server.md b/docs/docs/references/configuration/cli/trivy_server.md index 9edb2fe931f7..c1e74c45b09b 100644 --- a/docs/docs/references/configuration/cli/trivy_server.md +++ b/docs/docs/references/configuration/cli/trivy_server.md @@ -22,7 +22,7 @@ trivy server [flags] ``` --cache-backend string [EXPERIMENTAL] cache backend (e.g. redis://localhost:6379) (default "fs") --cache-ttl duration cache TTL when using redis as cache backend - --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --download-db-only download/update vulnerability database but don't run a scan --enable-modules strings [EXPERIMENTAL] module names to enable -h, --help help for server diff --git a/docs/docs/references/configuration/cli/trivy_vm.md b/docs/docs/references/configuration/cli/trivy_vm.md index 27af05423b65..11ba6758c3cc 100644 --- a/docs/docs/references/configuration/cli/trivy_vm.md +++ b/docs/docs/references/configuration/cli/trivy_vm.md @@ -27,7 +27,7 @@ trivy vm [flags] VM_IMAGE --compliance string compliance report to generate --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db from (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -52,7 +52,7 @@ trivy vm [flags] VM_IMAGE --ignore-unfixed display only fixed vulnerabilities --ignorefile string specify .trivyignore file (default ".trivyignore") --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db from (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) --list-all-pkgs output all packages in the JSON report regardless of vulnerability --misconfig-scanners strings comma-separated list of misconfig scanners to use for misconfiguration scanning (default [azure-arm,cloudformation,dockerfile,helm,kubernetes,terraform,terraformplan-json,terraformplan-snapshot]) --module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules") diff --git a/pkg/flag/db_flags.go b/pkg/flag/db_flags.go index f6ff24450305..a0261f98783d 100644 --- a/pkg/flag/db_flags.go +++ b/pkg/flag/db_flags.go @@ -54,13 +54,13 @@ var ( Name: "db-repository", ConfigName: "db.repository", Default: []string{db.DefaultGHCRRepository, db.DefaultECRRepository}, - Usage: "OCI repository(ies) to retrieve trivy-db from", + Usage: "OCI repository(ies) to retrieve trivy-db in order of priority", } JavaDBRepositoryFlag = Flag[[]string]{ Name: "java-db-repository", ConfigName: "db.java-repository", Default: []string{javadb.DefaultGHCRRepository, javadb.DefaultECRRepository}, - Usage: "OCI repository(ies) to retrieve trivy-java-db from", + Usage: "OCI repository(ies) to retrieve trivy-java-db in order of priority", } LightFlag = Flag[bool]{ Name: "light", From d75a5846290a44af6afe462b386efe179a298343 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Sat, 28 Sep 2024 15:51:50 +0600 Subject: [PATCH 4/9] chore: add successful download logs Signed-off-by: nikpivkin --- pkg/db/db.go | 2 ++ pkg/javadb/client.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pkg/db/db.go b/pkg/db/db.go index e271d872eab4..cc762eb9f9f3 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -241,6 +241,8 @@ func (c *Client) downloadDB(ctx context.Context, opt types.RegistryOptions, dst } continue } + + log.Info("DB successfully downloaded", log.String("repo", repo.String())) return nil } diff --git a/pkg/javadb/client.go b/pkg/javadb/client.go index a1485c125681..9057b4e3bd79 100644 --- a/pkg/javadb/client.go +++ b/pkg/javadb/client.go @@ -111,6 +111,8 @@ func (u *Updater) downloadDB() error { } continue } + + log.Info("DB successfully downloaded", log.String("repo", repo.String())) return nil } From b08004ab47c824c5de7cb04011ccfcdfd2e20365 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Mon, 30 Sep 2024 11:51:22 +0600 Subject: [PATCH 5/9] refactor: remove duplicate artifact downloading Signed-off-by: nikpivkin --- pkg/db/db.go | 35 ++++++++++++++++++++++------------- pkg/db/db_test.go | 2 +- pkg/oci/artifact.go | 4 ++++ 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/pkg/db/db.go b/pkg/db/db.go index cc762eb9f9f3..46c2a0584c5e 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -199,11 +199,8 @@ func (c *Client) updateDownloadedAt(ctx context.Context, dbDir string) error { } func (c *Client) initOCIArtifact(repository name.Reference, opt types.RegistryOptions) (*oci.Artifact, error) { - if c.artifact != nil { - return c.artifact, nil - } - art, err := oci.NewArtifact(repository.String(), c.quiet, opt) + // TODO: NewArtifact never returns an error if err != nil { var terr *transport.Error if errors.As(err, &terr) { @@ -221,28 +218,40 @@ func (c *Client) initOCIArtifact(repository name.Reference, opt types.RegistryOp return art, nil } -func (c *Client) downloadDB(ctx context.Context, opt types.RegistryOptions, dst string) error { - downloadOpt := oci.DownloadOption{MediaType: dbMediaType} +func (c *Client) initArtifacts(opt types.RegistryOptions) ([]*oci.Artifact, error) { if c.artifact != nil { - return c.artifact.Download(ctx, dst, downloadOpt) + return []*oci.Artifact{c.artifact}, nil } - for i, repo := range c.dbRepositories { + artifacts := make([]*oci.Artifact, 0, len(c.dbRepositories)) + + for _, repo := range c.dbRepositories { a, err := c.initOCIArtifact(repo, opt) if err != nil { - return err + return nil, err } + artifacts = append(artifacts, a) + } + return artifacts, nil +} + +func (c *Client) downloadDB(ctx context.Context, opt types.RegistryOptions, dst string) error { + arts, err := c.initArtifacts(opt) + if err != nil { + return err + } - log.Info("Downloading vulnerability DB...", log.String("repo", repo.String())) - if err := a.Download(ctx, dst, downloadOpt); err != nil { - log.Error("Failed to download DB", log.String("repo", repo.String()), log.Err(err)) + for i, art := range arts { + log.Info("Downloading vulnerability DB...", log.String("repo", art.Repository())) + if err := art.Download(ctx, dst, oci.DownloadOption{MediaType: dbMediaType}); err != nil { + log.Error("Failed to download DB", log.String("repo", art.Repository()), log.Err(err)) if i < len(c.dbRepositories)-1 { log.Info("Trying to download DB from other repository...") } continue } - log.Info("DB successfully downloaded", log.String("repo", repo.String())) + log.Info("DB successfully downloaded", log.String("repo", art.Repository())) return nil } diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go index ff5e36e2b194..4591964c5e75 100644 --- a/pkg/db/db_test.go +++ b/pkg/db/db_test.go @@ -159,7 +159,7 @@ func TestClient_Download(t *testing.T) { { name: "invalid gzip", input: "testdata/trivy.db", - wantErr: "unexpected EOF", + wantErr: "OCI artifact error: failed to download vulnerability DB from any source", }, } diff --git a/pkg/oci/artifact.go b/pkg/oci/artifact.go index 04d62ee15d36..968e487c40c8 100644 --- a/pkg/oci/artifact.go +++ b/pkg/oci/artifact.go @@ -207,3 +207,7 @@ func (a *Artifact) Digest(ctx context.Context) (string, error) { } return digest.String(), nil } + +func (a *Artifact) Repository() string { + return a.repository +} From 73c40122047cfbe4db5ca071c50ce105c021341f Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Mon, 30 Sep 2024 19:41:09 +0600 Subject: [PATCH 6/9] refactor: update download policy Signed-off-by: nikpivkin --- internal/dbtest/fake.go | 2 +- pkg/db/db.go | 46 +++++------------------ pkg/db/db_test.go | 2 +- pkg/fanal/artifact/image/remote_sbom.go | 3 +- pkg/javadb/client.go | 20 ++++------ pkg/module/command.go | 4 +- pkg/oci/artifact.go | 49 ++++++++++++++++++++++--- pkg/oci/artifact_test.go | 5 ++- pkg/policy/policy.go | 6 ++- pkg/policy/policy_test.go | 6 +-- 10 files changed, 76 insertions(+), 67 deletions(-) diff --git a/internal/dbtest/fake.go b/internal/dbtest/fake.go index 7943c72273e2..528c549bda7f 100644 --- a/internal/dbtest/fake.go +++ b/internal/dbtest/fake.go @@ -62,7 +62,7 @@ func NewFakeDB(t *testing.T, dbPath string, opts FakeDBOptions) *oci.Artifact { opt := ftypes.RegistryOptions{ Insecure: false, } - return oci.NewArtifact("dummy", true, opt, oci.WithImage(img)) + return oci.NewArtifact("dummy", opt, oci.WithImage(img)) } func ArchiveDir(t *testing.T, dir string) string { diff --git a/pkg/db/db.go b/pkg/db/db.go index 0747bffa04cb..7953c84a970c 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -2,14 +2,12 @@ package db import ( "context" - "errors" "fmt" "os" "path/filepath" "time" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/samber/lo" "golang.org/x/xerrors" @@ -19,7 +17,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/oci" - "github.com/aquasecurity/trivy/pkg/version/doc" ) const ( @@ -202,53 +199,28 @@ func (c *Client) initOCIArtifact(repository name.Reference, opt types.RegistryOp if c.artifact != nil { return c.artifact } - return oci.NewArtifact(repository.String(), c.quiet, opt) + return oci.NewArtifact(repository.String(), opt) } -func (c *Client) initArtifacts(opt types.RegistryOptions) ([]*oci.Artifact, error) { +func (c *Client) initArtifacts(opt types.RegistryOptions) []*oci.Artifact { if c.artifact != nil { - return []*oci.Artifact{c.artifact}, nil + return []*oci.Artifact{c.artifact} } artifacts := make([]*oci.Artifact, 0, len(c.dbRepositories)) for _, repo := range c.dbRepositories { artifacts = append(artifacts, c.initOCIArtifact(repo, opt)) } - return artifacts, nil + return artifacts } func (c *Client) downloadDB(ctx context.Context, opt types.RegistryOptions, dst string) error { - arts, err := c.initArtifacts(opt) - if err != nil { - return err + log.Info("Downloading vulnerability DB...") + downloadOpt := oci.DownloadOption{MediaType: dbMediaType, Quiet: c.quiet} + if err := oci.DownloadArtifact(ctx, c.initArtifacts(opt), dst, downloadOpt); err != nil { + return xerrors.Errorf("failed to download vulnerability DB: %w", err) } - - for i, art := range arts { - log.Info("Downloading vulnerability DB...", log.String("repo", art.Repository())) - if err := art.Download(ctx, dst, oci.DownloadOption{MediaType: dbMediaType}); err != nil { - var terr *transport.Error - if errors.As(err, &terr) { - for _, diagnostic := range terr.Errors { - // For better user experience - if diagnostic.Code == transport.DeniedErrorCode || diagnostic.Code == transport.UnauthorizedErrorCode { - // e.g. https://aquasecurity.github.io/trivy/latest/docs/references/troubleshooting/#db - log.Warnf("See %s", doc.URL("/docs/references/troubleshooting/", "db")) - break - } - } - } - log.Error("Failed to download DB", log.String("repo", art.Repository()), log.Err(err)) - if i < len(c.dbRepositories)-1 { - log.Info("Trying to download DB from other repository...") - } - continue - } - - log.Info("DB successfully downloaded", log.String("repo", art.Repository())) - return nil - } - - return xerrors.New("failed to download vulnerability DB from any source") + return nil } func (c *Client) ShowInfo() error { diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go index 4591964c5e75..f64c0dc4a5c6 100644 --- a/pkg/db/db_test.go +++ b/pkg/db/db_test.go @@ -159,7 +159,7 @@ func TestClient_Download(t *testing.T) { { name: "invalid gzip", input: "testdata/trivy.db", - wantErr: "OCI artifact error: failed to download vulnerability DB from any source", + wantErr: "OCI artifact error: failed to download vulnerability DB", }, } diff --git a/pkg/fanal/artifact/image/remote_sbom.go b/pkg/fanal/artifact/image/remote_sbom.go index e95fef23140c..d07d9cafe3ac 100644 --- a/pkg/fanal/artifact/image/remote_sbom.go +++ b/pkg/fanal/artifact/image/remote_sbom.go @@ -95,10 +95,11 @@ func (a Artifact) parseReferrer(ctx context.Context, repo string, desc v1.Descri defer os.RemoveAll(tmpDir) // Download SBOM to local filesystem - referrer := oci.NewArtifact(repoName, true, a.artifactOption.ImageOption.RegistryOptions) + referrer := oci.NewArtifact(repoName, a.artifactOption.ImageOption.RegistryOptions) if err = referrer.Download(ctx, tmpDir, oci.DownloadOption{ MediaType: desc.ArtifactType, Filename: fileName, + Quiet: true, }); err != nil { return artifact.Reference{}, xerrors.Errorf("SBOM download error: %w", err) } diff --git a/pkg/javadb/client.go b/pkg/javadb/client.go index 4aa74bce7052..f18239515fa0 100644 --- a/pkg/javadb/client.go +++ b/pkg/javadb/client.go @@ -11,6 +11,7 @@ import ( "time" "github.com/google/go-containerregistry/pkg/name" + "github.com/samber/lo" "golang.org/x/xerrors" "github.com/aquasecurity/trivy-java-db/pkg/db" @@ -97,19 +98,14 @@ func (u *Updater) isNewDB(meta db.Metadata) bool { } func (u *Updater) downloadDB() error { - for i, repo := range u.repos { - a := oci.NewArtifact(repo.String(), u.quiet, u.registryOption) - log.Info("Downloading the Java DB...", log.String("repository", repo.String())) - if err := a.Download(context.Background(), u.dbDir, oci.DownloadOption{MediaType: mediaType}); err != nil { - log.Error("Failed to download DB", log.String("repo", repo.String()), log.Err(err)) - if i < len(u.repos)-1 { - log.Info("Trying to download DB from other repository...") - } - continue - } + log.Info("Downloading Java DB...") - log.Info("DB successfully downloaded", log.String("repo", repo.String())) - return nil + artifacts := lo.Map(u.repos, func(r name.Reference, _ int) *oci.Artifact { + return oci.NewArtifact(r.String(), u.registryOption) + }) + downloadOpt := oci.DownloadOption{MediaType: mediaType, Quiet: u.quiet} + if err := oci.DownloadArtifact(context.Background(), artifacts, u.dbDir, downloadOpt); err != nil { + return xerrors.Errorf("failed to download vulnerability DB: %w", err) } return xerrors.New("failed to download Java DB from any source") diff --git a/pkg/module/command.go b/pkg/module/command.go index 2dd955aa0e4b..990d61c5824c 100644 --- a/pkg/module/command.go +++ b/pkg/module/command.go @@ -23,11 +23,11 @@ func Install(ctx context.Context, dir, repo string, quiet bool, opt types.Regist } log.Info("Installing the module from the repository...", log.String("repo", repo)) - art := oci.NewArtifact(repo, quiet, opt) + art := oci.NewArtifact(repo, opt) dst := filepath.Join(dir, ref.Context().Name()) log.Debug("Installing the module...", log.String("dst", dst)) - if err = art.Download(ctx, dst, oci.DownloadOption{MediaType: mediaType}); err != nil { + if err = art.Download(ctx, dst, oci.DownloadOption{MediaType: mediaType, Quiet: true}); err != nil { return xerrors.Errorf("module download error: %w", err) } diff --git a/pkg/oci/artifact.go b/pkg/oci/artifact.go index a4a725c4babd..84a9bfc8471d 100644 --- a/pkg/oci/artifact.go +++ b/pkg/oci/artifact.go @@ -2,6 +2,7 @@ package oci import ( "context" + "errors" "io" "os" "path/filepath" @@ -10,11 +11,14 @@ import ( "github.com/cheggaaa/pb/v3" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/downloader" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/remote" + "github.com/aquasecurity/trivy/pkg/version/doc" ) const ( @@ -48,7 +52,6 @@ func WithImage(img v1.Image) Option { type Artifact struct { m sync.Mutex repository string - quiet bool // For OCI registries types.RegistryOptions @@ -57,10 +60,9 @@ type Artifact struct { } // NewArtifact returns a new artifact -func NewArtifact(repo string, quiet bool, registryOpt types.RegistryOptions, opts ...Option) *Artifact { +func NewArtifact(repo string, registryOpt types.RegistryOptions, opts ...Option) *Artifact { art := &Artifact{ repository: repo, - quiet: quiet, RegistryOptions: registryOpt, } @@ -98,6 +100,7 @@ func (a *Artifact) populate(ctx context.Context, opt types.RegistryOptions) erro type DownloadOption struct { MediaType string // Accept any media type if not specified Filename string // Use the annotation if not specified + Quiet bool } func (a *Artifact) Download(ctx context.Context, dir string, opt DownloadOption) error { @@ -140,14 +143,14 @@ func (a *Artifact) Download(ctx context.Context, dir string, opt DownloadOption) return xerrors.Errorf("unacceptable media type: %s", string(layerMediaType)) } - if err = a.download(ctx, layer, fileName, dir); err != nil { + if err = a.download(ctx, layer, fileName, dir, opt.Quiet); err != nil { return xerrors.Errorf("oci download error: %w", err) } return nil } -func (a *Artifact) download(ctx context.Context, layer v1.Layer, fileName, dir string) error { +func (a *Artifact) download(ctx context.Context, layer v1.Layer, fileName, dir string, quiet bool) error { size, err := layer.Size() if err != nil { return xerrors.Errorf("size error: %w", err) @@ -161,7 +164,7 @@ func (a *Artifact) download(ctx context.Context, layer v1.Layer, fileName, dir s // Show progress bar bar := pb.Full.Start64(size) - if a.quiet { + if quiet { bar.SetWriter(io.Discard) } pr := bar.NewProxyReader(rc) @@ -211,3 +214,37 @@ func (a *Artifact) Digest(ctx context.Context) (string, error) { func (a *Artifact) Repository() string { return a.repository } + +func DownloadArtifact(ctx context.Context, artifacts []*Artifact, dst string, opt DownloadOption) error { + for i, art := range artifacts { + log.Info("Downloading artifact...", log.String("repo", art.Repository())) + err := art.Download(ctx, dst, opt) + if err == nil { + log.Info("Artifact successfully downloaded", log.String("repo", art.Repository())) + return nil + } + + log.Error("Failed to download artifact", log.String("repo", art.Repository()), log.Err(err)) + + var terr *transport.Error + if errors.As(err, &terr) { + for _, diagnostic := range terr.Errors { + // For better user experience + if diagnostic.Code == transport.DeniedErrorCode || diagnostic.Code == transport.UnauthorizedErrorCode { + // e.g. https://aquasecurity.github.io/trivy/latest/docs/references/troubleshooting/#db + log.Warnf("See %s", doc.URL("/docs/references/troubleshooting/", "db")) + break + } + } + + // try the following artifact only if a temporary error occurs + if terr.Temporary() && i < len(artifacts)-1 { + log.Info("Trying to download artifact from other repository...") + continue + } + } + return xerrors.Errorf("failed to download artifact from %s", art.Repository()) + } + + return xerrors.New("failed to download artifact from any source") +} diff --git a/pkg/oci/artifact_test.go b/pkg/oci/artifact_test.go index 7cc8cd19d3c1..3872c7bc9ab5 100644 --- a/pkg/oci/artifact_test.go +++ b/pkg/oci/artifact_test.go @@ -81,7 +81,7 @@ func TestArtifact_Download(t *testing.T) { layersReturns: layersReturns{ layers: []v1.Layer{txtLayer}, }, - wantErr: "unexpected EOF", + wantErr: "failed to download vulnerability DB: failed to download artifact", }, { name: "sad: media type doesn't match", @@ -116,9 +116,10 @@ func TestArtifact_Download(t *testing.T) { }, }, nil) - artifact := oci.NewArtifact("repo", true, ftypes.RegistryOptions{}, oci.WithImage(img)) + artifact := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img)) err = artifact.Download(context.Background(), tempDir, oci.DownloadOption{ MediaType: tt.mediaType, + Quiet: true, }) if tt.wantErr != "" { assert.ErrorContains(t, err, tt.wantErr) diff --git a/pkg/policy/policy.go b/pkg/policy/policy.go index c8c05cf067c7..2dbe2fc1b89a 100644 --- a/pkg/policy/policy.go +++ b/pkg/policy/policy.go @@ -92,7 +92,7 @@ func NewClient(cacheDir string, quiet bool, checkBundleRepo string, opts ...Opti func (c *Client) populateOCIArtifact(registryOpts types.RegistryOptions) { if c.artifact == nil { log.Debug("Loading check bundle", log.String("repository", c.checkBundleRepo)) - c.artifact = oci.NewArtifact(c.checkBundleRepo, c.quiet, registryOpts) + c.artifact = oci.NewArtifact(c.checkBundleRepo, registryOpts) } } @@ -101,7 +101,9 @@ func (c *Client) DownloadBuiltinPolicies(ctx context.Context, registryOpts types c.populateOCIArtifact(registryOpts) dst := c.contentDir() - if err := c.artifact.Download(ctx, dst, oci.DownloadOption{MediaType: policyMediaType}); err != nil { + if err := c.artifact.Download(ctx, dst, + oci.DownloadOption{MediaType: policyMediaType, Quiet: c.quiet}, + ); err != nil { return xerrors.Errorf("download error: %w", err) } diff --git a/pkg/policy/policy_test.go b/pkg/policy/policy_test.go index 4dbd6b9a7558..f6875eb1c9cb 100644 --- a/pkg/policy/policy_test.go +++ b/pkg/policy/policy_test.go @@ -116,7 +116,7 @@ func TestClient_LoadBuiltinPolicies(t *testing.T) { }, nil) // Mock OCI artifact - art := oci.NewArtifact("repo", true, ftypes.RegistryOptions{}, oci.WithImage(img)) + art := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img)) c, err := policy.NewClient(tt.cacheDir, true, "", policy.WithOCIArtifact(art)) require.NoError(t, err) @@ -255,7 +255,7 @@ func TestClient_NeedsUpdate(t *testing.T) { require.NoError(t, err) } - art := oci.NewArtifact("repo", true, ftypes.RegistryOptions{}, oci.WithImage(img)) + art := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img)) c, err := policy.NewClient(tmpDir, true, "", policy.WithOCIArtifact(art), policy.WithClock(tt.clock)) require.NoError(t, err) @@ -357,7 +357,7 @@ func TestClient_DownloadBuiltinPolicies(t *testing.T) { }, nil) // Mock OCI artifact - art := oci.NewArtifact("repo", true, ftypes.RegistryOptions{}, oci.WithImage(img)) + art := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img)) c, err := policy.NewClient(tempDir, true, "", policy.WithClock(tt.clock), policy.WithOCIArtifact(art)) require.NoError(t, err) From 9fe203fe05c553f8498f923448adf93c31b4b6f9 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Mon, 30 Sep 2024 19:43:14 +0600 Subject: [PATCH 7/9] cli: remove default ECR repository Signed-off-by: nikpivkin --- docs/docs/references/configuration/cli/trivy_filesystem.md | 4 ++-- docs/docs/references/configuration/cli/trivy_image.md | 4 ++-- docs/docs/references/configuration/cli/trivy_kubernetes.md | 4 ++-- docs/docs/references/configuration/cli/trivy_repository.md | 4 ++-- docs/docs/references/configuration/cli/trivy_rootfs.md | 4 ++-- docs/docs/references/configuration/cli/trivy_sbom.md | 4 ++-- docs/docs/references/configuration/cli/trivy_server.md | 2 +- docs/docs/references/configuration/cli/trivy_vm.md | 4 ++-- docs/docs/references/configuration/config-file.md | 2 -- pkg/db/db.go | 5 ----- pkg/flag/db_flags.go | 4 ++-- pkg/javadb/client.go | 2 -- 12 files changed, 17 insertions(+), 26 deletions(-) diff --git a/docs/docs/references/configuration/cli/trivy_filesystem.md b/docs/docs/references/configuration/cli/trivy_filesystem.md index f907b74fce18..9944a2d618ff 100644 --- a/docs/docs/references/configuration/cli/trivy_filesystem.md +++ b/docs/docs/references/configuration/cli/trivy_filesystem.md @@ -29,7 +29,7 @@ trivy filesystem [flags] PATH --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -56,7 +56,7 @@ trivy filesystem [flags] PATH --include-deprecated-checks include deprecated checks (default true) --include-dev-deps include development dependencies in the report (supported: npm, yarn) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_image.md b/docs/docs/references/configuration/cli/trivy_image.md index 919570031ae0..b63daffbf4ad 100644 --- a/docs/docs/references/configuration/cli/trivy_image.md +++ b/docs/docs/references/configuration/cli/trivy_image.md @@ -43,7 +43,7 @@ trivy image [flags] IMAGE_NAME --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -74,7 +74,7 @@ trivy image [flags] IMAGE_NAME --include-deprecated-checks include deprecated checks (default true) --include-non-failures include successes and exceptions, available with '--scanners misconfig' --input string input file path instead of image name - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_kubernetes.md b/docs/docs/references/configuration/cli/trivy_kubernetes.md index 9dae1722bbe2..35190b1b82b5 100644 --- a/docs/docs/references/configuration/cli/trivy_kubernetes.md +++ b/docs/docs/references/configuration/cli/trivy_kubernetes.md @@ -38,7 +38,7 @@ trivy kubernetes [flags] [CONTEXT] --config-check strings specify the paths to the Rego check files or to the directories containing them, applying config files --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking - --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -70,7 +70,7 @@ trivy kubernetes [flags] [CONTEXT] --include-kinds strings indicate the kinds included in scanning (example: node) --include-namespaces strings indicate the namespaces included in scanning (example: kube-system) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --k8s-version string specify k8s version to validate outdated api by it (example: 1.21.0) --kubeconfig string specify the kubeconfig file path to use --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_repository.md b/docs/docs/references/configuration/cli/trivy_repository.md index 8a4bbd35f56e..59e4f2c08cb0 100644 --- a/docs/docs/references/configuration/cli/trivy_repository.md +++ b/docs/docs/references/configuration/cli/trivy_repository.md @@ -29,7 +29,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -56,7 +56,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --include-deprecated-checks include deprecated checks (default true) --include-dev-deps include development dependencies in the report (supported: npm, yarn) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_rootfs.md b/docs/docs/references/configuration/cli/trivy_rootfs.md index b7ce12fc32fd..6022f4589446 100644 --- a/docs/docs/references/configuration/cli/trivy_rootfs.md +++ b/docs/docs/references/configuration/cli/trivy_rootfs.md @@ -31,7 +31,7 @@ trivy rootfs [flags] ROOTDIR --config-data strings specify paths from which data for the Rego checks will be recursively loaded --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -58,7 +58,7 @@ trivy rootfs [flags] ROOTDIR --ignorefile string specify .trivyignore file (default ".trivyignore") --include-deprecated-checks include deprecated checks (default true) --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --license-confidence-level float specify license classifier's confidence level (default 0.9) --license-full eagerly look for licenses in source code headers and license files --list-all-pkgs output all packages in the JSON report regardless of vulnerability diff --git a/docs/docs/references/configuration/cli/trivy_sbom.md b/docs/docs/references/configuration/cli/trivy_sbom.md index 133f8d8eaebe..f09453845518 100644 --- a/docs/docs/references/configuration/cli/trivy_sbom.md +++ b/docs/docs/references/configuration/cli/trivy_sbom.md @@ -24,7 +24,7 @@ trivy sbom [flags] SBOM_PATH --cache-ttl duration cache TTL when using redis as cache backend --compliance string compliance report to generate --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. - "comprehensive": Aims to detect more security findings at the cost of potential false positives. @@ -41,7 +41,7 @@ trivy sbom [flags] SBOM_PATH --ignore-unfixed display only fixed vulnerabilities --ignored-licenses strings specify a list of license to ignore --ignorefile string specify .trivyignore file (default ".trivyignore") - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --list-all-pkgs output all packages in the JSON report regardless of vulnerability --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies diff --git a/docs/docs/references/configuration/cli/trivy_server.md b/docs/docs/references/configuration/cli/trivy_server.md index c1e74c45b09b..b7ada8f2c19b 100644 --- a/docs/docs/references/configuration/cli/trivy_server.md +++ b/docs/docs/references/configuration/cli/trivy_server.md @@ -22,7 +22,7 @@ trivy server [flags] ``` --cache-backend string [EXPERIMENTAL] cache backend (e.g. redis://localhost:6379) (default "fs") --cache-ttl duration cache TTL when using redis as cache backend - --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --download-db-only download/update vulnerability database but don't run a scan --enable-modules strings [EXPERIMENTAL] module names to enable -h, --help help for server diff --git a/docs/docs/references/configuration/cli/trivy_vm.md b/docs/docs/references/configuration/cli/trivy_vm.md index 11ba6758c3cc..9b7f6a6fef43 100644 --- a/docs/docs/references/configuration/cli/trivy_vm.md +++ b/docs/docs/references/configuration/cli/trivy_vm.md @@ -27,7 +27,7 @@ trivy vm [flags] VM_IMAGE --compliance string compliance report to generate --config-file-schemas strings specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking --custom-headers strings custom headers in client mode - --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2]) + --db-repository strings OCI repository(ies) to retrieve trivy-db in order of priority (default [ghcr.io/aquasecurity/trivy-db:2]) --dependency-tree [EXPERIMENTAL] show dependency origin tree of vulnerable packages --detection-priority string specify the detection priority: - "precise": Prioritizes precise by minimizing false positives. @@ -52,7 +52,7 @@ trivy vm [flags] VM_IMAGE --ignore-unfixed display only fixed vulnerabilities --ignorefile string specify .trivyignore file (default ".trivyignore") --include-non-failures include successes and exceptions, available with '--scanners misconfig' - --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1,public.ecr.aws/aquasecurity/trivy-java-db:1]) + --java-db-repository strings OCI repository(ies) to retrieve trivy-java-db in order of priority (default [ghcr.io/aquasecurity/trivy-java-db:1]) --list-all-pkgs output all packages in the JSON report regardless of vulnerability --misconfig-scanners strings comma-separated list of misconfig scanners to use for misconfiguration scanning (default [azure-arm,cloudformation,dockerfile,helm,kubernetes,terraform,terraformplan-json,terraformplan-snapshot]) --module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules") diff --git a/docs/docs/references/configuration/config-file.md b/docs/docs/references/configuration/config-file.md index 5bb27e589057..ae3a017fda05 100644 --- a/docs/docs/references/configuration/config-file.md +++ b/docs/docs/references/configuration/config-file.md @@ -106,7 +106,6 @@ db: # Same as '--java-db-repository' java-repository: - ghcr.io/aquasecurity/trivy-java-db:1 - - public.ecr.aws/aquasecurity/trivy-java-db:1 # Same as '--skip-java-db-update' java-skip-update: false @@ -117,7 +116,6 @@ db: # Same as '--db-repository' repository: - ghcr.io/aquasecurity/trivy-db:2 - - public.ecr.aws/aquasecurity/trivy-db:2 # Same as '--skip-db-update' skip-update: false diff --git a/pkg/db/db.go b/pkg/db/db.go index 7953c84a970c..affbf6070839 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -29,10 +29,6 @@ var ( DefaultGHCRRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-db", db.SchemaVersion) defaultGHCRRepository = lo.Must(name.NewTag(DefaultGHCRRepository)) - // Amazon Elastic Container Registry - DefaultECRRepository = fmt.Sprintf("%s:%d", "public.ecr.aws/aquasecurity/trivy-db", db.SchemaVersion) - defaultECRRepository = lo.Must(name.NewTag(DefaultECRRepository)) - Init = db.Init Close = db.Close Path = db.Path @@ -78,7 +74,6 @@ func NewClient(dbDir string, quiet bool, opts ...Option) *Client { o := &options{ dbRepositories: []name.Reference{ defaultGHCRRepository, - defaultECRRepository, }, } diff --git a/pkg/flag/db_flags.go b/pkg/flag/db_flags.go index a0261f98783d..df0d6c6f5194 100644 --- a/pkg/flag/db_flags.go +++ b/pkg/flag/db_flags.go @@ -53,13 +53,13 @@ var ( DBRepositoryFlag = Flag[[]string]{ Name: "db-repository", ConfigName: "db.repository", - Default: []string{db.DefaultGHCRRepository, db.DefaultECRRepository}, + Default: []string{db.DefaultGHCRRepository}, Usage: "OCI repository(ies) to retrieve trivy-db in order of priority", } JavaDBRepositoryFlag = Flag[[]string]{ Name: "java-db-repository", ConfigName: "db.java-repository", - Default: []string{javadb.DefaultGHCRRepository, javadb.DefaultECRRepository}, + Default: []string{javadb.DefaultGHCRRepository}, Usage: "OCI repository(ies) to retrieve trivy-java-db in order of priority", } LightFlag = Flag[bool]{ diff --git a/pkg/javadb/client.go b/pkg/javadb/client.go index f18239515fa0..44a2115e129b 100644 --- a/pkg/javadb/client.go +++ b/pkg/javadb/client.go @@ -30,8 +30,6 @@ const ( var ( // GitHub Container Registry DefaultGHCRRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-java-db", SchemaVersion) - // Amazon Elastic Container Registry - DefaultECRRepository = fmt.Sprintf("%s:%d", "public.ecr.aws/aquasecurity/trivy-java-db", SchemaVersion) ) var updater *Updater From 179e628572faebb97bb5ac6364c5e619ab811337 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Tue, 1 Oct 2024 11:51:05 +0600 Subject: [PATCH 8/9] simplify download artifact Signed-off-by: nikpivkin --- pkg/oci/artifact.go | 42 +++++++++++++++++++++++----------------- pkg/oci/artifact_test.go | 2 +- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/pkg/oci/artifact.go b/pkg/oci/artifact.go index 84a9bfc8471d..7e0f42b6d0e0 100644 --- a/pkg/oci/artifact.go +++ b/pkg/oci/artifact.go @@ -224,27 +224,33 @@ func DownloadArtifact(ctx context.Context, artifacts []*Artifact, dst string, op return nil } + if !shouldTryOtherRepo(err) { + return xerrors.Errorf("failed to download artifact from %s: %w", art.Repository(), err) + } log.Error("Failed to download artifact", log.String("repo", art.Repository()), log.Err(err)) - - var terr *transport.Error - if errors.As(err, &terr) { - for _, diagnostic := range terr.Errors { - // For better user experience - if diagnostic.Code == transport.DeniedErrorCode || diagnostic.Code == transport.UnauthorizedErrorCode { - // e.g. https://aquasecurity.github.io/trivy/latest/docs/references/troubleshooting/#db - log.Warnf("See %s", doc.URL("/docs/references/troubleshooting/", "db")) - break - } - } - - // try the following artifact only if a temporary error occurs - if terr.Temporary() && i < len(artifacts)-1 { - log.Info("Trying to download artifact from other repository...") - continue - } + if i < len(artifacts)-1 { + log.Info("Trying to download artifact from other repository...") } - return xerrors.Errorf("failed to download artifact from %s", art.Repository()) } return xerrors.New("failed to download artifact from any source") } + +func shouldTryOtherRepo(err error) bool { + var terr *transport.Error + if !errors.As(err, &terr) { + return false + } + + for _, diagnostic := range terr.Errors { + // For better user experience + if diagnostic.Code == transport.DeniedErrorCode || diagnostic.Code == transport.UnauthorizedErrorCode { + // e.g. https://aquasecurity.github.io/trivy/latest/docs/references/troubleshooting/#db + log.Warnf("See %s", doc.URL("/docs/references/troubleshooting/", "db")) + break + } + } + + // try the following artifact only if a temporary error occurs + return terr.Temporary() +} diff --git a/pkg/oci/artifact_test.go b/pkg/oci/artifact_test.go index 3872c7bc9ab5..a8ce6e542641 100644 --- a/pkg/oci/artifact_test.go +++ b/pkg/oci/artifact_test.go @@ -81,7 +81,7 @@ func TestArtifact_Download(t *testing.T) { layersReturns: layersReturns{ layers: []v1.Layer{txtLayer}, }, - wantErr: "failed to download vulnerability DB: failed to download artifact", + wantErr: "unexpected EOF", }, { name: "sad: media type doesn't match", From 8a960b3a826d3617d88692a8247b81f6bf0ee82b Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Tue, 1 Oct 2024 18:12:40 +0600 Subject: [PATCH 9/9] refactor: use Artifacts to align with Artifact Signed-off-by: nikpivkin --- pkg/db/db.go | 20 ++++---------------- pkg/javadb/client.go | 7 ++----- pkg/module/command.go | 2 +- pkg/oci/artifact.go | 26 +++++++++++++++++--------- 4 files changed, 24 insertions(+), 31 deletions(-) diff --git a/pkg/db/db.go b/pkg/db/db.go index affbf6070839..f7ceb09af4c9 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -190,29 +190,17 @@ func (c *Client) updateDownloadedAt(ctx context.Context, dbDir string) error { return nil } -func (c *Client) initOCIArtifact(repository name.Reference, opt types.RegistryOptions) *oci.Artifact { +func (c *Client) initArtifacts(opt types.RegistryOptions) oci.Artifacts { if c.artifact != nil { - return c.artifact + return oci.Artifacts{c.artifact} } - return oci.NewArtifact(repository.String(), opt) -} - -func (c *Client) initArtifacts(opt types.RegistryOptions) []*oci.Artifact { - if c.artifact != nil { - return []*oci.Artifact{c.artifact} - } - - artifacts := make([]*oci.Artifact, 0, len(c.dbRepositories)) - for _, repo := range c.dbRepositories { - artifacts = append(artifacts, c.initOCIArtifact(repo, opt)) - } - return artifacts + return oci.NewArtifacts(c.dbRepositories, opt) } func (c *Client) downloadDB(ctx context.Context, opt types.RegistryOptions, dst string) error { log.Info("Downloading vulnerability DB...") downloadOpt := oci.DownloadOption{MediaType: dbMediaType, Quiet: c.quiet} - if err := oci.DownloadArtifact(ctx, c.initArtifacts(opt), dst, downloadOpt); err != nil { + if err := c.initArtifacts(opt).Download(ctx, dst, downloadOpt); err != nil { return xerrors.Errorf("failed to download vulnerability DB: %w", err) } return nil diff --git a/pkg/javadb/client.go b/pkg/javadb/client.go index 44a2115e129b..8c7aa81df9a3 100644 --- a/pkg/javadb/client.go +++ b/pkg/javadb/client.go @@ -11,7 +11,6 @@ import ( "time" "github.com/google/go-containerregistry/pkg/name" - "github.com/samber/lo" "golang.org/x/xerrors" "github.com/aquasecurity/trivy-java-db/pkg/db" @@ -98,11 +97,9 @@ func (u *Updater) isNewDB(meta db.Metadata) bool { func (u *Updater) downloadDB() error { log.Info("Downloading Java DB...") - artifacts := lo.Map(u.repos, func(r name.Reference, _ int) *oci.Artifact { - return oci.NewArtifact(r.String(), u.registryOption) - }) + artifacts := oci.NewArtifacts(u.repos, u.registryOption) downloadOpt := oci.DownloadOption{MediaType: mediaType, Quiet: u.quiet} - if err := oci.DownloadArtifact(context.Background(), artifacts, u.dbDir, downloadOpt); err != nil { + if err := artifacts.Download(context.Background(), u.dbDir, downloadOpt); err != nil { return xerrors.Errorf("failed to download vulnerability DB: %w", err) } diff --git a/pkg/module/command.go b/pkg/module/command.go index 990d61c5824c..a74da8384c3f 100644 --- a/pkg/module/command.go +++ b/pkg/module/command.go @@ -27,7 +27,7 @@ func Install(ctx context.Context, dir, repo string, quiet bool, opt types.Regist dst := filepath.Join(dir, ref.Context().Name()) log.Debug("Installing the module...", log.String("dst", dst)) - if err = art.Download(ctx, dst, oci.DownloadOption{MediaType: mediaType, Quiet: true}); err != nil { + if err = art.Download(ctx, dst, oci.DownloadOption{MediaType: mediaType, Quiet: quiet}); err != nil { return xerrors.Errorf("module download error: %w", err) } diff --git a/pkg/oci/artifact.go b/pkg/oci/artifact.go index 7e0f42b6d0e0..345149892a4e 100644 --- a/pkg/oci/artifact.go +++ b/pkg/oci/artifact.go @@ -12,6 +12,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/samber/lo" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/downloader" @@ -211,24 +212,31 @@ func (a *Artifact) Digest(ctx context.Context) (string, error) { return digest.String(), nil } -func (a *Artifact) Repository() string { - return a.repository +type Artifacts []*Artifact + +// NewArtifacts returns a slice of artifacts. +func NewArtifacts(repos []name.Reference, opt types.RegistryOptions, opts ...Option) Artifacts { + return lo.Map(repos, func(r name.Reference, _ int) *Artifact { + return NewArtifact(r.String(), opt, opts...) + }) } -func DownloadArtifact(ctx context.Context, artifacts []*Artifact, dst string, opt DownloadOption) error { - for i, art := range artifacts { - log.Info("Downloading artifact...", log.String("repo", art.Repository())) +// Download downloads artifacts until one of them succeeds. +// Attempts to download next artifact if the first one fails due to a temporary error. +func (a Artifacts) Download(ctx context.Context, dst string, opt DownloadOption) error { + for i, art := range a { + log.Info("Downloading artifact...", log.String("repo", art.repository)) err := art.Download(ctx, dst, opt) if err == nil { - log.Info("Artifact successfully downloaded", log.String("repo", art.Repository())) + log.Info("Artifact successfully downloaded", log.String("repo", art.repository)) return nil } if !shouldTryOtherRepo(err) { - return xerrors.Errorf("failed to download artifact from %s: %w", art.Repository(), err) + return xerrors.Errorf("failed to download artifact from %s: %w", art.repository, err) } - log.Error("Failed to download artifact", log.String("repo", art.Repository()), log.Err(err)) - if i < len(artifacts)-1 { + log.Error("Failed to download artifact", log.String("repo", art.repository), log.Err(err)) + if i < len(a)-1 { log.Info("Trying to download artifact from other repository...") } }