feat(misconf): scanning support for YAML and JSON (#7311)

Signed-off-by: nikpivkin <[email protected]>

docs/docs/coverage/iac/index.md

@@ -17,6 +17,9 @@ Trivy scans Infrastructure as Code (IaC) files for
 | [CloudFormation](cloudformation.md) | \*.yml, \*.yaml, \*.json                      |
 | [Azure ARM Template](azure-arm.md)  | \*.json                                       |
 | [Helm](helm.md)                     | \*.yaml, \*.tpl, \*.tar.gz, etc.              |
+| [YAML][json-and-yaml]               | \*.yaml, \*.yml                               |
+| [JSON][json-and-yaml]               | \*.json                                       |
 
 [misconf]: ../../scanner/misconfiguration/index.md
 [secret]: ../../scanner/secret.md
+[json-and-yaml]: ../../scanner/misconfiguration/index.md#scan-arbitrary-json-and-yaml-configurations

docs/docs/references/configuration/cli/trivy_config.md

@@ -17,6 +17,7 @@ trivy config [flags] DIR
       --compliance string                 compliance report to generate
       --config-check strings              specify the paths to the Rego check files or to the directories containing them, applying config files
       --config-data strings               specify paths from which data for the Rego checks will be recursively loaded
+      --config-file-schemas strings       specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking
       --enable-modules strings            [EXPERIMENTAL] module names to enable
       --exit-code int                     specify exit code when any security issues are found
       --file-patterns strings             specify config file patterns

docs/docs/references/configuration/cli/trivy_filesystem.md

@@ -27,6 +27,7 @@ trivy filesystem [flags] PATH
       --compliance string                 compliance report to generate
       --config-check strings              specify the paths to the Rego check files or to the directories containing them, applying config files
       --config-data strings               specify paths from which data for the Rego checks will be recursively loaded
+      --config-file-schemas strings       specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking
       --custom-headers strings            custom headers in client mode
       --db-repository string              OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2")
       --dependency-tree                   [EXPERIMENTAL] show dependency origin tree of vulnerable packages

docs/docs/references/configuration/cli/trivy_image.md

@@ -41,6 +41,7 @@ trivy image [flags] IMAGE_NAME
       --compliance string                 compliance report to generate (docker-cis-1.6.0)
       --config-check strings              specify the paths to the Rego check files or to the directories containing them, applying config files
       --config-data strings               specify paths from which data for the Rego checks will be recursively loaded
+      --config-file-schemas strings       specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking
       --custom-headers strings            custom headers in client mode
       --db-repository string              OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2")
       --dependency-tree                   [EXPERIMENTAL] show dependency origin tree of vulnerable packages

docs/docs/references/configuration/cli/trivy_kubernetes.md

@@ -37,6 +37,7 @@ trivy kubernetes [flags] [CONTEXT]
       --compliance string                 compliance report to generate (k8s-nsa-1.0,k8s-cis-1.23,eks-cis-1.4,rke2-cis-1.24,k8s-pss-baseline-0.1,k8s-pss-restricted-0.1)
       --config-check strings              specify the paths to the Rego check files or to the directories containing them, applying config files
       --config-data strings               specify paths from which data for the Rego checks will be recursively loaded
+      --config-file-schemas strings       specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking
       --db-repository string              OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2")
       --dependency-tree                   [EXPERIMENTAL] show dependency origin tree of vulnerable packages
       --detection-priority string         specify the detection priority:

docs/docs/references/configuration/cli/trivy_repository.md

@@ -27,6 +27,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL)
       --commit string                     pass the commit hash to be scanned
       --config-check strings              specify the paths to the Rego check files or to the directories containing them, applying config files
       --config-data strings               specify paths from which data for the Rego checks will be recursively loaded
+      --config-file-schemas strings       specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking
       --custom-headers strings            custom headers in client mode
       --db-repository string              OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2")
       --dependency-tree                   [EXPERIMENTAL] show dependency origin tree of vulnerable packages

docs/docs/references/configuration/cli/trivy_rootfs.md

@@ -29,6 +29,7 @@ trivy rootfs [flags] ROOTDIR
       --checks-bundle-repository string   OCI registry URL to retrieve checks bundle from (default "ghcr.io/aquasecurity/trivy-checks:0")
       --config-check strings              specify the paths to the Rego check files or to the directories containing them, applying config files
       --config-data strings               specify paths from which data for the Rego checks will be recursively loaded
+      --config-file-schemas strings       specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking
       --custom-headers strings            custom headers in client mode
       --db-repository string              OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2")
       --dependency-tree                   [EXPERIMENTAL] show dependency origin tree of vulnerable packages

docs/docs/references/configuration/cli/trivy_vm.md

@@ -25,6 +25,7 @@ trivy vm [flags] VM_IMAGE
       --cache-ttl duration                cache TTL when using redis as cache backend
       --checks-bundle-repository string   OCI registry URL to retrieve checks bundle from (default "ghcr.io/aquasecurity/trivy-checks:0")
       --compliance string                 compliance report to generate
+      --config-file-schemas strings       specify paths to JSON configuration file schemas to determine that a file matches some configuration and pass the schema to Rego checks for type checking
       --custom-headers strings            custom headers in client mode
       --db-repository string              OCI repository to retrieve trivy-db from (default "ghcr.io/aquasecurity/trivy-db:2")
       --dependency-tree                   [EXPERIMENTAL] show dependency origin tree of vulnerable packages

docs/docs/references/configuration/config-file.md

@@ -386,6 +386,9 @@ misconfiguration:
     # Same as '--cf-params'
     params: []
 
+  # Same as '--config-file-schemas'
+  config-file-schemas: []
+
   helm:
     # Same as '--helm-api-versions'
     api-versions: []

docs/docs/scanner/misconfiguration/index.md

@@ -107,7 +107,7 @@ $ trivy config --severity HIGH,CRITICAL ./iac
 <details>
 <summary>Result</summary>
 
-```
+```bash
 2022-06-06T11:01:21.142+0100	INFO	Detected config files: 8
 
 Dockerfile (dockerfile)
@@ -343,6 +343,61 @@ You can load checks bundle as OCI Image from a Container Registry using the `--c
 trivy config --checks-bundle-repository myregistry.local/mychecks --namespaces user myapp
 ```
 
+
+### Scan arbitrary JSON and YAML configurations
+By default, scanning JSON and YAML configurations is disabled, since Trivy does not contain built-in checks for these configurations. To enable it, pass the `json` or `yaml` to `--misconfig-scanners`. See [Enabling a subset of misconfiguration scanners](#enabling-a-subset-of-misconfiguration-scanners) for more information. Trivy will pass each file as is to the checks input.
+
+
+!!! example
+```bash
+$ cat iac/serverless.yaml
+service: serverless-rest-api-with-pynamodb
+
+frameworkVersion: ">=2.24.0"
+
+plugins:
+  - serverless-python-requirements
+...
+
+$ cat serverless.rego
+# METADATA
+# title: Serverless Framework service name not starting with "aws-"
+# description: Ensure that Serverless Framework service names start with "aws-"
+# schemas:
+#   - input: schema["serverless-schema"]
+# custom:
+#   id: SF001
+#   severity: LOW
+package user.serverless001
+
+deny[res] {
+    not startswith(input.service, "aws-")
+    res := result.new(
+        sprintf("Service name %q is not allowed", [input.service]),
+        input.service
+    )
+}
+
+$ trivy config --misconfig-scanners=json,yaml --config-check ./serverless.rego --check-namespaces user ./iac
+serverless.yaml (yaml)
+
+Tests: 4 (SUCCESSES: 3, FAILURES: 1, EXCEPTIONS: 0)
+Failures: 1 (UNKNOWN: 0, LOW: 1, MEDIUM: 0, HIGH: 0, CRITICAL: 0)
+
+LOW: Service name "serverless-rest-api-with-pynamodb" is not allowed
+═════════════════════════════════════════════════════════════════════════════════════════════════════════
+Ensure that Serverless Framework service names start with "aws-"
+```
+
+You can also pass schemas using the `config-file-schemas` flag. Trivy will use these schemas for file filtering and type checking in Rego checks. If the file does not match any of the passed schemas, it will be ignored.
+
+!!! example
+```bash
+$ trivy config --misconfig-scanners=json,yaml --config-check ./serverless.rego --check-namespaces user --config-file-schemas ./serverless-schema.json ./iac
+```
+
+If the schema is specified in the check metadata and is in the directory specified in the `--config-check` argument, it will be automatically loaded as specified [here](./custom/schema.md#custom-checks-with-custom-schemas), and will only be used for type checking in Rego.
+
 ### Passing custom data
 You can pass directories including your custom data through `--data` option.
 This can be repeated for specifying multiple directories.
@@ -363,12 +418,12 @@ This can be repeated for specifying multiple packages.
 trivy config --config-check ./my-check --namespaces main --namespaces user ./configs
 ```
 
-### Private terraform registries
-Trivy can download terraform code from private registries.
+### Private Terraform registries
+Trivy can download Terraform code from private registries.
 To pass credentials you must use the `TF_TOKEN_` environment variables.
 You cannot use a `.terraformrc` or `terraform.rc` file, these are not supported by trivy yet.
 
-From the terraform [docs](https://developer.hashicorp.com/terraform/cli/config/config-file#environment-variable-credentials):
+From the Terraform [docs](https://developer.hashicorp.com/terraform/cli/config/config-file#environment-variable-credentials):
 
 > Environment variable names should have the prefix TF_TOKEN_ added to the domain name, with periods encoded as underscores.
 > For example, the value of a variable named `TF_TOKEN_app_terraform_io` will be used as a bearer authorization token when the CLI makes service requests to the hostname `app.terraform.io`.

pkg/commands/artifact/run.go

@@ -501,43 +501,13 @@ func (r *runner) initScannerConfig(opts flag.Options) (ScannerConfig, types.Scan
 		log.WithPrefix(log.PrefixVulnerability).Info("Vulnerability scanning is enabled")
 	}
 
-	// ScannerOption is filled only when config scanning is enabled.
+	// Misconfig ScannerOption is filled only when config scanning is enabled.
 	var configScannerOptions misconf.ScannerOption
 	if opts.Scanners.Enabled(types.MisconfigScanner) || opts.ImageConfigScanners.Enabled(types.MisconfigScanner) {
-		logger := log.WithPrefix(log.PrefixMisconfiguration)
-		logger.Info("Misconfiguration scanning is enabled")
-
-		var downloadedPolicyPaths []string
-		var disableEmbedded bool
-
-		downloadedPolicyPaths, err := operation.InitBuiltinPolicies(context.Background(), opts.CacheDir, opts.Quiet, opts.SkipCheckUpdate, opts.MisconfOptions.ChecksBundleRepository, opts.RegistryOpts())
+		var err error
+		configScannerOptions, err = initMisconfScannerOption(opts)
 		if err != nil {
-			if !opts.SkipCheckUpdate {
-				logger.Error("Falling back to embedded checks", log.Err(err))
-			}
-		} else {
-			logger.Debug("Policies successfully loaded from disk")
-			disableEmbedded = true
-		}
-		configScannerOptions = misconf.ScannerOption{
-			Debug:                    opts.Debug,
-			Trace:                    opts.Trace,
-			Namespaces:               append(opts.CheckNamespaces, rego.BuiltinNamespaces()...),
-			PolicyPaths:              append(opts.CheckPaths, downloadedPolicyPaths...),
-			DataPaths:                opts.DataPaths,
-			HelmValues:               opts.HelmValues,
-			HelmValueFiles:           opts.HelmValueFiles,
-			HelmFileValues:           opts.HelmFileValues,
-			HelmStringValues:         opts.HelmStringValues,
-			HelmAPIVersions:          opts.HelmAPIVersions,
-			HelmKubeVersion:          opts.HelmKubeVersion,
-			TerraformTFVars:          opts.TerraformTFVars,
-			CloudFormationParamVars:  opts.CloudFormationParamVars,
-			K8sVersion:               opts.K8sVersion,
-			DisableEmbeddedPolicies:  disableEmbedded,
-			DisableEmbeddedLibraries: disableEmbedded,
-			IncludeDeprecatedChecks:  opts.IncludeDeprecatedChecks,
-			TfExcludeDownloaded:      opts.TfExcludeDownloaded,
+			return ScannerConfig{}, types.ScanOptions{}, err
 		}
 	}
 
@@ -650,3 +620,49 @@ func (r *runner) scan(ctx context.Context, opts flag.Options, initializeScanner
 	}
 	return report, nil
 }
+
+func initMisconfScannerOption(opts flag.Options) (misconf.ScannerOption, error) {
+	logger := log.WithPrefix(log.PrefixMisconfiguration)
+	logger.Info("Misconfiguration scanning is enabled")
+
+	var downloadedPolicyPaths []string
+	var disableEmbedded bool
+
+	downloadedPolicyPaths, err := operation.InitBuiltinPolicies(context.Background(), opts.CacheDir, opts.Quiet, opts.SkipCheckUpdate, opts.MisconfOptions.ChecksBundleRepository, opts.RegistryOpts())
+	if err != nil {
+		if !opts.SkipCheckUpdate {
+			logger.Error("Falling back to embedded checks", log.Err(err))
+		}
+	} else {
+		logger.Debug("Checks successfully loaded from disk")
+		disableEmbedded = true
+	}
+
+	configSchemas, err := misconf.LoadConfigSchemas(opts.ConfigFileSchemas)
+	if err != nil {
+		return misconf.ScannerOption{}, xerrors.Errorf("load schemas error: %w", err)
+	}
+
+	return misconf.ScannerOption{
+		Debug:                    opts.Debug,
+		Trace:                    opts.Trace,
+		Namespaces:               append(opts.CheckNamespaces, rego.BuiltinNamespaces()...),
+		PolicyPaths:              append(opts.CheckPaths, downloadedPolicyPaths...),
+		DataPaths:                opts.DataPaths,
+		HelmValues:               opts.HelmValues,
+		HelmValueFiles:           opts.HelmValueFiles,
+		HelmFileValues:           opts.HelmFileValues,
+		HelmStringValues:         opts.HelmStringValues,
+		HelmAPIVersions:          opts.HelmAPIVersions,
+		HelmKubeVersion:          opts.HelmKubeVersion,
+		TerraformTFVars:          opts.TerraformTFVars,
+		CloudFormationParamVars:  opts.CloudFormationParamVars,
+		K8sVersion:               opts.K8sVersion,
+		DisableEmbeddedPolicies:  disableEmbedded,
+		DisableEmbeddedLibraries: disableEmbedded,
+		IncludeDeprecatedChecks:  opts.IncludeDeprecatedChecks,
+		TfExcludeDownloaded:      opts.TfExcludeDownloaded,
+		FilePatterns:             opts.FilePatterns,
+		ConfigFileSchemas:        configSchemas,
+	}, nil
+}

pkg/fanal/analyzer/analyzer.go

@@ -214,7 +214,11 @@ func (r *AnalysisResult) Sort() {
 
 	// Misconfigurations
 	sort.Slice(r.Misconfigurations, func(i, j int) bool {
-		return r.Misconfigurations[i].FilePath < r.Misconfigurations[j].FilePath
+		if r.Misconfigurations[i].FileType != r.Misconfigurations[j].FileType {
+			return r.Misconfigurations[i].FileType < r.Misconfigurations[j].FileType
+		} else {
+			return r.Misconfigurations[i].FilePath < r.Misconfigurations[j].FilePath
+		}
 	})
 
 	// Secrets

pkg/fanal/analyzer/config/all/import.go

@@ -5,8 +5,10 @@ import (
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/cloudformation"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/dockerfile"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/helm"
+	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/json"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/k8s"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/terraform"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/terraformplan/json"
 	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/terraformplan/snapshot"
+	_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/yaml"
 )

pkg/fanal/analyzer/config/azurearm/azurearm.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
 	"github.com/aquasecurity/trivy/pkg/fanal/analyzer/config"
-	"github.com/aquasecurity/trivy/pkg/misconf"
+	"github.com/aquasecurity/trivy/pkg/iac/detection"
 )
 
 const (
@@ -25,7 +25,7 @@ type azureARMConfigAnalyzer struct {
 }
 
 func newAzureARMConfigAnalyzer(opts analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
-	a, err := config.NewAnalyzer(analyzerType, version, misconf.NewAzureARMScanner, opts)
+	a, err := config.NewAnalyzer(analyzerType, version, detection.FileTypeAzureARM, opts)
 	if err != nil {
 		return nil, err
 	}

pkg/fanal/analyzer/config/cloudformation/cloudformation.go

@@ -3,7 +3,7 @@ package cloudformation
 import (
 	"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
 	"github.com/aquasecurity/trivy/pkg/fanal/analyzer/config"
-	"github.com/aquasecurity/trivy/pkg/misconf"
+	"github.com/aquasecurity/trivy/pkg/iac/detection"
 )
 
 const (
@@ -22,7 +22,7 @@ type cloudFormationConfigAnalyzer struct {
 }
 
 func newCloudFormationConfigAnalyzer(opts analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
-	a, err := config.NewAnalyzer(analyzerType, version, misconf.NewCloudFormationScanner, opts)
+	a, err := config.NewAnalyzer(analyzerType, version, detection.FileTypeCloudFormation, opts)
 	if err != nil {
 		return nil, err
 	}

pkg/fanal/analyzer/config/config.go

@@ -9,6 +9,7 @@ import (
 	"k8s.io/utils/strings/slices"
 
 	"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
+	"github.com/aquasecurity/trivy/pkg/iac/detection"
 	"github.com/aquasecurity/trivy/pkg/misconf"
 )
 
@@ -26,10 +27,8 @@ type Analyzer struct {
 	scanner *misconf.Scanner
 }
 
-type NewScanner func([]string, misconf.ScannerOption) (*misconf.Scanner, error)
-
-func NewAnalyzer(t analyzer.Type, version int, newScanner NewScanner, opts analyzer.AnalyzerOptions) (*Analyzer, error) {
-	s, err := newScanner(opts.FilePatterns, opts.MisconfScannerOption)
+func NewAnalyzer(t analyzer.Type, version int, fileType detection.FileType, opts analyzer.AnalyzerOptions) (*Analyzer, error) {
+	s, err := misconf.NewScanner(fileType, opts.MisconfScannerOption)
 	if err != nil {
 		return nil, xerrors.Errorf("%s scanner init error: %w", t, err)
 	}