refactor(test): run Trivy once

Signed-off-by: Nikita Pivkin <[email protected]>

Makefile

@@ -8,7 +8,7 @@ test:
 
 .PHONY: integration-test
 test-integration:
-	go test -v -timeout 15m -tags=integration ./integration/...
+	go test -v -timeout 5m -tags=integration ./integration/...
 
 .PHONY: rego
 rego: fmt-rego test-rego

integration/check_examples_test.go

@@ -3,11 +3,12 @@
 package integration
 
 import (
-	"fmt"
 	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -22,66 +23,30 @@ import (
 
 func TestValidateCheckExamples(t *testing.T) {
 	cacheDir := setupCache(t)
-
-	// TODO(nikpivkin): load examples from fs
-	rego.LoadAndRegister()
-
-	for _, r := range rules.GetRegistered(framework.ALL) {
-		if _, ok := r.Frameworks[framework.Default]; !ok {
-			// TODO(nikpivkin): Trivy does not load non default checks
-			continue
-		}
-
-		t.Run(r.AVDID, func(t *testing.T) {
-			examples, path, err := examples.GetCheckExamples(r.Rule)
-			require.NoError(t, err)
-
-			if path == "" {
-				return
-			}
-
-			for provider, providerExamples := range examples {
-				validateExamples(t, providerExamples.Bad.ToStrings(), provider, cacheDir, r.AVDID, true)
-				validateExamples(t, providerExamples.Good.ToStrings(), provider, cacheDir, r.AVDID, false)
-			}
-		})
+	targetDir := setupTarget(t)
+	outputFile := filepath.Join(t.TempDir(), "report.json")
+
+	args := []string{
+		"conf",
+		"--skip-check-update",
+		"--quiet",
+		"--format", "json",
+		"--output", outputFile,
+		"--cache-dir", cacheDir,
+		targetDir,
 	}
-}
-
-func validateExamples(t *testing.T, examples []string, provider, cacheDir, avdID string, expected bool) {
-	for i, example := range examples {
-		fileName := fmt.Sprintf("test-%d%s", i, extensionByProvider(provider))
-		t.Run(fileName, func(t *testing.T) {
-			targetFile := filepath.Join(t.TempDir(), fileName)
-
-			require.NoError(t, os.WriteFile(targetFile, []byte(example), fs.ModePerm))
-
-			outputFile := filepath.Join(t.TempDir(), "report.json")
-
-			args := []string{
-				"conf",
-				"--skip-check-update",
-				"--quiet",
-				"--format", "json",
-				"--output", outputFile,
-				"--cache-dir", cacheDir,
-				targetFile,
-			}
-			runTrivy(t, args)
+	runTrivy(t, args)
 
-			report := readTrivyReport(t, outputFile)
+	report := readTrivyReport(t, outputFile)
 
-			assert.Equal(t, expected, reportContainsMisconfig(report, fileName, avdID))
-		})
-	}
+	verifyExamples(t, report, targetDir)
 }
 
 func setupCache(t *testing.T) string {
 	t.Helper()
 
 	cmd := exec.Command("make", "create-bundle")
 	cmd.Dir = ".."
-
 	require.NoError(t, cmd.Run())
 	defer os.Remove("bundle.tar.gz")
 
@@ -97,20 +62,96 @@ func setupCache(t *testing.T) string {
 	return cacheDir
 }
 
-func reportContainsMisconfig(report types.Report, path string, id string) bool {
-	for _, res := range report.Results {
-		if res.Target != path {
+func setupTarget(t *testing.T) string {
+	t.Helper()
+
+	targetDir := t.TempDir()
+
+	// TODO(nikpivkin): load examples from fs
+	rego.LoadAndRegister()
+
+	for _, r := range rules.GetRegistered(framework.ALL) {
+		if _, ok := r.Frameworks[framework.Default]; !ok {
+			// TODO(nikpivkin): Trivy does not load non default checks
+			continue
+		}
+
+		examples, path, err := examples.GetCheckExamples(r.Rule)
+		require.NoError(t, err)
+
+		if path == "" {
 			continue
 		}
 
-		for _, misconf := range res.Misconfigurations {
-			if misconf.AVDID == id && misconf.Status == types.MisconfStatusFailure {
-				return true
+		for provider, providerExamples := range examples {
+			writeExamples(t, providerExamples.Bad.ToStrings(), provider, targetDir, r.AVDID, "bad")
+			writeExamples(t, providerExamples.Good.ToStrings(), provider, targetDir, r.AVDID, "good")
+		}
+	}
+
+	return targetDir
+}
+
+func writeExamples(t *testing.T, examples []string, provider, cacheDir string, id string, typ string) {
+	for i, example := range examples {
+		name := "test" + extensionByProvider(provider)
+		file := filepath.Join(cacheDir, id, provider, typ, strconv.Itoa(i), name)
+		require.NoError(t, os.MkdirAll(filepath.Dir(file), fs.ModePerm))
+		require.NoError(t, os.WriteFile(file, []byte(example), fs.ModePerm))
+	}
+}
+
+func verifyExamples(t *testing.T, report types.Report, targetDir string) {
+	got := getFailureIDs(report)
+
+	err := filepath.Walk(targetDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(targetDir, path)
+		require.NoError(t, err)
+
+		parts := strings.Split(relPath, string(os.PathSeparator))
+		require.Len(t, parts, 5) // should never happen
+
+		id, _, exampleType := parts[0], parts[1], parts[2]
+
+		shouldBePresent := exampleType == "bad"
+
+		t.Run(relPath, func(t *testing.T) {
+			if shouldBePresent {
+				ids, exists := got[relPath]
+				assert.True(t, exists)
+				assert.Contains(t, ids, id)
+			} else {
+				ids, exists := got[relPath]
+				if exists {
+					assert.NotContains(t, ids, id)
+				}
+			}
+		})
+		return nil
+	})
+
+	require.NoError(t, err)
+}
+
+func getFailureIDs(report types.Report) map[string][]string {
+	ids := make(map[string][]string)
+
+	for _, result := range report.Results {
+		for _, misconf := range result.Misconfigurations {
+			if misconf.Status == types.MisconfStatusFailure {
+				ids[result.Target] = append(ids[result.Target], misconf.AVDID)
 			}
 		}
 	}
 
-	return false
+	return ids
 }
 
 func extensionByProvider(provider string) string {