From 6f87522af6b96251710b8d23875c4a375f199cdd Mon Sep 17 00:00:00 2001 From: Nikita Pivkin Date: Tue, 3 Dec 2024 20:40:10 +0600 Subject: [PATCH] refactor(test): run Trivy once Signed-off-by: Nikita Pivkin --- Makefile | 2 +- integration/check_examples_test.go | 157 ++++++++++++++++++----------- 2 files changed, 100 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index 2f050ec1..1353b715 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ test: .PHONY: integration-test test-integration: - go test -v -timeout 15m -tags=integration ./integration/... + go test -v -timeout 5m -tags=integration ./integration/... .PHONY: rego rego: fmt-rego test-rego diff --git a/integration/check_examples_test.go b/integration/check_examples_test.go index fdc33489..ef2440c6 100644 --- a/integration/check_examples_test.go +++ b/integration/check_examples_test.go @@ -3,11 +3,12 @@ package integration import ( - "fmt" "io/fs" "os" "os/exec" "path/filepath" + "strconv" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -22,58 +23,23 @@ import ( func TestValidateCheckExamples(t *testing.T) { cacheDir := setupCache(t) - - // TODO(nikpivkin): load examples from fs - rego.LoadAndRegister() - - for _, r := range rules.GetRegistered(framework.ALL) { - if _, ok := r.Frameworks[framework.Default]; !ok { - // TODO(nikpivkin): Trivy does not load non default checks - continue - } - - t.Run(r.AVDID, func(t *testing.T) { - examples, path, err := examples.GetCheckExamples(r.Rule) - require.NoError(t, err) - - if path == "" { - return - } - - for provider, providerExamples := range examples { - validateExamples(t, providerExamples.Bad.ToStrings(), provider, cacheDir, r.AVDID, true) - validateExamples(t, providerExamples.Good.ToStrings(), provider, cacheDir, r.AVDID, false) - } - }) + targetDir := setupTarget(t) + outputFile := filepath.Join(t.TempDir(), "report.json") + + args := []string{ + "conf", + "--skip-check-update", + "--quiet", + "--format", "json", + "--output", outputFile, + "--cache-dir", cacheDir, + targetDir, } -} - -func validateExamples(t *testing.T, examples []string, provider, cacheDir, avdID string, expected bool) { - for i, example := range examples { - fileName := fmt.Sprintf("test-%d%s", i, extensionByProvider(provider)) - t.Run(fileName, func(t *testing.T) { - targetFile := filepath.Join(t.TempDir(), fileName) - - require.NoError(t, os.WriteFile(targetFile, []byte(example), fs.ModePerm)) - - outputFile := filepath.Join(t.TempDir(), "report.json") - - args := []string{ - "conf", - "--skip-check-update", - "--quiet", - "--format", "json", - "--output", outputFile, - "--cache-dir", cacheDir, - targetFile, - } - runTrivy(t, args) + runTrivy(t, args) - report := readTrivyReport(t, outputFile) + report := readTrivyReport(t, outputFile) - assert.Equal(t, expected, reportContainsMisconfig(report, fileName, avdID)) - }) - } + verifyExamples(t, report, targetDir) } func setupCache(t *testing.T) string { @@ -81,7 +47,6 @@ func setupCache(t *testing.T) string { cmd := exec.Command("make", "create-bundle") cmd.Dir = ".." - require.NoError(t, cmd.Run()) defer os.Remove("bundle.tar.gz") @@ -97,20 +62,96 @@ func setupCache(t *testing.T) string { return cacheDir } -func reportContainsMisconfig(report types.Report, path string, id string) bool { - for _, res := range report.Results { - if res.Target != path { +func setupTarget(t *testing.T) string { + t.Helper() + + targetDir := t.TempDir() + + // TODO(nikpivkin): load examples from fs + rego.LoadAndRegister() + + for _, r := range rules.GetRegistered(framework.ALL) { + if _, ok := r.Frameworks[framework.Default]; !ok { + // TODO(nikpivkin): Trivy does not load non default checks + continue + } + + examples, path, err := examples.GetCheckExamples(r.Rule) + require.NoError(t, err) + + if path == "" { continue } - for _, misconf := range res.Misconfigurations { - if misconf.AVDID == id && misconf.Status == types.MisconfStatusFailure { - return true + for provider, providerExamples := range examples { + writeExamples(t, providerExamples.Bad.ToStrings(), provider, targetDir, r.AVDID, "bad") + writeExamples(t, providerExamples.Good.ToStrings(), provider, targetDir, r.AVDID, "good") + } + } + + return targetDir +} + +func writeExamples(t *testing.T, examples []string, provider, cacheDir string, id string, typ string) { + for i, example := range examples { + name := "test" + extensionByProvider(provider) + file := filepath.Join(cacheDir, id, provider, typ, strconv.Itoa(i), name) + require.NoError(t, os.MkdirAll(filepath.Dir(file), fs.ModePerm)) + require.NoError(t, os.WriteFile(file, []byte(example), fs.ModePerm)) + } +} + +func verifyExamples(t *testing.T, report types.Report, targetDir string) { + got := getFailureIDs(report) + + err := filepath.Walk(targetDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(targetDir, path) + require.NoError(t, err) + + parts := strings.Split(relPath, string(os.PathSeparator)) + require.Len(t, parts, 5) // should never happen + + id, _, exampleType := parts[0], parts[1], parts[2] + + shouldBePresent := exampleType == "bad" + + t.Run(relPath, func(t *testing.T) { + if shouldBePresent { + ids, exists := got[relPath] + assert.True(t, exists) + assert.Contains(t, ids, id) + } else { + ids, exists := got[relPath] + if exists { + assert.NotContains(t, ids, id) + } + } + }) + return nil + }) + + require.NoError(t, err) +} + +func getFailureIDs(report types.Report) map[string][]string { + ids := make(map[string][]string) + + for _, result := range report.Results { + for _, misconf := range result.Misconfigurations { + if misconf.Status == types.MisconfStatusFailure { + ids[result.Target] = append(ids[result.Target], misconf.AVDID) } } } - return false + return ids } func extensionByProvider(provider string) string {