From 24c79593288fe4115d3ab358e864dd69beeb9f6f Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 1 Aug 2024 15:13:38 +0600 Subject: [PATCH 1/5] test(integration): add plugin tests --- integration/integration_test.go | 2 +- integration/plugin_test.go | 71 +++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 integration/plugin_test.go diff --git a/integration/integration_test.go b/integration/integration_test.go index c263ec2fc51e..639228c0a639 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1,4 +1,4 @@ -//go:build integration || vm_integration || module_integration || k8s_integration +//go:build integration || vm_integration || module_integration || k8s_integration || plugin_integration package integration diff --git a/integration/plugin_test.go b/integration/plugin_test.go new file mode 100644 index 000000000000..a2939d597cd3 --- /dev/null +++ b/integration/plugin_test.go @@ -0,0 +1,71 @@ +//go:build plugin_integration + +package integration + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPlugin(t *testing.T) { + tests := []struct { + name string + plugin string + golden string + }{ + { + // TODO add plugin flags + name: "count plugin installed from `index`", + plugin: "count@v0.2.0", + golden: "Number of vulnerabilities: 5", + }, + { + name: "count plugin installed from github archive", + plugin: "https://github.com/aquasecurity/trivy-plugin-count/archive/refs/tags/v0.1.0.zip", + golden: "Number of vulnerabilities: 5", + }, + } + + // Set up testing DB + cacheDir := initDB(t) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("XDG_DATA_HOME", tmpDir) + + // Install plugin + err := execute([]string{ + "plugin", + "install", + tt.plugin, + }) + require.NoError(t, err) + + // Overwrite Stdout to get output of plugin + tmpFile, err := os.Create(filepath.Join(tmpDir, "tmp.txt")) + require.NoError(t, err) + os.Stdout = tmpFile + + // Run Trivy with plugin as output + err = execute([]string{ + "--cache-dir", + cacheDir, + "fs", + "-f", + "json", + "-o", + "plugin=count", + "testdata/fixtures/repo/gomod", + }) + + got, err := os.ReadFile(tmpFile.Name()) + require.NoError(t, err) + require.Equal(t, tt.golden, strings.TrimSpace(string(got))) + }) + } +} From a06bb7597c4432c9492077a77c7da3b30bc9a6ce Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 1 Aug 2024 15:13:47 +0600 Subject: [PATCH 2/5] mage: add test:plugin --- magefiles/magefile.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/magefiles/magefile.go b/magefiles/magefile.go index 7ce148d885a0..8ee3697c7570 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -285,6 +285,11 @@ func (t Test) K8s() error { return sh.RunWithV(ENV, "go", "test", "-v", "-tags=k8s_integration", "./integration/...") } +// Plugin runs plugin integration tests +func (t Test) Plugin() error { + return sh.RunWithV(ENV, "go", "test", "-v", "-tags=plugin_integration", "./integration/...") +} + // Module runs Wasm integration tests func (t Test) Module() error { mg.Deps(t.FixtureContainerImages, t.GenerateExampleModules) From 60cf5a44262202bf27ecb0a10045487f5ce7c6ee Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 2 Aug 2024 15:34:27 +0600 Subject: [PATCH 3/5] refactor: save golden to files add `update` flag use `pip` dir --- integration/plugin_test.go | 80 ++++++++++++++----- ...t-0.1.0-plugin-with-before-flag.txt.golden | 5 ++ .../testdata/count-0.2.0-plugin.txt.golden | 5 ++ 3 files changed, 70 insertions(+), 20 deletions(-) create mode 100644 integration/testdata/count-0.1.0-plugin-with-before-flag.txt.golden create mode 100644 integration/testdata/count-0.2.0-plugin.txt.golden diff --git a/integration/plugin_test.go b/integration/plugin_test.go index a2939d597cd3..42acda79a1b2 100644 --- a/integration/plugin_test.go +++ b/integration/plugin_test.go @@ -3,40 +3,47 @@ package integration import ( + "io" "os" "path/filepath" - "strings" "testing" "github.com/stretchr/testify/require" + + "github.com/aquasecurity/trivy/pkg/utils/fsutils" ) func TestPlugin(t *testing.T) { tests := []struct { - name string - plugin string - golden string + name string + plugin string + pluginArgs string + golden string }{ { - // TODO add plugin flags name: "count plugin installed from `index`", plugin: "count@v0.2.0", - golden: "Number of vulnerabilities: 5", + golden: "testdata/count-0.2.0-plugin.txt.golden", }, { - name: "count plugin installed from github archive", - plugin: "https://github.com/aquasecurity/trivy-plugin-count/archive/refs/tags/v0.1.0.zip", - golden: "Number of vulnerabilities: 5", + name: "count plugin installed from github archive", + plugin: "https://github.com/aquasecurity/trivy-plugin-count/archive/refs/tags/v0.1.0.zip", + pluginArgs: "--published-before=2020-01-01", + golden: "testdata/count-0.1.0-plugin-with-before-flag.txt.golden", }, } // Set up testing DB cacheDir := initDB(t) + tempStdOut := setTempStdout(t) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tmpDir := t.TempDir() - t.Setenv("XDG_DATA_HOME", tmpDir) + // We can overwrite stdout for `_default_Manager` only once. + // So we need to clear the temporary stdout file before each test case. + clearFile(t, tempStdOut) + + t.Setenv("XDG_DATA_HOME", t.TempDir()) // Install plugin err := execute([]string{ @@ -46,13 +53,15 @@ func TestPlugin(t *testing.T) { }) require.NoError(t, err) - // Overwrite Stdout to get output of plugin - tmpFile, err := os.Create(filepath.Join(tmpDir, "tmp.txt")) + // Get list of plugins + err = execute([]string{ + "plugin", + "list", + }) require.NoError(t, err) - os.Stdout = tmpFile // Run Trivy with plugin as output - err = execute([]string{ + args := []string{ "--cache-dir", cacheDir, "fs", @@ -60,12 +69,43 @@ func TestPlugin(t *testing.T) { "json", "-o", "plugin=count", - "testdata/fixtures/repo/gomod", - }) + "testdata/fixtures/repo/pip", + } - got, err := os.ReadFile(tmpFile.Name()) - require.NoError(t, err) - require.Equal(t, tt.golden, strings.TrimSpace(string(got))) + if tt.pluginArgs != "" { + args = append(args, "--output-plugin-arg", tt.pluginArgs) + } + + err = execute(args) + + if *update { + fsutils.CopyFile(tempStdOut.Name(), tt.golden) + } + + compareRawFiles(t, tt.golden, tempStdOut.Name()) }) } } + +func setTempStdout(t *testing.T) *os.File { + tmpFile := filepath.Join(t.TempDir(), "output.txt") + f, err := os.Create(tmpFile) + require.NoError(t, err) + + // Overwrite Stdout to get output of plugin + defaultStdout := os.Stdout + os.Stdout = f + t.Cleanup(func() { + os.Stdout = defaultStdout + f.Close() + }) + return f +} + +func clearFile(t *testing.T, file *os.File) { + _, err := file.Seek(0, io.SeekStart) + require.NoError(t, err) + + _, err = file.Write([]byte{}) + require.NoError(t, err) +} diff --git a/integration/testdata/count-0.1.0-plugin-with-before-flag.txt.golden b/integration/testdata/count-0.1.0-plugin-with-before-flag.txt.golden new file mode 100644 index 000000000000..4c9d2bbaeb41 --- /dev/null +++ b/integration/testdata/count-0.1.0-plugin-with-before-flag.txt.golden @@ -0,0 +1,5 @@ +Installed Plugins: + Name: count + Version: 0.1.0 + +Number of vulnerabilities: 1 diff --git a/integration/testdata/count-0.2.0-plugin.txt.golden b/integration/testdata/count-0.2.0-plugin.txt.golden new file mode 100644 index 000000000000..9f86fc48e6dd --- /dev/null +++ b/integration/testdata/count-0.2.0-plugin.txt.golden @@ -0,0 +1,5 @@ +Installed Plugins: + Name: count + Version: 0.2.0 + +Number of vulnerabilities: 2 From eeda1c81d03cd69bed12839dad17ab0687d46f4f Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 2 Aug 2024 15:46:43 +0600 Subject: [PATCH 4/5] ci: add plugin tests to `test` workflow --- .github/workflows/test.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 13f279b519b7..ff2062639226 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -126,6 +126,27 @@ jobs: shell: bash run: | mage test:module + plugin-test: + name: Plugin Integration Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4.1.6 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install tools + uses: aquaproj/aqua-installer@v3.0.1 + with: + aqua_version: v1.25.0 + + - name: Run plugin integration tests + shell: bash + run: | + mage test:plugin vm-test: name: VM Integration Test From 33cf9d5ee6faed00aa939cd33731c11ef2f8b738 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 2 Aug 2024 16:51:57 +0600 Subject: [PATCH 5/5] refactor: run plugin tests with `integration` tests --- .github/workflows/test.yaml | 21 --------------------- integration/integration_test.go | 2 +- integration/plugin_test.go | 2 +- magefiles/magefile.go | 5 ----- 4 files changed, 2 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ff2062639226..13f279b519b7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -126,27 +126,6 @@ jobs: shell: bash run: | mage test:module - plugin-test: - name: Plugin Integration Test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4.1.6 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION }} - - - name: Install tools - uses: aquaproj/aqua-installer@v3.0.1 - with: - aqua_version: v1.25.0 - - - name: Run plugin integration tests - shell: bash - run: | - mage test:plugin vm-test: name: VM Integration Test diff --git a/integration/integration_test.go b/integration/integration_test.go index 639228c0a639..c263ec2fc51e 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1,4 +1,4 @@ -//go:build integration || vm_integration || module_integration || k8s_integration || plugin_integration +//go:build integration || vm_integration || module_integration || k8s_integration package integration diff --git a/integration/plugin_test.go b/integration/plugin_test.go index 42acda79a1b2..e4a6bf4d0014 100644 --- a/integration/plugin_test.go +++ b/integration/plugin_test.go @@ -1,4 +1,4 @@ -//go:build plugin_integration +//go:build integration package integration diff --git a/magefiles/magefile.go b/magefiles/magefile.go index 8ee3697c7570..7ce148d885a0 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -285,11 +285,6 @@ func (t Test) K8s() error { return sh.RunWithV(ENV, "go", "test", "-v", "-tags=k8s_integration", "./integration/...") } -// Plugin runs plugin integration tests -func (t Test) Plugin() error { - return sh.RunWithV(ENV, "go", "test", "-v", "-tags=plugin_integration", "./integration/...") -} - // Module runs Wasm integration tests func (t Test) Module() error { mg.Deps(t.FixtureContainerImages, t.GenerateExampleModules)