Add custom parameters to structured tests (#6055)

Custom parameters can now be added to structured tests in form of yaml config list.
GoogleContainerTools · Jun 23, 2021 · c326595 · c326595
1 parent 6ac28cc
commit c326595
Show file tree

Hide file tree

Showing 6 changed files with 96 additions and 12 deletions.
diff --git a/docs/content/en/docs/pipeline-stages/testers/structure.md b/docs/content/en/docs/pipeline-stages/testers/structure.md
@@ -34,4 +34,8 @@ In order to restrict the executed structure tests, a `profile` section can overr
 
 {{% readfile file="samples/testers/structure/testProfile.yaml" %}}
 
+User can customize `container-structure-test` behavior by passing a list of configuration flags as a value of `structureTestsArgs` yaml property in `skaffold.yaml`, e.g.:
+
+{{% readfile file="samples/testers/structure/structureTestArgs.yaml" %}}
+
 To execute the tests once, run `skaffold test --profile quickcheck`.
diff --git a/docs/content/en/samples/testers/structure/structureTestArgs.yaml b/docs/content/en/samples/testers/structure/structureTestArgs.yaml
@@ -0,0 +1,9 @@
+test:
+  - image: gcr.io/k8s-skaffold/skaffold-example
+    structureTests:
+      - './structure-test/*'
+    structureTestsArgs:
+      - --driver=tar
+      - -q
+      - --no-color
+      -  --test-report=TEST_REPORT_NAME
diff --git a/docs/content/en/schemas/v2beta18.json b/docs/content/en/schemas/v2beta18.json
@@ -3146,13 +3146,26 @@
           "examples": [
             "[\"./test/*\"]"
           ]
+        },
+        "structureTestsArgs": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "description": "additional configuration arguments passed to `container-structure-test` binary.",
+          "x-intellij-html-description": "additional configuration arguments passed to <code>container-structure-test</code> binary.",
+          "default": "[]",
+          "examples": [
+            "[\"--driver=tar\", \"--no-color\", \"-q\"]"
+          ]
         }
       },
       "preferredOrder": [
         "image",
         "context",
         "custom",
-        "structureTests"
+        "structureTests",
+        "structureTestsArgs"
       ],
       "additionalProperties": false,
       "type": "object",

diff --git a/pkg/skaffold/schema/latest/v1/config.go b/pkg/skaffold/schema/latest/v1/config.go
@@ -491,6 +491,10 @@ type TestCase struct {
 	// to run on that artifact.
 	// For example: `["./test/*"]`.
 	StructureTests []string `yaml:"structureTests,omitempty" skaffold:"filepath"`
+
+	// StructureTestArgs lists additional configuration arguments passed to `container-structure-test` binary.
+	// For example: `["--driver=tar", "--no-color", "-q"]`.
+	StructureTestArgs []string `yaml:"structureTestsArgs,omitempty"`
 }
 
 // DeployConfig contains all the configuration needed by the deploy steps.

diff --git a/pkg/skaffold/test/structure/structure.go b/pkg/skaffold/test/structure/structure.go
@@ -32,11 +32,12 @@ import (
 )
 
 type Runner struct {
-	structureTests []string
-	imageName      string
-	imageIsLocal   bool
-	workspace      string
-	localDaemon    docker.LocalDaemon
+	structureTests    []string
+	structureTestArgs []string
+	imageName         string
+	imageIsLocal      bool
+	workspace         string
+	localDaemon       docker.LocalDaemon
 }
 
 // New creates a new structure.Runner.
@@ -46,11 +47,12 @@ func New(cfg docker.Config, tc *latestV1.TestCase, imageIsLocal bool) (*Runner,
 		return nil, err
 	}
 	return &Runner{
-		structureTests: tc.StructureTests,
-		imageName:      tc.ImageName,
-		workspace:      tc.Workspace,
-		localDaemon:    localDaemon,
-		imageIsLocal:   imageIsLocal,
+		structureTests:    tc.StructureTests,
+		structureTestArgs: tc.StructureTestArgs,
+		imageName:         tc.ImageName,
+		workspace:         tc.Workspace,
+		localDaemon:       localDaemon,
+		imageIsLocal:      imageIsLocal,
 	}, nil
 }
 
@@ -86,7 +88,7 @@ func (cst *Runner) runStructureTests(ctx context.Context, out io.Writer, imageTa
 	for _, f := range files {
 		args = append(args, "--config", f)
 	}
-
+	args = append(args, cst.structureTestArgs...)
 	cmd := exec.CommandContext(ctx, "container-structure-test", args...)
 	cmd.Stdout = out
 	cmd.Stderr = out

diff --git a/pkg/skaffold/test/structure/structure_test.go b/pkg/skaffold/test/structure/structure_test.go
@@ -94,6 +94,58 @@ func TestIgnoreDockerNotFound(t *testing.T) {
 	})
 }
 
+func TestCustomParams(t *testing.T) {
+	testCases := []struct {
+		structureTestArgs []string
+		expectedCmd       string
+	}{
+		{
+			structureTestArgs: []string{"--driver=tar", "--force", "-q", "--save"},
+			expectedCmd:       " --driver=tar --force -q --save",
+		},
+		{
+			structureTestArgs: []string{},
+			expectedCmd:       "",
+		},
+		{
+			structureTestArgs: nil,
+			expectedCmd:       "",
+		},
+	}
+
+	for _, tc := range testCases {
+		testutil.Run(t, "", func(t *testutil.T) {
+			tmpDir := t.NewTempDir().Touch("test.yaml")
+			t.Override(&cluster.FindMinikubeBinary, func() (string, semver.Version, error) { return "", semver.Version{}, errors.New("not found") })
+
+			baseCmd := "container-structure-test test -v warn --image image:tag --config " + tmpDir.Path("test.yaml")
+			t.Override(&util.DefaultExecCommand, testutil.CmdRun(baseCmd+tc.expectedCmd))
+
+			cfg := &mockConfig{
+				tests: []*latestV1.TestCase{{
+					ImageName:         "image",
+					Workspace:         tmpDir.Root(),
+					StructureTests:    []string{"test.yaml"},
+					StructureTestArgs: tc.structureTestArgs,
+				}},
+			}
+
+			testCase := &latestV1.TestCase{
+				ImageName:         "image",
+				Workspace:         tmpDir.Root(),
+				StructureTests:    []string{"test.yaml"},
+				StructureTestArgs: tc.structureTestArgs,
+			}
+			testEvent.InitializeState([]latestV1.Pipeline{{}})
+
+			testRunner, err := New(cfg, testCase, true)
+			t.CheckNoError(err)
+			err = testRunner.Test(context.Background(), ioutil.Discard, "image:tag")
+			t.CheckNoError(err)
+		})
+	}
+}
+
 type mockConfig struct {
 	runcontext.RunContext // Embedded to provide the default values.
 	tests                 []*latestV1.TestCase