Skip to content

Commit

Permalink
Run container-structure-test on remote images
Browse files Browse the repository at this point in the history 
Fixes #3907
Fixes #1082

Signed-off-by: David Gageot <[email protected]>
  • Loading branch information
@dgageot
dgageot committed Apr 21, 2020
1 parent 992ced4 commit bfe58e1
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 26 deletions.
6 changes: 3 additions & 3 deletions pkg/skaffold/runner/new.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func NewForConfig(runCtx *runcontext.RunContext) (*SkaffoldRunner, error) {
return nil, fmt.Errorf("initializing cache: %w", err)
}

tester := getTester(runCtx)
tester := getTester(runCtx, imagesAreLocal)
syncer := getSyncer(runCtx)

deployer, err := getDeployer(runCtx)
Expand Down Expand Up @@ -199,8 +199,8 @@ func getBuilder(runCtx *runcontext.RunContext) (build.Builder, error) {
}
}

func getTester(runCtx *runcontext.RunContext) test.Tester {
return test.NewTester(runCtx)
func getTester(runCtx *runcontext.RunContext, imagesAreLocal bool) test.Tester {
return test.NewTester(runCtx, imagesAreLocal)
}

func getSyncer(runCtx *runcontext.RunContext) sync.Syncer {
Expand Down
21 changes: 15 additions & 6 deletions pkg/skaffold/test/test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,17 @@ import (
// NewTester parses the provided test cases from the Skaffold config,
// and returns a Tester instance with all the necessary test runners
// to run all specified tests.
func NewTester(runCtx *runcontext.RunContext) Tester {
client, err := docker.NewAPIClient(runCtx)
func NewTester(runCtx *runcontext.RunContext, imagesAreLocal bool) Tester {
localDaemon, err := docker.NewAPIClient(runCtx)
if err != nil {
return nil
}

return FullTester{
testCases: runCtx.Cfg.Test,
workingDir: runCtx.WorkingDir,
extraEnv: client.ExtraEnv(),
testCases: runCtx.Cfg.Test,
workingDir: runCtx.WorkingDir,
localDaemon: localDaemon,
imagesAreLocal: imagesAreLocal,
}
}

Expand Down Expand Up @@ -84,8 +85,16 @@ func (t FullTester) runStructureTests(ctx context.Context, out io.Writer, bRes [
}

fqn := resolveArtifactImageTag(testCase.ImageName, bRes)
if !t.imagesAreLocal {
// The image is remote so we have to pull it locally.
// `container-structure-test` currently can't do it:
// https://github.com/GoogleContainerTools/container-structure-test/issues/253.
if err := t.localDaemon.Pull(ctx, out, fqn); err != nil {
return fmt.Errorf("unable to docker pull image %q: %w", fqn, err)
}
}

runner := structure.NewRunner(files, t.extraEnv)
runner := structure.NewRunner(files, t.localDaemon.ExtraEnv())
return runner.Test(ctx, out, fqn)
}

Expand Down
82 changes: 68 additions & 14 deletions pkg/skaffold/test/test_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"testing"

"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/runcontext"
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
Expand All @@ -32,7 +33,7 @@ import (
func TestNoTestDependencies(t *testing.T) {
runCtx := &runcontext.RunContext{}

deps, err := NewTester(runCtx).TestDependencies()
deps, err := NewTester(runCtx, true).TestDependencies()

testutil.CheckErrorAndDeepEqual(t, false, err, 0, len(deps))
}
Expand All @@ -53,7 +54,7 @@ func TestTestDependencies(t *testing.T) {
},
}

deps, err := NewTester(runCtx).TestDependencies()
deps, err := NewTester(runCtx, true).TestDependencies()

expectedDeps := tmpDir.Paths("tests/test1.yaml", "tests/test2.yaml", "test3.yaml")
testutil.CheckErrorAndDeepEqual(t, false, err, expectedDeps, deps)
Expand All @@ -68,7 +69,7 @@ func TestWrongPattern(t *testing.T) {
},
}

tester := NewTester(runCtx)
tester := NewTester(runCtx, true)

_, err := tester.TestDependencies()
testutil.CheckError(t, true, err)
Expand All @@ -80,19 +81,18 @@ func TestWrongPattern(t *testing.T) {
func TestNoTest(t *testing.T) {
runCtx := &runcontext.RunContext{}

err := NewTester(runCtx).Test(context.Background(), ioutil.Discard, nil)
err := NewTester(runCtx, true).Test(context.Background(), ioutil.Discard, nil)

testutil.CheckError(t, false, err)
}

func TestTestSuccess(t *testing.T) {
testutil.Run(t, "", func(t *testutil.T) {
tmpDir := t.NewTempDir()
tmpDir.Touch("tests/test1.yaml", "tests/test2.yaml", "test3.yaml")
tmpDir := t.NewTempDir().Touch("tests/test1.yaml", "tests/test2.yaml", "test3.yaml")

t.Override(&util.DefaultExecCommand, testutil.
CmdRun("container-structure-test test -v warn --image TAG --config "+tmpDir.Path("tests/test1.yaml")+" --config "+tmpDir.Path("tests/test2.yaml")).
AndRun("container-structure-test test -v warn --image TAG --config "+tmpDir.Path("test3.yaml")))
CmdRun("container-structure-test test -v warn --image image:tag --config "+tmpDir.Path("tests/test1.yaml")+" --config "+tmpDir.Path("tests/test2.yaml")).
AndRun("container-structure-test test -v warn --image image:tag --config "+tmpDir.Path("test3.yaml")))

runCtx := &runcontext.RunContext{
WorkingDir: tmpDir.Root(),
Expand All @@ -111,26 +111,80 @@ func TestTestSuccess(t *testing.T) {
},
}

err := NewTester(runCtx).Test(context.Background(), ioutil.Discard, []build.Artifact{{
imagesAreLocal := true
err := NewTester(runCtx, imagesAreLocal).Test(context.Background(), ioutil.Discard, []build.Artifact{{
ImageName: "image",
Tag: "TAG",
Tag: "image:tag",
}})

t.CheckNoError(err)
})
}

func TestTestSuccessRemoteImage(t *testing.T) {
testutil.Run(t, "", func(t *testutil.T) {
t.NewTempDir().Touch("test.yaml").Chdir()
t.Override(&util.DefaultExecCommand, testutil.CmdRun("container-structure-test test -v warn --image image:tag --config test.yaml"))
t.Override(&docker.NewAPIClient, func(*runcontext.RunContext) (docker.LocalDaemon, error) {
return docker.NewLocalDaemon(&testutil.FakeAPIClient{}, nil, false, nil), nil
})

runCtx := &runcontext.RunContext{
Cfg: latest.Pipeline{
Test: []*latest.TestCase{{
ImageName: "image",
StructureTests: []string{"test.yaml"},
}},
},
}

imagesAreLocal := false
err := NewTester(runCtx, imagesAreLocal).Test(context.Background(), ioutil.Discard, []build.Artifact{{
ImageName: "image",
Tag: "image:tag",
}})

t.CheckNoError(err)
})
}

func TestTestFailureRemoteImage(t *testing.T) {
testutil.Run(t, "", func(t *testutil.T) {
t.NewTempDir().Touch("test.yaml").Chdir()
t.Override(&util.DefaultExecCommand, testutil.CmdRun("container-structure-test test -v warn --image image:tag --config test.yaml"))
t.Override(&docker.NewAPIClient, func(*runcontext.RunContext) (docker.LocalDaemon, error) {
return docker.NewLocalDaemon(&testutil.FakeAPIClient{ErrImagePull: true}, nil, false, nil), nil
})

runCtx := &runcontext.RunContext{
Cfg: latest.Pipeline{
Test: []*latest.TestCase{{
ImageName: "image",
StructureTests: []string{"test.yaml"},
}},
},
}

imagesAreLocal := false
err := NewTester(runCtx, imagesAreLocal).Test(context.Background(), ioutil.Discard, []build.Artifact{{
ImageName: "image",
Tag: "image:tag",
}})

t.CheckErrorContains(`unable to docker pull image "image:tag"`, err)
})
}

func TestTestFailure(t *testing.T) {
testutil.Run(t, "", func(t *testutil.T) {
tmpDir := t.NewTempDir().Touch("test.yaml")
t.NewTempDir().Touch("test.yaml").Chdir()

t.Override(&util.DefaultExecCommand, testutil.CmdRunErr(
"container-structure-test test -v warn --image broken-image --config "+tmpDir.Path("test.yaml"),
"container-structure-test test -v warn --image broken-image --config test.yaml",
errors.New("FAIL"),
))

runCtx := &runcontext.RunContext{
WorkingDir: tmpDir.Root(),
Cfg: latest.Pipeline{
Test: []*latest.TestCase{
{
Expand All @@ -141,7 +195,7 @@ func TestTestFailure(t *testing.T) {
},
}

err := NewTester(runCtx).Test(context.Background(), ioutil.Discard, []build.Artifact{{}})
err := NewTester(runCtx, true).Test(context.Background(), ioutil.Discard, []build.Artifact{{}})
t.CheckError(true, err)
})
}
8 changes: 5 additions & 3 deletions pkg/skaffold/test/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"io"

"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
)

Expand All @@ -42,9 +43,10 @@ type Tester interface {
// FullTester should always be the ONLY implementation of the Tester interface;
// newly added testing implementations should implement the Runner interface.
type FullTester struct {
testCases []*latest.TestCase
workingDir string
extraEnv []string
testCases []*latest.TestCase
localDaemon docker.LocalDaemon
workingDir string
imagesAreLocal bool
}

// Runner is the lowest-level test executor in Skaffold, responsible for
Expand Down

0 comments on commit bfe58e1

Please sign in to comment.