Simplify integration tests

 + Move `Gcloud` command to the Makefile
 + Remove TestMain. It makes it complicated to run a single test
 + Replace build tags with a check on `-short` so that tests are always compiled and linted
 + Fix lint errors on rpc_test.go
 + Some test projects should not be in ./integration/examples because they shouldn’t be in ./examples
 + Fix flakiness on TestDev

Signed-off-by: David Gageot <[email protected]>

Makefile

@@ -86,15 +86,17 @@ install: $(GO_FILES) $(BUILD_DIR)
 
 .PHONY: integration
 integration: install
-	go test -v -tags integration $(REPOPATH)/integration -timeout 10m \
-		--remote=$(REMOTE_INTEGRATION) \
-		--gcp-project=$(GCP_PROJECT) \
-		--gke-cluster-name=$(GKE_CLUSTER_NAME) \
-		--gke-zone=$(GKE_ZONE)
+ifeq ($(REMOTE_INTEGRATION),true)
+	gcloud container clusters get-credentials \
+		$(GKE_CLUSTER_NAME) \
+		--zone $(GKE_ZONE) \
+		--project $(GCP_PROJECT)
+endif
+	REMOTE_INTEGRATION=$(REMOTE_INTEGRATION) go test -v $(REPOPATH)/integration -timeout 10m
 
 .PHONY: coverage
 coverage: $(BUILD_DIR)
-	go test -coverprofile=$(BUILD_DIR)/coverage.txt -covermode=atomic ./...
+	go test -short -coverprofile=$(BUILD_DIR)/coverage.txt -covermode=atomic ./...
 
 .PHONY: release
 release: cross $(BUILD_DIR)/VERSION

appveyor.yml

@@ -5,4 +5,4 @@ environment:
 build_script:
   - go build -o out/skaffold.exe cmd/skaffold/skaffold.go
 test_script:
-  - go test ./...
+  - go test -short -v -timeout 60s ./...

integration/build_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 /*
 Copyright 2019 The Skaffold Authors
 
@@ -26,6 +24,10 @@ import (
 )
 
 func TestBuild(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
 	tests := []struct {
 		description string
 		dir         string

integration/config_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 /*
 Copyright 2019 The Skaffold Authors
 
@@ -24,14 +22,17 @@ import (
 	"strings"
 	"testing"
 
-	yaml "gopkg.in/yaml.v2"
-
 	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/cmd/config"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 	"github.com/GoogleContainerTools/skaffold/testutil"
+	yaml "gopkg.in/yaml.v2"
 )
 
 func TestListConfig(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
 	baseConfig := &config.Config{
 		Global: &config.ContextConfig{
 			DefaultRepo: "global-repository",
@@ -95,6 +96,10 @@ func TestListConfig(t *testing.T) {
 }
 
 func TestSetConfig(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
 	baseConfig := &config.Config{
 		Global: &config.ContextConfig{
 			DefaultRepo: "global-repository",

integration/deploy_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 /*
 Copyright 2019 The Skaffold Authors
 
@@ -20,27 +18,29 @@ package integration
 
 import (
 	"context"
-	"time"
-
-	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"testing"
+	"time"
 
 	kubernetesutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestDeploy(t *testing.T) {
-	ns, deleteNs := SetupNamespace(t)
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ns, client, deleteNs := SetupNamespace(t)
 	defer deleteNs()
 
 	RunSkaffold(t, "deploy", "examples/kustomize", ns.Name, "", nil, "--images", "index.docker.io/library/busybox:1")
 
 	depName := "kustomize-test"
-	if err := kubernetesutil.WaitForDeploymentToStabilize(context.Background(), Client, ns.Name, depName, 10*time.Minute); err != nil {
+	if err := kubernetesutil.WaitForDeploymentToStabilize(context.Background(), client, ns.Name, depName, 10*time.Minute); err != nil {
 		t.Fatalf("Timed out waiting for deployment to stabilize")
 	}
 
-	dep, err := Client.AppsV1().Deployments(ns.Name).Get(depName, meta_v1.GetOptions{})
+	dep, err := client.AppsV1().Deployments(ns.Name).Get(depName, meta_v1.GetOptions{})
 	if err != nil {
 		t.Fatalf("Could not find deployment: %s %s", ns.Name, depName)
 	}

integration/dev_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 /*
 Copyright 2019 The Skaffold Authors
 
@@ -29,38 +27,44 @@ import (
 )
 
 func TestDev(t *testing.T) {
-	ns, deleteNs := SetupNamespace(t)
-	defer deleteNs()
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	Run(t, "testdata/dev", "sh", "-c", "echo foo > foo")
+	defer Run(t, "testdata/dev", "rm", "foo")
 
-	Run(t, "examples/test-dev-job", "touch", "foo")
-	defer Run(t, "examples/test-dev-job", "rm", "foo")
+	// Run skaffold build first to fail quickly on a build failure
+	RunSkaffold(t, "build", "testdata/dev", "", "", nil)
+
+	ns, client, deleteNs := SetupNamespace(t)
+	defer deleteNs()
 
 	cancel := make(chan bool)
-	go RunSkaffoldNoFail(cancel, "dev", "examples/test-dev-job", ns.Name, "", nil)
+	go RunSkaffoldNoFail(cancel, "dev", "testdata/dev", ns.Name, "", nil)
 	defer func() { cancel <- true }()
 
-	jobName := "test-dev-job"
-	if err := kubernetesutil.WaitForJobToStabilize(context.Background(), Client, ns.Name, jobName, 10*time.Minute); err != nil {
-		t.Fatalf("Timed out waiting for job to stabilize")
+	deployName := "test-dev"
+	if err := kubernetesutil.WaitForDeploymentToStabilize(context.Background(), client, ns.Name, deployName, 10*time.Minute); err != nil {
+		t.Fatalf("Timed out waiting for deployment to stabilize")
 	}
 
-	job, err := Client.BatchV1().Jobs(ns.Name).Get(jobName, meta_v1.GetOptions{})
+	dep, err := client.AppsV1().Deployments(ns.Name).Get(deployName, meta_v1.GetOptions{})
 	if err != nil {
-		t.Fatalf("Could not find job: %s %s", ns.Name, jobName)
+		t.Fatalf("Could not find dep: %s %s", ns.Name, deployName)
 	}
 
-	time.Sleep(5 * time.Second)
+	// Make a change to foo so that dev is forced to delete the Deployment and redeploy
+	Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")
 
-	// Make a change to foo so that dev is forced to delete the job and redeploy
-	Run(t, "examples/test-dev-job", "sh", "-c", "echo bar > foo")
-
-	// Make sure the UID of the old Job and the UID of the new Job is different
+	// Make sure the old Deployment and the new Deployment are different
 	err = wait.PollImmediate(time.Millisecond*500, 10*time.Minute, func() (bool, error) {
-		newJob, err := Client.BatchV1().Jobs(ns.Name).Get(job.Name, meta_v1.GetOptions{})
+		newDep, err := client.AppsV1().Deployments(ns.Name).Get(deployName, meta_v1.GetOptions{})
 		if err != nil {
 			return false, nil
 		}
-		return job.GetUID() != newJob.GetUID(), nil
+
+		return dep.GetGeneration() != newDep.GetGeneration(), nil
 	})
 	if err != nil {
 		t.Fatalf("redeploy failed: %v", err)

integration/fix_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 /*
 Copyright 2019 The Skaffold Authors
 
@@ -27,21 +25,25 @@ import (
 )
 
 func TestFix(t *testing.T) {
-	ns, deleteNs := SetupNamespace(t)
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ns, _, deleteNs := SetupNamespace(t)
 	defer deleteNs()
 
 	fixCmd := exec.Command("skaffold", "fix", "-f", "skaffold.yaml")
 	fixCmd.Dir = "testdata/fix"
 	out, err := util.RunCmdOut(fixCmd)
 	if err != nil {
-		t.Fatalf("testing error: %v", err)
+		t.Fatalf("skaffold fix: %v", err)
 	}
 
 	runCmd := exec.Command("skaffold", "run", "--namespace", ns.Name, "-f", "-")
 	runCmd.Dir = "testdata/fix"
 	runCmd.Stdin = bytes.NewReader(out)
 
-	if err := util.RunCmd(runCmd); err != nil {
-		t.Fatalf("testing error: %v", err)
+	if out, err := util.RunCmdOut(runCmd); err != nil {
+		t.Fatalf("skaffold run: %v, %s", err, out)
 	}
 }

integration/init_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 /*
 Copyright 2019 The Skaffold Authors
 
@@ -29,6 +27,10 @@ import (
 )
 
 func TestInit(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
 	type testCase struct {
 		name             string
 		dir              string