Stress test

.travis.yml

@@ -1,35 +1,33 @@
-language: go
+dist: xenial
+os: linux
 
 git:
   submodules: false
 
+language: go
+go: "1.12.x"
 go_import_path: github.com/GoogleContainerTools/skaffold
 
-jobs:
-  include:
-    - os: linux
-      go: "1.12.x"
-      script:
-        - make
-        - make test
-      after_success:
-        - bash <(curl -s https://codecov.io/bash)
-    - os: osx
-      go: "1.12.x"
-      script:
-        - make
-        - make test
-      after_success:
-        - bash <(curl -s https://codecov.io/bash)
-    - os: windows
-      go: "1.12.x"
-      script:
-        - go build -o out/skaffold.exe cmd/skaffold/skaffold.go
-        - go test -short -timeout 60s ./...
-    - stage: integration
-      os: linux
-      go: "1.12.x"
-      before_install:
-        - curl -Lo ${HOME}/bin/kind https://github.com/kubernetes-sigs/kind/releases/download/v0.4.0/kind-linux-amd64
-        - chmod +x ${HOME}/bin/kind
-      script: make integration-in-kind
+env:
+- TEST_RUN=0
+- TEST_RUN=1
+- TEST_RUN=2
+- TEST_RUN=3
+- TEST_RUN=4
+- TEST_RUN=5
+- TEST_RUN=6
+- TEST_RUN=7
+- TEST_RUN=8
+- TEST_RUN=9
+- TEST_RUN=10
+- TEST_RUN=11
+- TEST_RUN=12
+- TEST_RUN=13
+- TEST_RUN=14
+- TEST_RUN=15
+- TEST_RUN=16
+- TEST_RUN=17
+- TEST_RUN=18
+- TEST_RUN=19
+
+script: make test

Makefile

@@ -109,27 +109,28 @@ ifeq ($(GCP_ONLY),true)
 		--zone $(GKE_ZONE) \
 		--project $(GCP_PROJECT)
 endif
-	GCP_ONLY=$(GCP_ONLY) go test -v $(REPOPATH)/integration -timeout 15m $(INTEGRATION_TEST_ARGS)
+	kubectl get nodes -oyaml
+	GCP_ONLY=$(GCP_ONLY) go test -v $(REPOPATH)/integration -timeout 20m $(INTEGRATION_TEST_ARGS)
 
 .PHONY: release
 release: cross $(BUILD_DIR)/VERSION
 	docker build \
-        		-f deploy/skaffold/Dockerfile \
-        		--cache-from gcr.io/$(GCP_PROJECT)/skaffold-builder \
-        		--build-arg VERSION=$(VERSION) \
-        		-t gcr.io/$(GCP_PROJECT)/skaffold:latest \
-        		-t gcr.io/$(GCP_PROJECT)/skaffold:$(VERSION) .
+		-f deploy/skaffold/Dockerfile \
+		--cache-from gcr.io/$(GCP_PROJECT)/skaffold-builder \
+		--build-arg VERSION=$(VERSION) \
+		-t gcr.io/$(GCP_PROJECT)/skaffold:latest \
+		-t gcr.io/$(GCP_PROJECT)/skaffold:$(VERSION) .
 	gsutil -m cp $(BUILD_DIR)/$(PROJECT)-* $(GSC_RELEASE_PATH)/
 	gsutil -m cp $(BUILD_DIR)/VERSION $(GSC_RELEASE_PATH)/VERSION
 	gsutil -m cp -r $(GSC_RELEASE_PATH)/* $(GSC_RELEASE_LATEST)
 
 .PHONY: release-in-docker
 release-in-docker:
 	docker build \
-    		-f deploy/skaffold/Dockerfile \
-    		-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
-    		--target builder \
-    		.
+		-f deploy/skaffold/Dockerfile \
+		-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
+		--target builder \
+		.
 	docker run --rm \
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		-v $(HOME)/.config/gcloud:/root/.config/gcloud \
@@ -138,20 +139,20 @@ release-in-docker:
 .PHONY: release-build
 release-build: cross
 	docker build \
-    		-f deploy/skaffold/Dockerfile \
-    		--cache-from gcr.io/$(GCP_PROJECT)/skaffold-builder \
-    		-t gcr.io/$(GCP_PROJECT)/skaffold:edge \
-    		-t gcr.io/$(GCP_PROJECT)/skaffold:$(COMMIT) .
+		-f deploy/skaffold/Dockerfile \
+		--cache-from gcr.io/$(GCP_PROJECT)/skaffold-builder \
+		-t gcr.io/$(GCP_PROJECT)/skaffold:edge \
+		-t gcr.io/$(GCP_PROJECT)/skaffold:$(COMMIT) .
 	gsutil -m cp $(BUILD_DIR)/$(PROJECT)-* $(GSC_BUILD_PATH)/
 	gsutil -m cp -r $(GSC_BUILD_PATH)/* $(GSC_BUILD_LATEST)
 
 .PHONY: release-build-in-docker
 release-build-in-docker:
 	docker build \
-    		-f deploy/skaffold/Dockerfile \
-    		-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
-    		--target builder \
-    		.
+		-f deploy/skaffold/Dockerfile \
+		-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
+		--target builder \
+		.
 	docker run --rm \
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		-v $(HOME)/.config/gcloud:/root/.config/gcloud \

integration/dev_test.go

@@ -43,10 +43,6 @@ func TestDev(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
-			// TODO(nkubala): fix this test and remove the skip
-			if test.skipFlakyTest {
-				t.Skip("Skip flaky test")
-			}
 			if testing.Short() {
 				t.Skip("skipping integration test")
 			}

integration/util.go

@@ -103,6 +103,8 @@ func (k *NSKubernetesClient) WaitForPodsReady(podNames ...string) {
 	waitLoop:
 		select {
 		case <-ctx.Done():
+			k.printDiskFreeSpace()
+			k.debug("nodes")
 			k.debug("pods")
 			k.t.Fatalf("Timed out waiting for pods %v ready in namespace %s", podNames, k.ns)
 
@@ -158,6 +160,8 @@ func (k *NSKubernetesClient) WaitForDeploymentsToStabilize(depNames ...string) {
 	waitLoop:
 		select {
 		case <-ctx.Done():
+			k.printDiskFreeSpace()
+			k.debug("nodes")
 			k.debug("deployments.apps")
 			k.debug("pods")
 			k.t.Fatalf("Timed out waiting for deployments %v to stabilize in namespace %s", depNames, k.ns)
@@ -190,6 +194,12 @@ func (k *NSKubernetesClient) debug(entities string) {
 	fmt.Println(string(out))
 }
 
+func (k *NSKubernetesClient) printDiskFreeSpace() {
+	cmd := exec.Command("df", "-h")
+	out, _ := cmd.CombinedOutput()
+	fmt.Println(string(out))
+}
+
 func isStable(dp *appsv1.Deployment) bool {
 	return dp.Generation <= dp.Status.ObservedGeneration && *(dp.Spec.Replicas) == dp.Status.Replicas
 }

pkg/skaffold/filemon/monitor.go

@@ -17,16 +17,13 @@ limitations under the License.
 package filemon
 
 import (
-	"context"
-	"io"
-
 	"github.com/pkg/errors"
 )
 
 // Monitor monitors files changes for multiples components.
 type Monitor interface {
 	Register(deps func() ([]string, error), onChange func(Events)) error
-	Run(ctx context.Context, out io.Writer, debounce bool) error
+	Run(debounce bool) error
 	Reset()
 }
 
@@ -69,7 +66,7 @@ func (w *watchList) Reset() {
 }
 
 // Run watches files until the context is cancelled or an error occurs.
-func (w *watchList) Run(ctx context.Context, out io.Writer, debounce bool) error {
+func (w *watchList) Run(debounce bool) error {
 	changed := 0
 	for i, component := range w.components {
 		state, err := Stat(component.deps)

pkg/skaffold/filemon/monitor_test.go

@@ -17,9 +17,6 @@ limitations under the License.
 package filemon
 
 import (
-	"context"
-	"io/ioutil"
-	"sync"
 	"testing"
 	"time"
 
@@ -29,76 +26,59 @@ import (
 func TestFileMonitor(t *testing.T) {
 	var tests = []struct {
 		description string
-		update      func(folder *testutil.TempDir)
+		makeChanges func(folder *testutil.TempDir)
 	}{
 		{
 			description: "file change",
-			update: func(folder *testutil.TempDir) {
+			makeChanges: func(folder *testutil.TempDir) {
 				folder.Chtimes("file", time.Now().Add(2*time.Second))
 			},
 		},
 		{
 			description: "file delete",
-			update: func(folder *testutil.TempDir) {
+			makeChanges: func(folder *testutil.TempDir) {
 				folder.Remove("file")
 			},
 		},
 		{
 			description: "file create",
-			update: func(folder *testutil.TempDir) {
-				folder.Write("new", "content")
+			makeChanges: func(folder *testutil.TempDir) {
+				folder.Touch("new")
 			},
 		},
 	}
 	for _, test := range tests {
 		testutil.Run(t, test.description, func(t *testutil.T) {
-			tmpDir := t.NewTempDir().
-				Write("file", "content")
+			tmpDir := t.NewTempDir().Touch("file")
 
-			folderChanged := newCallback()
-
-			// Watch folder
 			monitor := NewMonitor()
-			err := monitor.Register(tmpDir.List, folderChanged.call)
-			t.CheckNoError(err)
 
-			// Run the watcher
-			ctx, cancel := context.WithCancel(context.Background())
-			var stopped sync.WaitGroup
-			stopped.Add(1)
-			go func() {
-				err = monitor.Run(ctx, ioutil.Discard, false)
-				stopped.Done()
-				t.CheckNoError(err)
-			}()
+			// Register files
+			changed := callback{}
+			err := monitor.Register(tmpDir.List, changed.call)
+			t.CheckErrorAndDeepEqual(false, err, 0, changed.calls())
+
+			test.makeChanges(tmpDir)
 
-			test.update(tmpDir)
+			// Verify the Monitor detects a change
+			err = monitor.Run(false)
+			t.CheckErrorAndDeepEqual(false, err, 1, changed.calls())
 
-			// Wait for the callbacks
-			folderChanged.wait()
-			cancel()
-			stopped.Wait() // Make sure the watcher is stopped before deleting the tmp folder
+			// Verify the Monitor doesn't detect more changes
+			err = monitor.Run(false)
+			t.CheckErrorAndDeepEqual(false, err, 1, changed.calls())
 		})
 	}
 }
 
 type callback struct {
-	wg *sync.WaitGroup
-}
-
-func newCallback() *callback {
-	var wg sync.WaitGroup
-	wg.Add(1)
-
-	return &callback{
-		wg: &wg,
-	}
+	events []Events
 }
 
 func (c *callback) call(e Events) {
-	c.wg.Done()
+	c.events = append(c.events, e)
 }
 
-func (c *callback) wait() {
-	c.wg.Wait()
+func (c *callback) calls() int {
+	return len(c.events)
 }

pkg/skaffold/runner/dev_test.go

@@ -19,7 +19,6 @@ package runner
 import (
 	"context"
 	"errors"
-	"io"
 	"io/ioutil"
 	"testing"
 
@@ -36,7 +35,7 @@ func (t *NoopMonitor) Register(func() ([]string, error), func(filemon.Events)) e
 	return nil
 }
 
-func (t *NoopMonitor) Run(context.Context, io.Writer, bool) error {
+func (t *NoopMonitor) Run(bool) error {
 	return nil
 }
 
@@ -48,7 +47,7 @@ func (t *FailMonitor) Register(func() ([]string, error), func(filemon.Events)) e
 	return nil
 }
 
-func (t *FailMonitor) Run(context.Context, io.Writer, bool) error {
+func (t *FailMonitor) Run(bool) error {
 	return errors.New("BUG")
 }
 
@@ -65,7 +64,7 @@ func (t *TestMonitor) Register(deps func() ([]string, error), onChange func(file
 	return nil
 }
 
-func (t *TestMonitor) Run(ctx context.Context, out io.Writer, _ bool) error {
+func (t *TestMonitor) Run(bool) error {
 	evt := t.events[t.testBench.currentCycle]
 
 	for _, file := range evt.Modified {

pkg/skaffold/runner/listen.go

@@ -40,7 +40,7 @@ func (l *SkaffoldListener) LogWatchToUser(out io.Writer) {
 	l.Trigger.LogWatchToUser(out)
 }
 
-// Listen listens to a trigger, and when one is received, computes file changes and
+// WatchForChanges listens to a trigger, and when one is received, computes file changes and
 // conditionally runs the dev loop.
 func (l *SkaffoldListener) WatchForChanges(ctx context.Context, out io.Writer, devLoop func(context.Context, io.Writer) error) error {
 	ctxTrigger, cancelTrigger := context.WithCancel(ctx)
@@ -51,7 +51,7 @@ func (l *SkaffoldListener) WatchForChanges(ctx context.Context, out io.Writer, d
 	}
 
 	// exit if file monitor fails the first time
-	if err := l.Monitor.Run(ctx, out, l.Trigger.Debounce()); err != nil {
+	if err := l.Monitor.Run(l.Trigger.Debounce()); err != nil {
 		return errors.Wrap(err, "failed to monitor files")
 	}
 
@@ -62,7 +62,7 @@ func (l *SkaffoldListener) WatchForChanges(ctx context.Context, out io.Writer, d
 		case <-ctx.Done():
 			return nil
 		case <-trigger:
-			if err := l.Monitor.Run(ctx, out, l.Trigger.Debounce()); err != nil {
+			if err := l.Monitor.Run(l.Trigger.Debounce()); err != nil {
 				logrus.Warnf("error computing file changes: %s", err.Error())
 				logrus.Warnf("skaffold may not run successfully!")
 			}