Adding events for Test phase (#5573)

* Adding test trigger events

* Updating Copyright year.

* Updating unit tests.

* Adding unit tests.

* Added integration test for test events.

* Adding integration test for structure test events.

* Updating integration tests to verify test completed event.

* Updated integration test to use profiles to reduce code duplication.

* Removed unused method

* Test state reset.

* Remove deploy state reset.

integration/constants.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2021 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+const (
+	InProgress = "In Progress"
+)

integration/dev_test.go

@@ -187,7 +187,7 @@ func verifyDeployment(t *testing.T, entries chan *proto.LogEntry, client *NSKube
 	// Ensure we see a deploy triggered in the event log
 	err := wait.Poll(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
 		e := <-entries
-		return e.GetEvent().GetDeployEvent().GetStatus() == "In Progress", nil
+		return e.GetEvent().GetDeployEvent().GetStatus() == InProgress, nil
 	})
 	failNowIfError(t, err)
 

integration/sync_test.go

@@ -246,7 +246,7 @@ func verifySyncCompletedWithEvents(t *testing.T, entries chan *proto.LogEntry, n
 	err := wait.Poll(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
 		e := <-entries
 		event := e.GetEvent().GetFileSyncEvent()
-		return event != nil && event.GetStatus() == "In Progress", nil
+		return event != nil && event.GetStatus() == InProgress, nil
 	})
 	failNowIfError(t, err)
 

integration/test_events_test.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2021 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
+	"github.com/GoogleContainerTools/skaffold/proto/v1"
+)
+
+func TestTestEvents(t *testing.T) {
+	MarkIntegrationTest(t, CanRunWithoutGcp)
+
+	tests := []struct {
+		description string
+		podName     string
+		testDir     string
+		config      string
+		args        []string
+		numOfTests  int
+	}{
+		{
+			description: "test events for custom test",
+			podName:     "test-events",
+			testDir:     "testdata/test-events",
+			config:      "skaffold.yaml",
+			args:        []string{"--profile", "custom"},
+			numOfTests:  1,
+		},
+		{
+			description: "test events for structure test",
+			podName:     "test-events",
+			testDir:     "testdata/test-events",
+			config:      "skaffold.yaml",
+			args:        []string{"--profile", "structure"},
+			numOfTests:  1,
+		},
+		{
+			description: "test events for custom & structure tests",
+			podName:     "test-events",
+			testDir:     "testdata/test-events",
+			config:      "skaffold.yaml",
+			args:        []string{"--profile", "customandstructure"},
+			numOfTests:  2,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			// Run skaffold build first to fail quickly on a build failure
+			skaffold.Build(test.args...).InDir(test.testDir).WithConfig(test.config).RunOrFail(t)
+
+			ns, client := SetupNamespace(t)
+			rpcAddr := randomPort()
+
+			// test.args...
+			args := append(test.args, "--rpc-port", rpcAddr)
+			skaffold.Dev(args...).InDir(test.testDir).WithConfig(test.config).InNs(ns.Name).RunLive(t)
+
+			client.WaitForPodsReady(test.podName)
+
+			// Ensure we see a test is triggered in the event log
+			_, entries := apiEvents(t, rpcAddr)
+
+			for i := 0; i < test.numOfTests; i++ {
+				verifyTestCompletedWithEvents(t, entries)
+			}
+		})
+	}
+}
+
+func verifyTestCompletedWithEvents(t *testing.T, entries chan *proto.LogEntry) {
+	// Ensure we see a test in progress triggered in the event log
+	err := wait.Poll(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
+		e := <-entries
+		event := e.GetEvent().GetTestEvent()
+		return event != nil && event.GetStatus() == InProgress, nil
+	})
+	failNowIfError(t, err)
+
+	// Ensure we see the test completed triggered in the event log
+	err = wait.Poll(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
+		e := <-entries
+		event := e.GetEvent().GetTestEvent()
+		return event != nil && event.GetStatus() == "Complete", nil
+	})
+	failNowIfError(t, err)
+}

integration/testdata/test-events/Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.15 as builder
+COPY main.go .
+# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
+ARG SKAFFOLD_GO_GCFLAGS
+RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o /app main.go
+
+FROM alpine:3
+# Define GOTRACEBACK to mark this container as using the Go language runtime
+# for `skaffold debug` (https://skaffold.dev/docs/workflows/debug/).
+ENV GOTRACEBACK=single
+CMD ["./app"]
+COPY --from=builder /app .

integration/testdata/test-events/k8s-pod.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-events
+spec:
+  containers:
+  - name: test-events
+    image: test-events

integration/testdata/test-events/main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"fmt"
+	"time"
+)
+
+func main() {
+	for {
+		fmt.Println("Hello world!")
+
+		time.Sleep(time.Second * 1)
+	}
+}

integration/testdata/test-events/skaffold.yaml

@@ -0,0 +1,27 @@
+apiVersion: skaffold/v2beta13
+kind: Config
+build:
+  artifacts:
+  - image: test-events
+deploy:
+  kubectl:
+    manifests:
+      - k8s-*
+profiles:
+  - name: custom
+    test:
+      - image: test-events
+        custom:
+          - command: echo Hello!
+  - name: structure
+    test:
+      - image: test-events
+        structureTests:
+          - ./test/*
+  - name: customandstructure
+    test:
+      - image: test-events
+        custom:
+          - command: echo Hello!
+        structureTests:
+          - ./test/*

integration/testdata/test-events/test/profile_structure_test.yaml

@@ -0,0 +1,6 @@
+schemaVersion: 2.0.0
+
+fileExistenceTests:
+  - name: 'no go binary'
+    path: '/usr/bin/go'
+    shouldExist: false

integration/testdata/test-events/test/structure_test.yaml

@@ -0,0 +1,6 @@
+schemaVersion: 2.0.0
+
+fileExistenceTests:
+  - name: 'no local go binary'
+    path: /usr/local/bin/go'
+    shouldExist: false

pkg/skaffold/errors/errors.go

@@ -151,6 +151,11 @@ var allErrors = map[Phase][]problem{
 		errCode:    proto.StatusCode_INIT_UNKNOWN,
 		suggestion: reportIssueSuggestion,
 	}),
+	Test: {{
+		regexp:     re(".*"),
+		errCode:    proto.StatusCode_TEST_UNKNOWN,
+		suggestion: reportIssueSuggestion,
+	}},
 	Deploy: append(knownDeployProblems, problem{
 		regexp:     re(".*"),
 		errCode:    proto.StatusCode_DEPLOY_UNKNOWN,

pkg/skaffold/event/event.go

@@ -174,6 +174,10 @@ func emptyStateWithArtifacts(builds map[string]string, metadata *proto.Metadata,
 			AutoTrigger: autoBuild,
 			StatusCode:  proto.StatusCode_OK,
 		},
+		TestState: &proto.TestState{
+			Status:     NotStarted,
+			StatusCode: proto.StatusCode_OK,
+		},
 		DeployState: &proto.DeployState{
 			Status:      NotStarted,
 			AutoTrigger: autoDeploy,
@@ -332,6 +336,31 @@ func BuildComplete(imageName string) {
 	handler.handleBuildEvent(&proto.BuildEvent{Artifact: imageName, Status: Complete})
 }
 
+// TestInProgress notifies that a test has been started.
+func TestInProgress() {
+	handler.handleTestEvent(&proto.TestEvent{Status: InProgress})
+}
+
+// TestCanceled notifies that a test has been canceled.
+func TestCanceled() {
+	handler.handleTestEvent(&proto.TestEvent{Status: Canceled})
+}
+
+// TestFailed notifies that a test has failed.
+func TestFailed(imageName string, err error) {
+	aiErr := sErrors.ActionableErr(sErrors.Test, err)
+	handler.stateLock.Lock()
+	handler.state.TestState.StatusCode = aiErr.ErrCode
+	handler.stateLock.Unlock()
+	handler.handleTestEvent(&proto.TestEvent{Status: Failed,
+		ActionableErr: aiErr})
+}
+
+// TestComplete notifies that a test has completed.
+func TestComplete() {
+	handler.handleTestEvent(&proto.TestEvent{Status: Complete})
+}
+
 // DevLoopInProgress notifies that a dev loop has been started.
 func DevLoopInProgress(i int) {
 	handler.handleDevLoopEvent(&proto.DevLoopEvent{Iteration: int32(i), Status: InProgress})
@@ -364,6 +393,8 @@ func DevLoopFailedInPhase(iteration int, phase sErrors.Phase, err error) {
 		DevLoopFailedWithErrorCode(iteration, state.StatusCheckState.StatusCode, err)
 	case sErrors.Build:
 		DevLoopFailedWithErrorCode(iteration, state.BuildState.StatusCode, err)
+	case sErrors.Test:
+		DevLoopFailedWithErrorCode(iteration, state.TestState.StatusCode, err)
 	default:
 		ai := sErrors.ActionableErr(phase, err)
 		DevLoopFailedWithErrorCode(iteration, ai.ErrCode, err)
@@ -493,6 +524,14 @@ func (ev *eventHandler) handleBuildEvent(e *proto.BuildEvent) {
 	})
 }
 
+func (ev *eventHandler) handleTestEvent(e *proto.TestEvent) {
+	ev.handle(&proto.Event{
+		EventType: &proto.Event_TestEvent{
+			TestEvent: e,
+		},
+	})
+}
+
 func (ev *eventHandler) handleDevLoopEvent(e *proto.DevLoopEvent) {
 	ev.handle(&proto.Event{
 		EventType: &proto.Event_DevLoopEvent{
@@ -560,7 +599,21 @@ func (ev *eventHandler) handleExec(f firedEvent) {
 			// logEntry.Err = be.Err
 		default:
 		}
-
+	case *proto.Event_TestEvent:
+		te := e.TestEvent
+		ev.stateLock.Lock()
+		ev.state.TestState.Status = te.Status
+		ev.stateLock.Unlock()
+		switch te.Status {
+		case InProgress:
+			logEntry.Entry = "Test started"
+		case Complete:
+			logEntry.Entry = "Test completed"
+		case Failed:
+			logEntry.Entry = "Test failed"
+			// logEntry.Err = te.Err
+		default:
+		}
 	case *proto.Event_DeployEvent:
 		de := e.DeployEvent
 		ev.stateLock.Lock()
@@ -677,6 +730,13 @@ func ResetStateOnBuild() {
 	handler.setState(newState)
 }
 
+// ResetStateOnTest resets the test, deploy, sync and status check state
+func ResetStateOnTest() {
+	newState := handler.getState()
+	newState.TestState.Status = NotStarted
+	handler.setState(newState)
+}
+
 // ResetStateOnDeploy resets the deploy, sync and status check state
 func ResetStateOnDeploy() {
 	newState := handler.getState()