gracefull shutdown RPC even when build is in error

cmd/skaffold/app/cmd/cmd.go

@@ -39,17 +39,17 @@ import (
 )
 
 var (
-	opts         config.SkaffoldOptions
-	v            string
-	forceColors  bool
-	defaultColor int
-	overwrite    bool
+	opts              config.SkaffoldOptions
+	v                 string
+	forceColors       bool
+	defaultColor      int
+	overwrite         bool
+	shutdownAPIServer func() error
 )
 
 func NewSkaffoldCommand(out, err io.Writer) *cobra.Command {
 	updateMsg := make(chan string)
 	surveyPrompt := make(chan bool)
-	var shutdownAPIServer func() error
 
 	rootCmd := &cobra.Command{
 		Use: "skaffold",
@@ -115,10 +115,6 @@ func NewSkaffoldCommand(out, err io.Writer) *cobra.Command {
 				}
 			default:
 			}
-
-			if shutdownAPIServer != nil {
-				shutdownAPIServer()
-			}
 		},
 	}
 

cmd/skaffold/app/cmd/commands.go

@@ -88,15 +88,27 @@ func (b *builder) Hidden() Builder {
 func (b *builder) ExactArgs(argCount int, action func(context.Context, io.Writer, []string) error) *cobra.Command {
 	b.cmd.Args = cobra.ExactArgs(argCount)
 	b.cmd.RunE = func(_ *cobra.Command, args []string) error {
-		return handleWellKnownErrors(action(b.cmd.Context(), b.cmd.OutOrStdout(), args))
+		err := handleWellKnownErrors(action(b.cmd.Context(), b.cmd.OutOrStdout(), args))
+		// clean up server at end of the execution since post run hooks are only executed if
+		// RunE is successful
+		if shutdownAPIServer != nil {
+			shutdownAPIServer()
+		}
+		return err
 	}
 	return &b.cmd
 }
 
 func (b *builder) NoArgs(action func(context.Context, io.Writer) error) *cobra.Command {
 	b.cmd.Args = cobra.NoArgs
 	b.cmd.RunE = func(*cobra.Command, []string) error {
-		return handleWellKnownErrors(action(b.cmd.Context(), b.cmd.OutOrStdout()))
+		err := handleWellKnownErrors(action(b.cmd.Context(), b.cmd.OutOrStdout()))
+		// clean up server at end of the execution since post run hooks are only executed if
+		// RunE is successful
+		if shutdownAPIServer != nil {
+			shutdownAPIServer()
+		}
+		return err
 	}
 	return &b.cmd
 }

pkg/skaffold/server/server.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"time"
 
 	"github.com/grpc-ecosystem/grpc-gateway/runtime"
 	"github.com/sirupsen/logrus"
@@ -34,7 +35,12 @@ import (
 	"github.com/GoogleContainerTools/skaffold/proto"
 )
 
-var srv *server
+var (
+	srv *server
+
+	// waits for 1 second before forcing a server shutdown
+	forceShutdownTimeout = 1 * time.Second
+)
 
 type server struct {
 	buildIntentCallback  func()
@@ -155,8 +161,21 @@ func newGRPCServer(port int) (func() error, error) {
 		}
 	}()
 	return func() error {
-		s.Stop()
-		return l.Close()
+		ctx, cancel := context.WithTimeout(context.Background(), forceShutdownTimeout)
+		defer cancel()
+		ch := make(chan bool, 1)
+		go func() {
+			s.GracefulStop()
+			ch <- true
+		}()
+		for {
+			select {
+			case <-ctx.Done():
+				return l.Close()
+			case <-ch:
+				return l.Close()
+			}
+		}
 	}, nil
 }
 
@@ -174,9 +193,17 @@ func newHTTPServer(port, proxyPort int) (func() error, error) {
 	}
 	logrus.Infof("starting gRPC HTTP server on port %d", port)
 
-	go http.Serve(l, mux)
+	server := &http.Server{
+		Handler: mux,
+	}
 
-	return l.Close, nil
+	go server.Serve(l)
+
+	return func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), forceShutdownTimeout)
+		defer cancel()
+		return server.Shutdown(ctx)
+	}, nil
 }
 
 type errResponse struct {