Pre-update lifecycle hook

docs/lifecycle-hooks.md

@@ -3,6 +3,9 @@
 > **DO NOTE**: These are shell commands executed with `sh`, and therefore require the
 > container to provide the `sh` executable.
 
+> **DO NOTE**: If the container is not running then lifecycle hooks can not run and therefore 
+> the update is executed without running any lifecycle hooks.
+
 It is possible to execute _pre/post\-check_ and _pre/post\-update_ commands
 **inside** every container updated by watchtower.
 
@@ -62,5 +65,5 @@ If the label value is explicitly set to `0`, the timeout will be disabled.
 ### Execution failure
 
 The failure of a command to execute, identified by an exit code different than
-0, will not prevent watchtower from updating the container. Only an error
+0 or 75 (EX_TEMPFAIL), will not prevent watchtower from updating the container. Only an error
 log statement containing the exit code will be reported.

internal/actions/mocks/client.go

@@ -74,8 +74,8 @@ func (client MockClient) GetContainer(containerID string) (container.Container,
 }
 
 // ExecuteCommand is a mock method
-func (client MockClient) ExecuteCommand(containerID string, command string, timeout int) error {
-	return nil
+func (client MockClient) ExecuteCommand(containerID string, command string, timeout int) (bool,error) {
+	return false, nil
 }
 
 // IsContainerStale is always true for the mock client

internal/actions/update.go

@@ -69,8 +69,9 @@ func Update(client container.Client, params types.UpdateParams) (*metrics2.Metri
 	if params.RollingRestart {
 		metric.Failed += performRollingRestart(containersToUpdate, client, params)
 	} else {
-		metric.Failed += stopContainersInReversedOrder(containersToUpdate, client, params)
-		metric.Failed += restartContainersInSortedOrder(containersToUpdate, client, params)
+		imageIDsOfStoppedContainers := make(map[string]bool)
+		metric.Failed,imageIDsOfStoppedContainers = stopContainersInReversedOrder(containersToUpdate, client, params)
+		metric.Failed += restartContainersInSortedOrder(containersToUpdate, client, params, imageIDsOfStoppedContainers)
 	}
 
 	metric.Updated = staleCount - (metric.Failed - staleCheckFailed)
@@ -87,13 +88,15 @@ func performRollingRestart(containers []container.Container, client container.Cl
 
 	for i := len(containers) - 1; i >= 0; i-- {
 		if containers[i].Stale {
-			if err := stopStaleContainer(containers[i], client, params); err != nil {
-				failed++
-			}
-			if err := restartStaleContainer(containers[i], client, params); err != nil {
+			err := stopStaleContainer(containers[i], client, params)
+			if err != nil {
 				failed++
+			} else {
+				if err := restartStaleContainer(containers[i], client, params); err != nil {
+					failed++
+				}
+				cleanupImageIDs[containers[i].ImageID()] = true
 			}
-			cleanupImageIDs[containers[i].ImageID()] = true
 		}
 	}
 
@@ -103,14 +106,18 @@ func performRollingRestart(containers []container.Container, client container.Cl
 	return failed
 }
 
-func stopContainersInReversedOrder(containers []container.Container, client container.Client, params types.UpdateParams) int {
+func stopContainersInReversedOrder(containers []container.Container, client container.Client, params types.UpdateParams) (int, map[string]bool) {
+	imageIDsOfStoppedContainers := make(map[string]bool)
 	failed := 0
 	for i := len(containers) - 1; i >= 0; i-- {
 		if err := stopStaleContainer(containers[i], client, params); err != nil {
 			failed++
+		} else {
+			imageIDsOfStoppedContainers[containers[i].ImageID()] = true
 		}
+
 	}
-	return failed
+	return failed, imageIDsOfStoppedContainers
 }
 
 func stopStaleContainer(container container.Container, client container.Client, params types.UpdateParams) error {
@@ -123,11 +130,16 @@ func stopStaleContainer(container container.Container, client container.Client,
 		return nil
 	}
 	if params.LifecycleHooks {
-		if err := lifecycle.ExecutePreUpdateCommand(client, container); err != nil {
+		SkipUpdate,err := lifecycle.ExecutePreUpdateCommand(client, container)
+		if  err != nil {
 			log.Error(err)
 			log.Info("Skipping container as the pre-update command failed")
 			return err
 		}
+		if SkipUpdate {
+			log.Debug("Skipping container as the pre-update command returned exit code 75 (EX_TEMPFAIL)")
+			return errors.New("Skipping container as the pre-update command returned exit code 75 (EX_TEMPFAIL)")
+		}
 	}
 
 	if err := client.StopContainer(container, params.Timeout); err != nil {
@@ -137,7 +149,7 @@ func stopStaleContainer(container container.Container, client container.Client,
 	return nil
 }
 
-func restartContainersInSortedOrder(containers []container.Container, client container.Client, params types.UpdateParams) int {
+func restartContainersInSortedOrder(containers []container.Container, client container.Client, params types.UpdateParams, imageIDsOfStoppedContainers map[string]bool) int {
 	imageIDs := make(map[string]bool)
 
 	failed := 0
@@ -146,10 +158,12 @@ func restartContainersInSortedOrder(containers []container.Container, client con
 		if !c.Stale {
 			continue
 		}
-		if err := restartStaleContainer(c, client, params); err != nil {
-			failed++
+		if imageIDsOfStoppedContainers[c.ImageID()] {
+			if err := restartStaleContainer(c, client, params); err != nil {
+				failed++
+			}
+			imageIDs[c.ImageID()] = true
 		}
-		imageIDs[c.ImageID()] = true
 	}
 
 	if params.Cleanup {

pkg/container/client.go

@@ -31,7 +31,7 @@ type Client interface {
 	StartContainer(Container) (string, error)
 	RenameContainer(Container, string) error
 	IsContainerStale(Container) (bool, error)
-	ExecuteCommand(containerID string, command string, timeout int) error
+	ExecuteCommand(containerID string, command string, timeout int) (bool,error)
 	RemoveImageByID(string) error
 }
 
@@ -338,7 +338,7 @@ func (client dockerClient) RemoveImageByID(id string) error {
 	return err
 }
 
-func (client dockerClient) ExecuteCommand(containerID string, command string, timeout int) error {
+func (client dockerClient) ExecuteCommand(containerID string, command string, timeout int) (bool,error) {
 	bg := context.Background()
 
 	// Create the exec
@@ -350,7 +350,7 @@ func (client dockerClient) ExecuteCommand(containerID string, command string, ti
 
 	exec, err := client.api.ContainerExecCreate(bg, containerID, execConfig)
 	if err != nil {
-		return err
+		return false,err
 	}
 
 	response, attachErr := client.api.ContainerExecAttach(bg, exec.ID, types.ExecStartCheck{
@@ -365,7 +365,7 @@ func (client dockerClient) ExecuteCommand(containerID string, command string, ti
 	execStartCheck := types.ExecStartCheck{Detach: false, Tty: true}
 	err = client.api.ContainerExecStart(bg, exec.ID, execStartCheck)
 	if err != nil {
-		return err
+		return false,err
 	}
 
 	var output string
@@ -382,15 +382,15 @@ func (client dockerClient) ExecuteCommand(containerID string, command string, ti
 
 	// Inspect the exec to get the exit code and print a message if the
 	// exit code is not success.
-	err = client.waitForExecOrTimeout(bg, exec.ID, output, timeout)
+	skipUpdate, err  := client.waitForExecOrTimeout(bg, exec.ID, output, timeout)
 	if err != nil {
-		return err
+		return true,err 
 	}
 
-	return nil
+	return skipUpdate,nil
 }
 
-func (client dockerClient) waitForExecOrTimeout(bg context.Context, ID string, execOutput string, timeout int) error {
+func (client dockerClient) waitForExecOrTimeout(bg context.Context, ID string, execOutput string, timeout int) (bool,error) {
 	var ctx context.Context
 	var cancel context.CancelFunc
 
@@ -411,7 +411,7 @@ func (client dockerClient) waitForExecOrTimeout(bg context.Context, ID string, e
 		}).Debug("Awaiting timeout or completion")
 
 		if err != nil {
-			return err
+			return false,err
 		}
 		if execInspect.Running == true {
 			time.Sleep(1 * time.Second)
@@ -420,13 +420,17 @@ func (client dockerClient) waitForExecOrTimeout(bg context.Context, ID string, e
 		if len(execOutput) > 0 {
 			log.Infof("Command output:\n%v", execOutput)
 		}
-		if execInspect.ExitCode > 0 {
-			log.Errorf("Command exited with code %v.", execInspect.ExitCode)
-			log.Error(execOutput)
+
+		if execInspect.ExitCode == 75{
+			return true,nil
+		}
+
+		if execInspect.ExitCode > 0  {
+			return false,fmt.Errorf("Command exited with code %v  %s", execInspect.ExitCode, execOutput)
 		}
 		break
 	}
-	return nil
+	return false,nil
 }
 
 func (client dockerClient) waitForStopOrTimeout(c Container, waitTime time.Duration) error {

pkg/container/container.go

@@ -43,7 +43,10 @@ func (c Container) ID() string {
 // container is running. The status is determined by the value of the
 // container's "State.Running" property.
 func (c Container) IsRunning() bool {
-	return c.containerInfo.State.Running
+	if c.containerInfo.State != nil {
+		return c.containerInfo.State.Running
+	}
+	return false
 }
 
 // Name returns the Docker container name.

pkg/lifecycle/lifecycle.go

@@ -37,7 +37,8 @@ func ExecutePreCheckCommand(client container.Client, container container.Contain
 	}
 
 	log.Debug("Executing pre-check command.")
-	if err := client.ExecuteCommand(container.ID(), command, 1); err != nil {
+	_,err := client.ExecuteCommand(container.ID(), command, 1);
+	if  err != nil {
 		log.Error(err)
 	}
 }
@@ -51,18 +52,24 @@ func ExecutePostCheckCommand(client container.Client, container container.Contai
 	}
 
 	log.Debug("Executing post-check command.")
-	if err := client.ExecuteCommand(container.ID(), command, 1); err != nil {
+	_,err := client.ExecuteCommand(container.ID(), command, 1);
+	if err != nil {
 		log.Error(err)
 	}
 }
 
 // ExecutePreUpdateCommand tries to run the pre-update lifecycle hook for a single container.
-func ExecutePreUpdateCommand(client container.Client, container container.Container) error {
+func ExecutePreUpdateCommand(client container.Client, container container.Container) (bool,error) {
 	timeout := container.PreUpdateTimeout()
 	command := container.GetLifecyclePreUpdateCommand()
 	if len(command) == 0 {
 		log.Debug("No pre-update command supplied. Skipping")
-		return nil
+		return false,nil
+	}
+
+	if !container.IsRunning() {
+		log.Debug("Container is not running. Skipping pre-update command.")
+		return false,nil
 	}
 
 	log.Debug("Executing pre-update command.")
@@ -84,7 +91,9 @@ func ExecutePostUpdateCommand(client container.Client, newContainerID string) {
 	}
 
 	log.Debug("Executing post-update command.")
-	if err := client.ExecuteCommand(newContainerID, command, 1); err != nil {
+	_,err = client.ExecuteCommand(newContainerID, command, 1);
+
+	if  err != nil {
 		log.Error(err)
 	}
 }