auto-update: simple rollback

cmd/podman/auto-update.go

@@ -54,6 +54,7 @@ func init() {
 	_ = autoUpdateCommand.RegisterFlagCompletionFunc(authfileFlagName, completion.AutocompleteDefault)
 
 	flags.BoolVar(&autoUpdateOptions.DryRun, "dry-run", false, "Check for pending updates")
+	flags.BoolVar(&autoUpdateOptions.Rollback, "rollback", true, "Rollback to previous image if update fails")
 
 	flags.StringVar(&autoUpdateOptions.format, "format", "", "Change the output format to JSON or a Go template")
 	_ = autoUpdateCommand.RegisterFlagCompletionFunc("format", common.AutocompleteFormat(autoUpdateOutput{}))

docs/source/markdown/podman-auto-update.1.md

@@ -51,6 +51,15 @@ The `UPDATED` field indicates the availability of a new image with "pending".
 Change the default output format.  This can be of a supported type like 'json' or a Go template.
 Valid placeholders for the Go template are listed below:
 
+#### **--rollback**=*true|false*
+
+If restarting a systemd unit after updating the image has failed, rollback to using the previous image and restart the unit another time.  Default is true.
+
+Please note that detecting if a systemd unit has failed is best done by the container sending the READY message via SDNOTIFY.  This way, restarting the unit will wait until having received the message or a timeout kicked in.  Without that, restarting the systemd unit may succeed even if the container has failed shortly after.
+
+For a container to send the READY message via SDNOTIFY it must be created with the `--sdnotify=container` option (see podman-run(1)).  The application running inside the container can then execute `systemd-notify --ready` when ready or use the sdnotify bindings of the specific programming language (e.g., sd_notify(3)).
+
+
 | **Placeholder** | **Description**                        |
 | --------------- | -------------------------------------- |
 | .Unit           | Name of the systemd unit               |
@@ -132,4 +141,4 @@ $ podman auto-update
 ```
 
 ## SEE ALSO
-**[podman(1)](podman.1.md)**, **[podman-generate-systemd(1)](podman-generate-systemd.1.md)**, **[podman-run(1)](podman-run.1.md)**, systemd.unit(5)
+**[podman(1)](podman.1.md)**, **[podman-generate-systemd(1)](podman-generate-systemd.1.md)**, **[podman-run(1)](podman-run.1.md)**, sd_notify(3), systemd.unit(5)

pkg/autoupdate/autoupdate.go

@@ -88,7 +88,7 @@ func ValidateImageReference(imageName string) error {
 	} else if err != nil {
 		repo, err := reference.Parse(imageName)
 		if err != nil {
-			return errors.Wrap(err, "error enforcing fully-qualified docker transport reference for auto updates")
+			return errors.Wrap(err, "enforcing fully-qualified docker transport reference for auto updates")
 		}
 		if _, ok := repo.(reference.NamedTagged); !ok {
 			return errors.Errorf("auto updates require fully-qualified image references (no tag): %q", imageName)
@@ -181,13 +181,13 @@ func autoUpdateRegistry(ctx context.Context, image *libimage.Image, ctr *libpod.
 	cid := ctr.ID()
 	rawImageName := ctr.RawImageName()
 	if rawImageName == "" {
-		return nil, errors.Errorf("error registry auto-updating container %q: raw-image name is empty", cid)
+		return nil, errors.Errorf("registry auto-updating container %q: raw-image name is empty", cid)
 	}
 
 	labels := ctr.Labels()
 	unit, exists := labels[systemdDefine.EnvVariable]
 	if !exists {
-		return nil, errors.Errorf("error auto-updating container %q: no %s label found", ctr.ID(), systemdDefine.EnvVariable)
+		return nil, errors.Errorf("auto-updating container %q: no %s label found", ctr.ID(), systemdDefine.EnvVariable)
 	}
 
 	report := &entities.AutoUpdateReport{
@@ -201,7 +201,7 @@ func autoUpdateRegistry(ctx context.Context, image *libimage.Image, ctr *libpod.
 
 	if _, updated := updatedRawImages[rawImageName]; updated {
 		logrus.Infof("Auto-updating container %q using registry image %q", cid, rawImageName)
-		if err := restartSystemdUnit(ctr, unit, conn); err != nil {
+		if err := restartSystemdUnit(ctx, ctr, unit, conn); err != nil {
 			return report, err
 		}
 		report.Updated = "true"
@@ -211,7 +211,7 @@ func autoUpdateRegistry(ctx context.Context, image *libimage.Image, ctr *libpod.
 	authfile := getAuthfilePath(ctr, options)
 	needsUpdate, err := newerRemoteImageAvailable(ctx, runtime, image, rawImageName, authfile)
 	if err != nil {
-		return report, errors.Wrapf(err, "error registry auto-updating container %q: image check for %q failed", cid, rawImageName)
+		return report, errors.Wrapf(err, "registry auto-updating container %q: image check for %q failed", cid, rawImageName)
 	}
 
 	if !needsUpdate {
@@ -225,16 +225,30 @@ func autoUpdateRegistry(ctx context.Context, image *libimage.Image, ctr *libpod.
 	}
 
 	if _, err := updateImage(ctx, runtime, rawImageName, options); err != nil {
-		return report, errors.Wrapf(err, "error registry auto-updating container %q: image update for %q failed", cid, rawImageName)
+		return report, errors.Wrapf(err, "registry auto-updating container %q: image update for %q failed", cid, rawImageName)
 	}
 	updatedRawImages[rawImageName] = true
 
 	logrus.Infof("Auto-updating container %q using registry image %q", cid, rawImageName)
-	if err := restartSystemdUnit(ctr, unit, conn); err != nil {
-		return report, err
+	updateErr := restartSystemdUnit(ctx, ctr, unit, conn)
+	if updateErr == nil {
+		report.Updated = "true"
+		return report, nil
+	}
+
+	if !options.Rollback {
+		return report, updateErr
+	}
+
+	// To fallback, simply retag the old image and restart the service.
+	if err := image.Tag(rawImageName); err != nil {
+		return report, errors.Wrap(err, "falling back to previous image")
+	}
+	if err := restartSystemdUnit(ctx, ctr, unit, conn); err != nil {
+		return report, errors.Wrap(err, "restarting unit with old image during fallback")
 	}
 
-	report.Updated = "true"
+	report.Updated = "rolled back"
 	return report, nil
 }
 
@@ -243,13 +257,13 @@ func autoUpdateLocally(ctx context.Context, image *libimage.Image, ctr *libpod.C
 	cid := ctr.ID()
 	rawImageName := ctr.RawImageName()
 	if rawImageName == "" {
-		return nil, errors.Errorf("error locally auto-updating container %q: raw-image name is empty", cid)
+		return nil, errors.Errorf("locally auto-updating container %q: raw-image name is empty", cid)
 	}
 
 	labels := ctr.Labels()
 	unit, exists := labels[systemdDefine.EnvVariable]
 	if !exists {
-		return nil, errors.Errorf("error auto-updating container %q: no %s label found", ctr.ID(), systemdDefine.EnvVariable)
+		return nil, errors.Errorf("auto-updating container %q: no %s label found", ctr.ID(), systemdDefine.EnvVariable)
 	}
 
 	report := &entities.AutoUpdateReport{
@@ -263,7 +277,7 @@ func autoUpdateLocally(ctx context.Context, image *libimage.Image, ctr *libpod.C
 
 	needsUpdate, err := newerLocalImageAvailable(runtime, image, rawImageName)
 	if err != nil {
-		return report, errors.Wrapf(err, "error locally auto-updating container %q: image check for %q failed", cid, rawImageName)
+		return report, errors.Wrapf(err, "locally auto-updating container %q: image check for %q failed", cid, rawImageName)
 	}
 
 	if !needsUpdate {
@@ -277,23 +291,47 @@ func autoUpdateLocally(ctx context.Context, image *libimage.Image, ctr *libpod.C
 	}
 
 	logrus.Infof("Auto-updating container %q using local image %q", cid, rawImageName)
-	if err := restartSystemdUnit(ctr, unit, conn); err != nil {
-		return report, err
+	updateErr := restartSystemdUnit(ctx, ctr, unit, conn)
+	if updateErr == nil {
+		report.Updated = "true"
+		return report, nil
 	}
 
-	report.Updated = "true"
+	if !options.Rollback {
+		return report, updateErr
+	}
+
+	// To fallback, simply retag the old image and restart the service.
+	if err := image.Tag(rawImageName); err != nil {
+		return report, errors.Wrap(err, "falling back to previous image")
+	}
+	if err := restartSystemdUnit(ctx, ctr, unit, conn); err != nil {
+		return report, errors.Wrap(err, "restarting unit with old image during fallback")
+	}
+
+	report.Updated = "rolled back"
 	return report, nil
 }
 
 // restartSystemdUnit restarts the systemd unit the container is running in.
-func restartSystemdUnit(ctr *libpod.Container, unit string, conn *dbus.Conn) error {
-	_, err := conn.RestartUnit(unit, "replace", nil)
-	if err != nil {
-		return errors.Wrapf(err, "error auto-updating container %q: restarting systemd unit %q failed", ctr.ID(), unit)
+func restartSystemdUnit(ctx context.Context, ctr *libpod.Container, unit string, conn *dbus.Conn) error {
+	restartChan := make(chan string)
+	if _, err := conn.RestartUnitContext(ctx, unit, "replace", restartChan); err != nil {
+		return errors.Wrapf(err, "auto-updating container %q: restarting systemd unit %q failed", ctr.ID(), unit)
 	}
 
-	logrus.Infof("Successfully restarted systemd unit %q of container %q", unit, ctr.ID())
-	return nil
+	// Wait for the restart to finish and actually check if it was
+	// successful or not.
+	result := <-restartChan
+
+	switch result {
+	case "done":
+		logrus.Infof("Successfully restarted systemd unit %q of container %q", unit, ctr.ID())
+		return nil
+
+	default:
+		return errors.Errorf("auto-updating container %q: restarting systemd unit %q failed: expected %q but received %q", ctr.ID(), unit, "done", result)
+	}
 }
 
 // imageContainersMap generates a map[image ID] -> [containers using the image]

pkg/domain/entities/auto-update.go

@@ -8,6 +8,9 @@ type AutoUpdateOptions struct {
 	// pending, it will be indicated in the Updated field of
 	// AutoUpdateReport.
 	DryRun bool
+	// If restarting the service with the new image failed, restart it
+	// another time with the previous image.
+	Rollback bool
 }
 
 // AutoUpdateReport contains the results from running auto-update.

pkg/systemd/generate/common.go

@@ -74,8 +74,7 @@ func filterCommonContainerFlags(command []string, argCount int) []string {
 		case s == "--sdnotify", s == "--cgroups":
 			i++
 			continue
-		case strings.HasPrefix(s, "--sdnotify="),
-			strings.HasPrefix(s, "--rm="),
+		case strings.HasPrefix(s, "--rm="),
 			strings.HasPrefix(s, "--cgroups="):
 			continue
 		}

pkg/systemd/generate/containers.go

@@ -258,7 +258,6 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
 		}
 		startCommand = append(startCommand,
 			"run",
-			"--sdnotify=conmon",
 			"--cgroups=no-conmon",
 			"--rm",
 		)
@@ -273,6 +272,7 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
 		fs.String("name", "", "")
 		fs.Bool("replace", false, "")
 		fs.StringArrayP("env", "e", nil, "")
+		fs.String("sdnotify", "", "")
 		fs.Parse(remainingCmd)
 
 		remainingCmd = filterCommonContainerFlags(remainingCmd, fs.NArg())
@@ -294,6 +294,13 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
 			return "", err
 		}
 
+		// Default to --sdnotify=conmon unless already set by the
+		// container.
+		hasSdnotifyParam := fs.Lookup("sdnotify").Changed
+		if !hasSdnotifyParam {
+			startCommand = append(startCommand, "--sdnotify=conmon")
+		}
+
 		if !hasDetachParam {
 			// Enforce detaching
 			//