Merge pull request #15248 from vrothberg/RUN-1606

kube play: sd-notify integration

cmd/podman/kube/play.go

@@ -182,6 +182,10 @@ func playFlags(cmd *cobra.Command) {
 }
 
 func Play(cmd *cobra.Command, args []string) error {
+	if playOptions.ServiceContainer && !playOptions.StartCLI { // Sanity check to be future proof
+		return fmt.Errorf("--service-container does not work with --start=stop")
+	}
+
 	// TLS verification in c/image is controlled via a `types.OptionalBool`
 	// which allows for distinguishing among set-true, set-false, unspecified
 	// which is important to implement a sane way of dealing with defaults of

libpod/container.go

@@ -124,10 +124,6 @@ type Container struct {
 	// This is true if a container is restored from a checkpoint.
 	restoreFromCheckpoint bool
 
-	// Used to query the NOTIFY_SOCKET once along with setting up
-	// mounts etc.
-	notifySocket string
-
 	slirp4netnsSubnet *net.IPNet
 }
 

libpod/container_config.go

@@ -386,6 +386,8 @@ type ContainerMiscConfig struct {
 	IsService bool `json:"isService"`
 	// SdNotifyMode tells libpod what to do with a NOTIFY_SOCKET if passed
 	SdNotifyMode string `json:"sdnotifyMode,omitempty"`
+	// SdNotifySocket stores NOTIFY_SOCKET in use by the container
+	SdNotifySocket string `json:"sdnotifySocket,omitempty"`
 	// Systemd tells libpod to set up the container in systemd mode, a value of nil denotes false
 	Systemd *bool `json:"systemd,omitempty"`
 	// HealthCheckConfig has the health check command and related timings

libpod/container_inspect.go

@@ -414,6 +414,8 @@ func (c *Container) generateInspectContainerConfig(spec *spec.Spec) *define.Insp
 	ctrConfig.Passwd = c.config.Passwd
 	ctrConfig.ChrootDirs = append(ctrConfig.ChrootDirs, c.config.ChrootDirs...)
 
+	ctrConfig.SdNotifyMode = c.config.SdNotifyMode
+	ctrConfig.SdNotifySocket = c.config.SdNotifySocket
 	return ctrConfig
 }
 

libpod/container_internal.go

@@ -31,6 +31,7 @@ import (
 	"github.com/containers/podman/v4/pkg/lookup"
 	"github.com/containers/podman/v4/pkg/rootless"
 	"github.com/containers/podman/v4/pkg/selinux"
+	"github.com/containers/podman/v4/pkg/systemd/notifyproxy"
 	"github.com/containers/podman/v4/pkg/util"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/archive"
@@ -1224,9 +1225,9 @@ func (c *Container) start() error {
 			payload += "\n"
 			payload += daemon.SdNotifyReady
 		}
-		if sent, err := daemon.SdNotify(false, payload); err != nil {
+		if err := notifyproxy.SendMessage(c.config.SdNotifySocket, payload); err != nil {
 			logrus.Errorf("Notifying systemd of Conmon PID: %s", err.Error())
-		} else if sent {
+		} else {
 			logrus.Debugf("Notify sent successfully")
 		}
 	}

libpod/container_internal_linux.go

@@ -971,12 +971,9 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 // and if the sdnotify mode is set to container.  It also sets c.notifySocket
 // to avoid redundantly looking up the env variable.
 func (c *Container) mountNotifySocket(g generate.Generator) error {
-	notify, ok := os.LookupEnv("NOTIFY_SOCKET")
-	if !ok {
+	if c.config.SdNotifySocket == "" {
 		return nil
 	}
-	c.notifySocket = notify
-
 	if c.config.SdNotifyMode != define.SdNotifyModeContainer {
 		return nil
 	}

libpod/container_validate.go

@@ -133,5 +133,9 @@ func (c *Container) validate() error {
 	if len(c.config.InitContainerType) > 0 && len(c.config.Pod) < 1 {
 		return fmt.Errorf("init containers must be created in a pod: %w", define.ErrInvalidArg)
 	}
+
+	if c.config.SdNotifyMode == define.SdNotifyModeIgnore && len(c.config.SdNotifySocket) > 0 {
+		return fmt.Errorf("cannot set sd-notify socket %q with sd-notify mode %q", c.config.SdNotifySocket, c.config.SdNotifyMode)
+	}
 	return nil
 }

libpod/define/config.go

@@ -81,13 +81,6 @@ const NoLogging = "none"
 // PassthroughLogging is the string conmon expects when specifying to use the passthrough driver
 const PassthroughLogging = "passthrough"
 
-// Strings used for --sdnotify option to podman
-const (
-	SdNotifyModeContainer = "container"
-	SdNotifyModeConmon    = "conmon"
-	SdNotifyModeIgnore    = "ignore"
-)
-
 // DefaultRlimitValue is the value set by default for nofile and nproc
 const RLimitDefaultValue = uint64(1048576)
 

libpod/define/container_inspect.go

@@ -79,6 +79,10 @@ type InspectContainerConfig struct {
 	// treated as root directories. Standard bind mounts will be mounted
 	// into paths relative to these directories.
 	ChrootDirs []string `json:"ChrootDirs,omitempty"`
+	// SdNotifyMode is the sd-notify mode of the container.
+	SdNotifyMode string `json:"sdNotifyMode,omitempty"`
+	// SdNotifySocket is the NOTIFY_SOCKET in use by/configured for the container.
+	SdNotifySocket string `json:"sdNotifySocket,omitempty"`
 }
 
 // InspectRestartPolicy holds information about the container's restart policy.

libpod/define/sdnotify.go

@@ -0,0 +1,20 @@
+package define
+
+import "fmt"
+
+// Strings used for --sdnotify option to podman
+const (
+	SdNotifyModeContainer = "container"
+	SdNotifyModeConmon    = "conmon"
+	SdNotifyModeIgnore    = "ignore"
+)
+
+// ValidateSdNotifyMode validates the specified mode.
+func ValidateSdNotifyMode(mode string) error {
+	switch mode {
+	case "", SdNotifyModeContainer, SdNotifyModeConmon, SdNotifyModeIgnore:
+		return nil
+	default:
+		return fmt.Errorf("%w: invalid sdnotify value %q: must be %s, %s or %s", ErrInvalidArg, mode, SdNotifyModeContainer, SdNotifyModeConmon, SdNotifyModeIgnore)
+	}
+}

libpod/oci_conmon_linux.go

@@ -1062,8 +1062,8 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co
 
 	args := r.sharedConmonArgs(ctr, ctr.ID(), ctr.bundlePath(), pidfile, ctr.LogPath(), r.exitsDir, ociLog, ctr.LogDriver(), logTag)
 
-	if ctr.config.SdNotifyMode == define.SdNotifyModeContainer && ctr.notifySocket != "" {
-		args = append(args, fmt.Sprintf("--sdnotify-socket=%s", ctr.notifySocket))
+	if ctr.config.SdNotifyMode == define.SdNotifyModeContainer && ctr.config.SdNotifySocket != "" {
+		args = append(args, fmt.Sprintf("--sdnotify-socket=%s", ctr.config.SdNotifySocket))
 	}
 
 	if ctr.config.Spec.Process.Terminal {
@@ -1391,14 +1391,13 @@ func startCommand(cmd *exec.Cmd, ctr *Container) error {
 	// Make sure to unset the NOTIFY_SOCKET and reset it afterwards if needed.
 	switch ctr.config.SdNotifyMode {
 	case define.SdNotifyModeContainer, define.SdNotifyModeIgnore:
-		if ctr.notifySocket != "" {
+		if prev := os.Getenv("NOTIFY_SOCKET"); prev != "" {
 			if err := os.Unsetenv("NOTIFY_SOCKET"); err != nil {
 				logrus.Warnf("Error unsetting NOTIFY_SOCKET %v", err)
 			}
-
 			defer func() {
-				if err := os.Setenv("NOTIFY_SOCKET", ctr.notifySocket); err != nil {
-					logrus.Errorf("Resetting NOTIFY_SOCKET=%s", ctr.notifySocket)
+				if err := os.Setenv("NOTIFY_SOCKET", prev); err != nil {
+					logrus.Errorf("Resetting NOTIFY_SOCKET=%s", prev)
 				}
 			}()
 		}

libpod/options.go

@@ -6,14 +6,12 @@ import (
 	"net"
 	"os"
 	"path/filepath"
-	"strings"
 	"syscall"
 
 	"github.com/containers/buildah/pkg/parse"
 	nettypes "github.com/containers/common/libnetwork/types"
 	"github.com/containers/common/pkg/config"
 	"github.com/containers/common/pkg/secrets"
-	cutil "github.com/containers/common/pkg/util"
 	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/podman/v4/libpod/define"
@@ -29,12 +27,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// Runtime Creation Options
-var (
-	// SdNotifyModeValues describes the only values that SdNotifyMode can be
-	SdNotifyModeValues = []string{define.SdNotifyModeContainer, define.SdNotifyModeConmon, define.SdNotifyModeIgnore}
-)
-
 // WithStorageConfig uses the given configuration to set up container storage.
 // If this is not specified, the system default configuration will be used
 // instead.
@@ -613,16 +605,26 @@ func WithSystemd() CtrCreateOption {
 	}
 }
 
+// WithSdNotifySocket sets the sd-notify of the container
+func WithSdNotifySocket(socketPath string) CtrCreateOption {
+	return func(ctr *Container) error {
+		if ctr.valid {
+			return define.ErrCtrFinalized
+		}
+		ctr.config.SdNotifySocket = socketPath
+		return nil
+	}
+}
+
 // WithSdNotifyMode sets the sd-notify method
 func WithSdNotifyMode(mode string) CtrCreateOption {
 	return func(ctr *Container) error {
 		if ctr.valid {
 			return define.ErrCtrFinalized
 		}
 
-		// verify values
-		if len(mode) > 0 && !cutil.StringInSlice(strings.ToLower(mode), SdNotifyModeValues) {
-			return fmt.Errorf("--sdnotify values must be one of %q: %w", strings.Join(SdNotifyModeValues, ", "), define.ErrInvalidArg)
+		if err := define.ValidateSdNotifyMode(mode); err != nil {
+			return err
 		}
 
 		ctr.config.SdNotifyMode = mode

pkg/domain/infra/abi/play.go

@@ -27,13 +27,19 @@ import (
 	"github.com/containers/podman/v4/pkg/specgen/generate"
 	"github.com/containers/podman/v4/pkg/specgen/generate/kube"
 	"github.com/containers/podman/v4/pkg/specgenutil"
+	"github.com/containers/podman/v4/pkg/systemd/notifyproxy"
 	"github.com/containers/podman/v4/pkg/util"
+	"github.com/coreos/go-systemd/v22/daemon"
 	"github.com/ghodss/yaml"
 	"github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
 	yamlv3 "gopkg.in/yaml.v3"
 )
 
+// sdNotifyAnnotation allows for configuring service-global and
+// container-specific sd-notify modes.
+const sdNotifyAnnotation = "io.containers.sdnotify"
+
 // createServiceContainer creates a container that can later on
 // be associated with the pods of a K8s yaml.  It will be started along with
 // the first pod.
@@ -73,7 +79,12 @@ func (ic *ContainerEngine) createServiceContainer(ctx context.Context, name stri
 		return nil, fmt.Errorf("creating runtime spec for service container: %w", err)
 	}
 	opts = append(opts, libpod.WithIsService())
-	opts = append(opts, libpod.WithSdNotifyMode(define.SdNotifyModeConmon))
+
+	// Set the sd-notify mode to "ignore".  Podman is responsible for
+	// sending the notify messages when all containers are ready.
+	// The mode for individual containers or entire pods can be configured
+	// via the `sdNotifyAnnotation` annotation in the K8s YAML.
+	opts = append(opts, libpod.WithSdNotifyMode(define.SdNotifyModeIgnore))
 
 	// Create a new libpod container based on the spec.
 	ctr, err := ic.Libpod.NewContainer(ctx, runtimeSpec, spec, false, opts...)
@@ -96,6 +107,10 @@ func k8sName(content []byte, suffix string) string {
 }
 
 func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options entities.PlayKubeOptions) (_ *entities.PlayKubeReport, finalErr error) {
+	if options.ServiceContainer && options.Start == types.OptionalBoolFalse { // Sanity check to be future proof
+		return nil, fmt.Errorf("running a service container requires starting the pod(s)")
+	}
+
 	report := &entities.PlayKubeReport{}
 	validKinds := 0
 
@@ -121,6 +136,8 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options
 
 	var configMaps []v1.ConfigMap
 
+	ranContainers := false
+	var serviceContainer *libpod.Container
 	// create pod on each document if it is a pod or deployment
 	// any other kube kind will be skipped
 	for _, document := range documentList {
@@ -130,8 +147,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options
 		}
 
 		// TODO: create constants for the various "kinds" of yaml files.
-		var serviceContainer *libpod.Container
-		if options.ServiceContainer && (kind == "Pod" || kind == "Deployment") {
+		if options.ServiceContainer && serviceContainer == nil && (kind == "Pod" || kind == "Deployment") {
 			ctr, err := ic.createServiceContainer(ctx, k8sName(content, "service"), options)
 			if err != nil {
 				return nil, err
@@ -178,6 +194,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options
 
 			report.Pods = append(report.Pods, r.Pods...)
 			validKinds++
+			ranContainers = true
 		case "Deployment":
 			var deploymentYAML v1apps.Deployment
 
@@ -192,6 +209,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options
 
 			report.Pods = append(report.Pods, r.Pods...)
 			validKinds++
+			ranContainers = true
 		case "PersistentVolumeClaim":
 			var pvcYAML v1.PersistentVolumeClaim
 
@@ -239,6 +257,20 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options
 		return nil, fmt.Errorf("YAML document does not contain any supported kube kind")
 	}
 
+	if options.ServiceContainer && ranContainers {
+		// We can consider the service to be up and running now.
+		// Send the sd-notify messages pointing systemd to the
+		// service container.
+		data, err := serviceContainer.Inspect(false)
+		if err != nil {
+			return nil, err
+		}
+		message := fmt.Sprintf("MAINPID=%d\n%s", data.State.ConmonPid, daemon.SdNotifyReady)
+		if err := notifyproxy.SendMessage("", message); err != nil {
+			return nil, err
+		}
+	}
+
 	return report, nil
 }
 
@@ -280,6 +312,11 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		report      entities.PlayKubeReport
 	)
 
+	mainSdNotifyMode, err := getSdNotifyMode(annotations, "")
+	if err != nil {
+		return nil, err
+	}
+
 	// Create the secret manager before hand
 	secretsManager, err := ic.Libpod.SecretsManager()
 	if err != nil {
@@ -562,6 +599,9 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 
 		initContainers = append(initContainers, ctr)
 	}
+
+	var sdNotifyProxies []*notifyproxy.NotifyProxy // containers' sd-notify proxies
+
 	for _, container := range podYAML.Spec.Containers {
 		// Error out if the same name is used for more than one container
 		if _, ok := ctrNames[container.Name]; ok {
@@ -606,7 +646,31 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		if err != nil {
 			return nil, err
 		}
-		opts = append(opts, libpod.WithSdNotifyMode(define.SdNotifyModeIgnore))
+
+		sdNotifyMode := mainSdNotifyMode
+		ctrNotifyMode, err := getSdNotifyMode(annotations, container.Name)
+		if err != nil {
+			return nil, err
+		}
+		if ctrNotifyMode != "" {
+			sdNotifyMode = ctrNotifyMode
+		}
+		if sdNotifyMode == "" { // Default to "ignore"
+			sdNotifyMode = define.SdNotifyModeIgnore
+		}
+
+		opts = append(opts, libpod.WithSdNotifyMode(sdNotifyMode))
+
+		// Create a notify proxy for the container.
+		if sdNotifyMode != "" && sdNotifyMode != define.SdNotifyModeIgnore {
+			proxy, err := notifyproxy.New("")
+			if err != nil {
+				return nil, err
+			}
+			sdNotifyProxies = append(sdNotifyProxies, proxy)
+			opts = append(opts, libpod.WithSdNotifySocket(proxy.SocketPath()))
+		}
+
 		ctr, err := generate.ExecuteCreate(ctx, ic.Libpod, rtSpec, spec, false, opts...)
 		if err != nil {
 			return nil, err
@@ -624,6 +688,13 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			playKubePod.ContainerErrors = append(playKubePod.ContainerErrors, fmt.Errorf("error starting container %s: %w", id, err).Error())
 			fmt.Println(playKubePod.ContainerErrors)
 		}
+
+		// Wait for each proxy to receive a READY message.
+		for _, proxy := range sdNotifyProxies {
+			if err := proxy.WaitAndClose(); err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	playKubePod.ID = pod.ID()