Implement SD-NOTIFY proxy in conmon

This leverages conmon's ability to proxy the SD-NOTIFY socket. This prevents locking caused by OCI runtime blocking, waiting for SD-NOTIFY messages, and instead passes the messages directly up to the host. Signed-off-by: Joseph Gooch <[email protected]>
containers · Oct 28, 2020 · 7c4a4c4 · 7c4a4c4
1 parent 4d87306
commit 7c4a4c4
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 13 deletions.
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
@@ -349,6 +349,7 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 			newMount.Options = append(newMount.Options, "ro", "nosuid", "noexec", "nodev")
 		}
 		if !MountExists(g.Mounts(), dstPath) {
+			logrus.Warnf("Mounting %q to %q", srcPath, dstPath)
 			g.AddMount(newMount)
 		} else {
 			logrus.Warnf("User mount overriding libpod mount at %q", dstPath)
@@ -544,6 +545,11 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 		g.AddProcessEnv("container", "libpod")
 	}
 
+	// Set notify
+	if c.config.SdNotifyMode == define.SdNotifyModeContainer {
+		g.AddProcessEnv("NOTIFY_SOCKET", "/run/notify/notify.sock")
+	}
+
 	cgroupPath, err := c.getOCICgroupPath()
 	if err != nil {
 		return nil, err
@@ -1384,6 +1390,24 @@ func (c *Container) makeBindMounts() error {
 		}
 	}
 
+	// Bind notify proxy
+	if c.config.SdNotifyMode == define.SdNotifyModeContainer {
+		notifyDir := filepath.Join(c.bundlePath(), "notify")
+		logrus.Debugf("checking notify %q dir", notifyDir)
+		if err := os.MkdirAll(notifyDir, 0755); err != nil {
+			if !os.IsExist(err) {
+				return errors.Wrapf(err, "unable to create notify %q dir", notifyDir)
+			}
+		}
+		if err := label.Relabel(notifyDir, c.MountLabel(), true); err != nil {
+			return errors.Wrapf(err, "relabel failed %q", notifyDir)
+		}
+		logrus.Debugf("add bindmount notify %q dir", notifyDir)
+		if _, ok := c.state.BindMounts["/run/notify"]; !ok {
+			c.state.BindMounts["/run/notify"] = notifyDir
+		}
+	}
+
 	return nil
 }
 

diff --git a/libpod/diff.go b/libpod/diff.go
@@ -15,6 +15,7 @@ var containerMounts = map[string]bool{
 	"/etc/resolv.conf":   true,
 	"/proc":              true,
 	"/run":               true,
+	"/run/notify":        true,
 	"/run/.containerenv": true,
 	"/run/secrets":       true,
 	"/sys":               true,

diff --git a/libpod/oci_conmon_exec_linux.go b/libpod/oci_conmon_exec_linux.go
@@ -444,7 +444,7 @@ func (r *ConmonOCIRuntime) startExec(c *Container, sessionID string, options *Ex
 	// 	}
 	// }
 
-	conmonEnv, extraFiles, err := r.configureConmonEnv(c, runtimeDir)
+	conmonEnv, extraFiles, err := r.configureConmonEnv(runtimeDir)
 	if err != nil {
 		return nil, nil, err
 	}

diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go
@@ -364,11 +364,6 @@ func (r *ConmonOCIRuntime) StartContainer(ctr *Container) error {
 		return err
 	}
 	env := []string{fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir)}
-	if ctr.config.SdNotifyMode == define.SdNotifyModeContainer {
-		if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok {
-			env = append(env, fmt.Sprintf("NOTIFY_SOCKET=%s", notify))
-		}
-	}
 	if path, ok := os.LookupEnv("PATH"); ok {
 		env = append(env, fmt.Sprintf("PATH=%s", path))
 	}
@@ -1065,7 +1060,7 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co
 	}
 
 	// 0, 1 and 2 are stdin, stdout and stderr
-	conmonEnv, envFiles, err := r.configureConmonEnv(ctr, runtimeDir)
+	conmonEnv, envFiles, err := r.configureConmonEnv(runtimeDir)
 	if err != nil {
 		return err
 	}
@@ -1268,7 +1263,7 @@ func prepareProcessExec(c *Container, cmd, env []string, tty bool, cwd, user, se
 
 // configureConmonEnv gets the environment values to add to conmon's exec struct
 // TODO this may want to be less hardcoded/more configurable in the future
-func (r *ConmonOCIRuntime) configureConmonEnv(ctr *Container, runtimeDir string) ([]string, []*os.File, error) {
+func (r *ConmonOCIRuntime) configureConmonEnv(runtimeDir string) ([]string, []*os.File, error) {
 	env := make([]string, 0, 6)
 	env = append(env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
 	env = append(env, fmt.Sprintf("_CONTAINERS_USERNS_CONFIGURED=%s", os.Getenv("_CONTAINERS_USERNS_CONFIGURED")))
@@ -1280,11 +1275,6 @@ func (r *ConmonOCIRuntime) configureConmonEnv(ctr *Container, runtimeDir string)
 	env = append(env, fmt.Sprintf("HOME=%s", home))
 
 	extraFiles := make([]*os.File, 0)
-	if ctr.config.SdNotifyMode == define.SdNotifyModeContainer {
-		if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok {
-			env = append(env, fmt.Sprintf("NOTIFY_SOCKET=%s", notify))
-		}
-	}
 	if !r.sdNotify {
 		if listenfds, ok := os.LookupEnv("LISTEN_FDS"); ok {
 			env = append(env, fmt.Sprintf("LISTEN_FDS=%s", listenfds), "LISTEN_PID=1")
@@ -1319,6 +1309,12 @@ func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, p
 		args = append(args, rFlags...)
 	}
 
+	if ctr.config.SdNotifyMode == define.SdNotifyModeContainer {
+		if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok {
+			args = append(args, fmt.Sprintf("--sdnotify-socket=%s", notify))
+		}
+	}
+
 	if ctr.CgroupManager() == config.SystemdCgroupsManager && !ctr.config.NoCgroups && ctr.config.CgroupsMode != cgroupSplit {
 		args = append(args, "-s")
 	}