Add RequiresMountsFor= to systemd generate

It is rare but possible that storage locations for the graphroot and the runroot are not mounted at boot time, and therefore might race when doing container operations. An example we've seen in the wild is that a slow tmpfs mount for the runroot would suddenly mount over /run, causing the container to lose all currently-running data, requiring a system refresh to get it back. This patch adds RequiresMountsFor= to the systemd.unit header to ensure the paths for both the graphroot and runroot are mounted prior to starting any generated unit files. Signed-off-by: Robb Manes <[email protected]>
vrothberg · Mar 26, 2021 · 748826f · 748826f
1 parent 604459b
commit 748826f
Show file tree

Hide file tree

Showing 6 changed files with 111 additions and 1 deletion.
diff --git a/docs/source/markdown/podman-generate-systemd.1.md b/docs/source/markdown/podman-generate-systemd.1.md
@@ -61,7 +61,7 @@ Set the systemd unit name separator between the name/id of a container/pod and t
 
 ### Generate and print a systemd unit file for a container
 
-Generate a systemd unit file for a container running nginx with an *always* restart policy and 1-second timeout to stdout.
+Generate a systemd unit file for a container running nginx with an *always* restart policy and 1-second timeout to stdout. Note that the **RequiresMountsFor** option in the **Unit** section ensures that the container storage for both the GraphRoot and the RunRoot are mounted prior to starting the service. For systems with container storage on disks like iSCSI or other remote block protocols, this ensures that Podman is not executed prior to any necessary storage operations coming online.
 
 ```
 $ podman create --name nginx nginx:latest
@@ -73,6 +73,9 @@ $ podman generate systemd --restart-policy=always -t 1 nginx
 [Unit]
 Description=Podman container-de1e3223b1b888bc02d0962dd6cb5855eb00734061013ffdd3479d225abacdc6.service
 Documentation=man:podman-generate-systemd(1)
+Wants=network.target
+After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/container/storage
 
 [Service]
 Restart=always
@@ -101,6 +104,7 @@ Description=Podman container-busy_moser.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/container/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -140,6 +144,9 @@ Description=Podman pod-systemd-pod.service
 Documentation=man:podman-generate-systemd(1)
 Requires=container-amazing_chandrasekhar.service container-jolly_shtern.service
 Before=container-amazing_chandrasekhar.service container-jolly_shtern.service
+Wants=network.target
+After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/container/storage
 
 [Service]
 Restart=on-failure

diff --git a/pkg/systemd/generate/common.go b/pkg/systemd/generate/common.go
@@ -36,6 +36,7 @@ Description=Podman {{{{.ServiceName}}}}.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor={{{{.GraphRoot}}}} {{{{.RunRoot}}}}
 `
 
 // filterPodFlags removes --pod and --pod-id-file from the specified command.

diff --git a/pkg/systemd/generate/containers.go b/pkg/systemd/generate/containers.go
@@ -71,6 +71,12 @@ type containerInfo struct {
 	// If not nil, the container is part of the pod.  We can use the
 	// podInfo to extract the relevant data.
 	Pod *podInfo
+	// Location of the GraphRoot for the container.  Required for ensuring the
+	// volume has finished mounting when coming online at boot.
+	GraphRoot string
+	// Location of the RunRoot for the container.  Required for ensuring the tmpfs
+	// or volume exists and is mounted when coming online at boot.
+	RunRoot string
 }
 
 const containerTemplate = headerTemplate + `
@@ -132,6 +138,21 @@ func generateContainerInfo(ctr *libpod.Container, options entities.GenerateSyste
 
 	nameOrID, serviceName := containerServiceName(ctr, options)
 
+	store := ctr.Runtime().GetStore()
+	if store == nil {
+		return nil, errors.Errorf("could not determine storage store for container")
+	}
+
+	graphRoot := store.GraphRoot()
+	if graphRoot == "" {
+		return nil, errors.Errorf("could not lookup container's graphroot: got empty string")
+	}
+
+	runRoot := store.RunRoot()
+	if runRoot == "" {
+		return nil, errors.Errorf("could not lookup container's runroot: got empty string")
+	}
+
 	info := containerInfo{
 		ServiceName:       serviceName,
 		ContainerNameOrID: nameOrID,
@@ -140,6 +161,8 @@ func generateContainerInfo(ctr *libpod.Container, options entities.GenerateSyste
 		StopTimeout:       timeout,
 		GenerateTimestamp: true,
 		CreateCommand:     createCommand,
+		GraphRoot:         graphRoot,
+		RunRoot:           runRoot,
 	}
 
 	return &info, nil

diff --git a/pkg/systemd/generate/containers_test.go b/pkg/systemd/generate/containers_test.go
@@ -48,6 +48,7 @@ Description=Podman container-639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -73,6 +74,7 @@ Description=Podman container-foobar.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -96,6 +98,7 @@ Description=Podman container-foobar.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 BindsTo=a.service b.service c.service pod.service
 After=a.service b.service c.service pod.service
 
@@ -121,6 +124,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -145,6 +149,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -169,6 +174,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -193,6 +199,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -217,6 +224,7 @@ Description=Podman container-639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -242,6 +250,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -270,6 +279,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -294,6 +304,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -318,6 +329,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -342,6 +354,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -366,6 +379,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -400,6 +414,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:       22,
 				PodmanVersion:     "CI",
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodID,
 			false,
@@ -416,6 +432,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:       22,
 				PodmanVersion:     "CI",
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodIDNoHeaderInfo,
 			false,
@@ -432,6 +450,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:       10,
 				PodmanVersion:     "CI",
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodName,
 			false,
@@ -449,6 +469,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				BoundToServices:   []string{"pod", "a", "b", "c"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameBoundTo,
 			false,
@@ -464,6 +486,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:   10,
 				PodmanVersion: "CI",
 				EnvVariable:   define.EnvVariable,
+				GraphRoot:     "/var/lib/containers/storage",
+				RunRoot:       "/var/run/containers/storage",
 			},
 			"",
 			false,
@@ -481,6 +505,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "container", "run", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN", "foo=arg \"with \" space"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodWithNameAndGeneric,
 			true,
@@ -498,6 +524,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-d", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodWithExplicitShortDetachParam,
 			true,
@@ -515,6 +543,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-d", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 				Pod: &podInfo{
 					PodIDFile: "%t/pod-foobar.pod-id-file",
 				},
@@ -535,6 +565,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--detach", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameNewDetach,
 			true,
@@ -552,6 +584,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodIDNew,
 			true,
@@ -569,6 +603,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--detach=true", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("--detach=true"),
 			true,
@@ -586,6 +622,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--detach=false", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("-d"),
 			true,
@@ -603,6 +641,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--name", "test", "-p", "80:80", "--detach=false", "awesome-image:latest", "somecmd", "--detach=false"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameNewDetachFalseWithCmd,
 			true,
@@ -620,6 +660,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--name", "test", "-p", "80:80", "--detach=false", "--detach=false", "awesome-image:latest", "somecmd", "--detach=false"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameNewDetachFalseWithCmd,
 			true,
@@ -637,6 +679,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-dti", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("-dti"),
 			true,
@@ -654,6 +698,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-tid", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("-tid"),
 			true,
@@ -671,6 +717,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "--events-backend", "none", "--runroot", "/root", "run", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNewRootFlags,
 			true,
@@ -688,6 +736,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "container", "create", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodContainerCreate,
 			true,
@@ -705,6 +755,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "create", "--name", "test", "--log-driver=journald", "--log-opt=tag={{.Name}}", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNewWithJournaldTag,
 			true,
@@ -722,6 +774,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "create", "--name", "test", "awesome-image:latest", "sh", "-c", "kill $$ && echo %\\"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNewWithSpecialChars,
 			true,

diff --git a/pkg/systemd/generate/pods.go b/pkg/systemd/generate/pods.go
@@ -73,6 +73,12 @@ type podInfo struct {
 	ExecStopPost string
 	// Removes autogenerated by Podman and timestamp if set to true
 	GenerateNoHeader bool
+	// Location of the GraphRoot for the pod.  Required for ensuring the
+	// volume has finished mounting when coming online at boot.
+	GraphRoot string
+	// Location of the RunRoot for the pod.  Required for ensuring the tmpfs
+	// or volume exists and is mounted when coming online at boot.
+	RunRoot string
 }
 
 const podTemplate = headerTemplate + `Requires={{{{- range $index, $value := .RequiredServices -}}}}{{{{if $index}}}} {{{{end}}}}{{{{ $value }}}}.service{{{{end}}}}