Support rootless Podman driver

Usage: `minikube start --driver=podman --rootless --container-runtime=(cri-o|containerd)`
Tested on Podman 3.4.1, Ubuntu 21.10.

Needs cgroup v2 (as in Rootless Docker): https://rootlesscontaine.rs/getting-started/common/cgroup2/
See also `site/content/en/docs/drivers/includes/podman_usage.inc`

Fix issue 8719
Fix issue 12460

Signed-off-by: Akihiro Suda <[email protected]>

cmd/minikube/cmd/start.go

@@ -135,6 +135,12 @@ func platform() string {
 
 // runStart handles the executes the flow of "minikube start"
 func runStart(cmd *cobra.Command, args []string) {
+	// viper maps $MINIKUBE_ROOTLESS to --rootless automatically, but it does not do vice versa,
+	// so we map --rootless to $MINIKUBE_ROOTLESS expliclity here.
+	// $MINIKUBE_ROOTLESS is referred by KIC runner, which isn't aware of the *cobra.Command object.
+	if cmd.Flag(rootless).Changed {
+		os.Setenv(constants.MinikubeRootlessEnv, strconv.FormatBool(viper.GetBool(rootless)))
+	}
 	register.SetEventLogPath(localpath.EventLog(ClusterFlagValue()))
 	ctx := context.Background()
 	out.SetJSON(outputFormat == "json")

cmd/minikube/cmd/start_flags.go

@@ -134,6 +134,7 @@ const (
 	certExpiration          = "cert-expiration"
 	binaryMirror            = "binary-mirror"
 	disableOptimizations    = "disable-optimizations"
+	rootless                = "rootless"
 )
 
 var (
@@ -249,6 +250,7 @@ func initDriverFlags() {
 	// docker & podman
 	startCmd.Flags().String(listenAddress, "", "IP Address to use to expose ports (docker and podman driver only)")
 	startCmd.Flags().StringSlice(ports, []string{}, "List of ports that should be exposed (docker and podman driver only)")
+	startCmd.Flags().Bool(rootless, false, "Force to use rootless driver (docker and podman driver only)")
 }
 
 // initNetworkingFlags inits the commandline flags for connectivity related flags for start
@@ -549,12 +551,22 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str
 			exit.Message(reason.Usage, "Ensure your {{.driver_name}} is running and is healthy.", out.V{"driver_name": driver.FullName(drvName)})
 		}
 		if si.Rootless {
+			out.Styled(style.Notice, "Using rootless {{.driver_name}} driver", out.V{"driver_name": driver.FullName(drvName)})
 			if cc.KubernetesConfig.ContainerRuntime == constants.Docker {
 				exit.Message(reason.Usage, "--container-runtime must be set to \"containerd\" or \"cri-o\" for rootless")
 			}
 			// KubeletInUserNamespace feature gate is essential for rootless driver.
 			// See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-in-userns/
 			cc.KubernetesConfig.FeatureGates = addFeatureGate(cc.KubernetesConfig.FeatureGates, "KubeletInUserNamespace=true")
+		} else {
+			if oci.IsRootlessForced() {
+				if driver.IsDocker(drvName) {
+					exit.Message(reason.Usage, "Using rootless Docker driver was required, but the current Docker does not seem rootless. Try 'docker context use rootless' .")
+				} else {
+					exit.Message(reason.Usage, "Using rootless driver was required, but the current driver does not seem rootless")
+				}
+			}
+			out.Styled(style.Notice, "Using {{.driver_name}} driver with the root privilege", out.V{"driver_name": driver.FullName(drvName)})
 		}
 		if si.StorageDriver == "btrfs" {
 			klog.Info("auto-setting LocalStorageCapacityIsolation to false because using btrfs storage driver")

pkg/drivers/kic/oci/cli_runner.go

@@ -31,6 +31,7 @@ import (
 
 	"k8s.io/klog/v2"
 
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/style"
 )
@@ -72,10 +73,40 @@ func (rr RunResult) Output() string {
 	return sb.String()
 }
 
+// IsRootlessForced returns whether rootless mode is explicitly required.
+func IsRootlessForced() bool {
+	s := os.Getenv(constants.MinikubeRootlessEnv)
+	if s == "" {
+		return false
+	}
+	v, err := strconv.ParseBool(s)
+	if err != nil {
+		klog.ErrorS(err, "failed to parse", "env", constants.MinikubeRootlessEnv, "value", s)
+		return false
+	}
+	return v
+}
+
+type prefixCmdOptions struct {
+	sudoFlags []string
+}
+
+type PrefixCmdOption func(*prefixCmdOptions)
+
+func WithSudoFlags(ss ...string) PrefixCmdOption {
+	return func(o *prefixCmdOptions) {
+		o.sudoFlags = ss
+	}
+}
+
 // PrefixCmd adds any needed prefix (such as sudo) to the command
-func PrefixCmd(cmd *exec.Cmd) *exec.Cmd {
-	if cmd.Args[0] == Podman && runtime.GOOS == "linux" { // want sudo when not running podman-remote
-		cmdWithSudo := exec.Command("sudo", append([]string{"-n"}, cmd.Args...)...)
+func PrefixCmd(cmd *exec.Cmd, opt ...PrefixCmdOption) *exec.Cmd {
+	var o prefixCmdOptions
+	for _, f := range opt {
+		f(&o)
+	}
+	if cmd.Args[0] == Podman && runtime.GOOS == "linux" && !IsRootlessForced() { // want sudo when not running podman-remote
+		cmdWithSudo := exec.Command("sudo", append(append([]string{"-n"}, o.sudoFlags...), cmd.Args...)...)
 		cmdWithSudo.Env = cmd.Env
 		cmdWithSudo.Dir = cmd.Dir
 		cmdWithSudo.Stdin = cmd.Stdin

pkg/drivers/kic/oci/info.go

@@ -59,7 +59,7 @@ func CachedDaemonInfo(ociBin string) (SysInfo, error) {
 func DaemonInfo(ociBin string) (SysInfo, error) {
 	if ociBin == Podman {
 		p, err := podmanSystemInfo()
-		cachedSysInfo = &SysInfo{CPUs: p.Host.Cpus, TotalMemory: p.Host.MemTotal, OSType: p.Host.Os, Swarm: false, StorageDriver: p.Store.GraphDriverName}
+		cachedSysInfo = &SysInfo{CPUs: p.Host.Cpus, TotalMemory: p.Host.MemTotal, OSType: p.Host.Os, Swarm: false, Rootless: p.Host.Security.Rootless, StorageDriver: p.Store.GraphDriverName}
 		return *cachedSysInfo, err
 	}
 	d, err := dockerSystemInfo()
@@ -213,8 +213,10 @@ type podmanSysInfo struct {
 		Hostname    string `json:"hostname"`
 		Kernel      string `json:"kernel"`
 		Os          string `json:"os"`
-		Rootless    bool   `json:"rootless"`
-		Uptime      string `json:"uptime"`
+		Security    struct {
+			Rootless bool `json:"rootless"`
+		} `json:"security"`
+		Uptime string `json:"uptime"`
 	} `json:"host"`
 	Registries struct {
 		Search []string `json:"search"`

pkg/drivers/kic/oci/oci.go

@@ -314,7 +314,7 @@ func createContainer(ociBin string, image string, opts ...createOpt) error {
 
 	// to run nested container from privileged container in podman https://bugzilla.redhat.com/show_bug.cgi?id=1687713
 	// only add when running locally (linux), when running remotely it needs to be configured on server in libpod.conf
-	if ociBin == Podman && runtime.GOOS == "linux" {
+	if ociBin == Podman && runtime.GOOS == "linux" && !IsRootlessForced() {
 		args = append(args, "--cgroup-manager", "cgroupfs")
 	}
 
@@ -344,7 +344,7 @@ func StartContainer(ociBin string, container string) error {
 
 	// to run nested container from privileged container in podman https://bugzilla.redhat.com/show_bug.cgi?id=1687713
 	// only add when running locally (linux), when running remotely it needs to be configured on server in libpod.conf
-	if ociBin == Podman && runtime.GOOS == "linux" {
+	if ociBin == Podman && runtime.GOOS == "linux" && !IsRootlessForced() {
 		args = append(args, "--cgroup-manager", "cgroupfs")
 	}
 

pkg/minikube/constants/constants.go

@@ -99,6 +99,8 @@ const (
 	TestDiskUsedEnv = "MINIKUBE_TEST_STORAGE_CAPACITY"
 	// TestDiskAvailableEnv is used in integration tests for insufficient storage with 'minikube status' (in GiB)
 	TestDiskAvailableEnv = "MINIKUBE_TEST_AVAILABLE_STORAGE"
+	// MinikubeRootlessEnv is used to force Rootless Docker/Podman driver
+	MinikubeRootlessEnv = "MINIKUBE_ROOTLESS"
 
 	// scheduled stop constants
 

pkg/minikube/registry/drvs/podman/podman.go

@@ -112,7 +112,8 @@ func status() registry.State {
 	cmd := exec.CommandContext(ctx, oci.Podman, "version", "--format", "{{.Server.Version}}")
 	// Run with sudo on linux (local), otherwise podman-remote (as podman)
 	if runtime.GOOS == "linux" {
-		cmd = exec.CommandContext(ctx, "sudo", "-k", "-n", oci.Podman, "version", "--format", "{{.Version}}")
+		cmd = exec.CommandContext(ctx, oci.Podman, "version", "--format", "{{.Version}}")
+		cmd = oci.PrefixCmd(cmd, oci.WithSudoFlags("-k"))
 		cmd.Env = append(os.Environ(), "LANG=C", "LC_ALL=C") // sudo is localized
 	}
 	o, err := cmd.Output()
@@ -150,7 +151,7 @@ func status() registry.State {
 		newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr)
 
 		if strings.Contains(stderr, "a password is required") && runtime.GOOS == "linux" {
-			return registry.State{Error: newErr, Installed: true, Healthy: false, Fix: fmt.Sprintf("Add your user to the 'sudoers' file: '%s ALL=(ALL) NOPASSWD: %s'", username, podman), Doc: "https://podman.io"}
+			return registry.State{Error: newErr, Installed: true, Healthy: false, Fix: fmt.Sprintf("Add your user to the 'sudoers' file: '%s ALL=(ALL) NOPASSWD: %s' , or specify '--rootless' to enable rootless mode", username, podman), Doc: "https://podman.io"}
 		}
 
 		// Typical low-level errors from running podman-remote:

site/content/en/docs/drivers/docker.md

@@ -45,6 +45,9 @@ docker context use rootless
 minikube start --driver=docker --container-runtime=containerd
 ```
 
+Unlike Podman driver, it is not necessary to supply the `--rootless` flag to the `minikube start` command.
+When the `--rootless` flag is explicitly specified but the current Docker host is not rootless, minikube fails.
+
 The `--container-runtime` flag must be set to "containerd" or "cri-o".
 {{% /tab %}}
 {{% /tabs %}}

site/content/en/docs/drivers/includes/podman_usage.inc

@@ -21,3 +21,14 @@ To make podman the default driver:
 ```shell
 minikube config set driver podman
 ```
+
+## Rootless Podman
+
+By default, minikube executes Podman with `sudo`.
+To use Podman without `sudo` (i.e., Rootless Podman), specify the `--rootless` flag in addition to `--driver=podman`.
+
+```shell
+minikube start --driver=podman --rootless --container-runtime=cri-o
+```
+
+See the [Rootless Docker](https://minikube.sigs.k8s.io/docs/drivers/docker/#rootless-docker) section for the list of supported `--container-runtime` values and restrictions.