Support for multiple podman sockets

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## UNRELEASED
 
+IMPROVEMENTS:
+
+  * config: Add `socket` stanza to allow multiple Podman sockets to be used. [[GH-371](https://github.com/hashicorp/nomad-driver-podman/pull/371)]
+
 ## 0.6.1 (July 15, 2024)
 
 BUG FIXES:

README.md

@@ -22,7 +22,7 @@ this plugin to Nomad!
 * Utilize podmans --init feature
 * Set username or UID used for the specified command within the container (podman --user option).
 * Fine tune memory usage: standard [Nomad memory resource](https://www.nomadproject.io/docs/job-specification/resources.html#memory) plus additional driver specific swap, swappiness and reservation parameters, OOM handling
-* Supports rootless containers with cgroup V2
+* Supports both rootful and rootless podman sockets with cgroup V2
 * Set DNS servers, searchlist and options via [Nomad dns parameters](https://www.nomadproject.io/docs/job-specification/network#dns-parameters)
 * Support for nomad shared network namespaces and consul connect
 * Quite flexible [network configuration](#network-configuration), allows to simply build pod-like structures within a nomad group
@@ -148,7 +148,7 @@ plugin "nomad-driver-podman" {
 }
 ```
 
-* socket_path (string) Defaults to `"unix:///run/podman/podman.sock"` when running as root or a cgroup V1 system, and `"unix:///run/user/<USER_ID>/podman/podman.sock"` for rootless cgroup V2 systems
+* socket_path (string) Defaults to `"unix:///run/podman/podman.sock"` when running as root or a cgroup V1 system, and `"unix:///run/user/<USER_ID>/podman/podman.sock"` for rootless cgroup V2 systems. Mutually exclusive with `socket` block.
 
 ```hcl
 plugin "nomad-driver-podman" {
@@ -158,6 +158,26 @@ plugin "nomad-driver-podman" {
 }
 ```
 
+* socket block: Configures a single podman socket. You can define multiple `socket` blocks if you need to use multiple podman sockets (for example, rootless vs rootful sockets). Mutually exclusive with the top-level `plugin.config.socket_path` option.
+
+  * name: Defaults to "default". If tasks don't mention a socket, the default socket is used.
+  * socket_path: Path to the socket.
+
+```hcl
+plugin "nomad-driver-podman" {
+  config {
+    socket {
+      name = "default"
+      socket_path = "unix://run/user/1000/podman/podman.sock"
+    }
+    socket {
+      name = "app1"
+      socket_path = "unix://run/user/1337/podman/podman.sock"
+    }
+  }
+}
+```
+
 * disable_log_collection (string) Defaults to `false`. Setting this to `true` will disable Nomad logs collection of Podman tasks. If you don't rely on nomad log capabilities and exclusively use host based log aggregation, you may consider this option to disable nomad log collection overhead. Beware to you also loose automatic log rotation.
 
 ```hcl
@@ -403,6 +423,14 @@ config {
 }
 ```
 
+* **socket** - (Optional) The name of the socket as defined in the socket block in the client agent's plugin configuration. Defaults to the socket named "default".
+
+```hcl
+config {
+  socket = "app1"
+}
+```
+
 * **cap_add** - (Optional)  A list of Linux capabilities as strings to pass to --cap-add.
 
 ```hcl

api/api.go

@@ -17,71 +17,120 @@ import (
 )
 
 type API struct {
+	apiVersion       string
 	baseUrl          string
+	defaultPodman    bool
+	appArmor         bool
+	cgroupV2         bool
+	cgroupMgr        string
+	rootless         bool
 	httpClient       *http.Client
 	httpStreamClient *http.Client
 	logger           hclog.Logger
+	defaultTimeout   time.Duration
 }
 
 type ClientConfig struct {
-	SocketPath  string
-	HttpTimeout time.Duration
+	SocketPath    string
+	HttpTimeout   time.Duration
+	DefaultPodman bool
 }
 
 func DefaultClientConfig() ClientConfig {
 	cfg := ClientConfig{
 		HttpTimeout: 60 * time.Second,
 	}
 	uid := os.Getuid()
-	// are we root?
 	if uid == 0 {
 		cfg.SocketPath = "unix:///run/podman/podman.sock"
 	} else {
-		// not? then let's try the default per-user socket location
 		cfg.SocketPath = fmt.Sprintf("unix:///run/user/%d/podman/podman.sock", uid)
 	}
+	cfg.DefaultPodman = true
 	return cfg
 }
 
 func NewClient(logger hclog.Logger, config ClientConfig) *API {
 	ac := &API{
-		logger: logger,
+		logger:        logger,
+		defaultPodman: config.DefaultPodman,
 	}
 
-	baseUrl := config.SocketPath
-	ac.logger.Debug("http baseurl", "url", baseUrl)
-	ac.httpClient = &http.Client{
-		Timeout: config.HttpTimeout,
-	}
-	// we do not want a timeout for streaming requests.
-	ac.httpStreamClient = &http.Client{}
-	if strings.HasPrefix(baseUrl, "unix:") {
+	ac.logger.Debug("http baseurl", "url", config.SocketPath)
+	ac.defaultTimeout = config.HttpTimeout
+	ac.httpClient = ac.CreateHttpClient(config.HttpTimeout, config.SocketPath, false)
+	ac.httpStreamClient = ac.CreateHttpClient(config.HttpTimeout, config.SocketPath, true)
+	if strings.HasPrefix(config.SocketPath, "unix:") {
 		ac.baseUrl = "http://u"
-		path := strings.TrimPrefix(baseUrl, "unix:")
-		ac.httpClient.Transport = &http.Transport{
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial("unix", path)
-			},
-		}
-		ac.httpStreamClient.Transport = ac.httpClient.Transport
 	} else {
-		ac.baseUrl = baseUrl
+		ac.baseUrl = config.SocketPath
 	}
 
 	return ac
 }
 
-func (c *API) Do(req *http.Request) (*http.Response, error) {
-	res, err := c.httpClient.Do(req)
-	return res, err
+func (c *API) SetAPIVersion(v string) {
+	c.apiVersion = v
+}
+
+func (c *API) GetAPIVersion() string {
+	return c.apiVersion
+}
+
+func (c *API) IsDefaultClient() bool {
+	return c.defaultPodman
+}
+
+func (c *API) SetClientAsDefault(d bool) {
+	c.defaultPodman = d
+}
+
+func (c *API) SetCgroupV2(isV2 bool) {
+	c.cgroupV2 = isV2
+}
+
+func (c *API) IsCgroupV2() bool {
+	return c.cgroupV2
+}
+
+func (c *API) SetCgroupMgr(mgr string) {
+	c.cgroupMgr = mgr
+}
+
+func (c *API) GetCgroupMgr() string {
+	return c.cgroupMgr
+}
+
+func (c *API) SetRootless(isRootless bool) {
+	c.rootless = isRootless
+}
+
+func (c *API) IsRootless() bool {
+	return c.rootless
+}
+
+func (c *API) SetAppArmor(appArmorEnabled bool) {
+	c.appArmor = appArmorEnabled
+}
+
+func (c *API) IsAppArmorEnabled() bool {
+	return c.appArmor
+}
+
+func (c *API) Do(req *http.Request, streaming bool) (*http.Response, error) {
+	if !streaming {
+		return c.httpClient.Do(req)
+	} else {
+		return c.httpStreamClient.Do(req)
+	}
 }
 
 func (c *API) Get(ctx context.Context, path string) (*http.Response, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", c.baseUrl+path, nil)
 	if err != nil {
 		return nil, err
 	}
-	return c.Do(req)
+	return c.Do(req, false)
 }
 
 func (c *API) GetStream(ctx context.Context, path string) (*http.Response, error) {
@@ -105,17 +154,45 @@ func (c *API) PostWithHeaders(ctx context.Context, path string, body io.Reader,
 	for k, v := range headers {
 		req.Header.Set(k, v)
 	}
-	return c.Do(req)
+	// If a context was passed with a Deadline longer than our HTTP client timeout,
+	// the deadline will ignored. So use the streaming HTTP client (with no timeout)
+	// so that the deadline is respected.
+	deadline, _ := ctx.Deadline()
+	if time.Until(deadline) > c.defaultTimeout {
+		return c.Do(req, true)
+	} else {
+		return c.Do(req, false)
+	}
 }
 
 func (c *API) Delete(ctx context.Context, path string) (*http.Response, error) {
 	req, err := http.NewRequestWithContext(ctx, "DELETE", c.baseUrl+path, nil)
 	if err != nil {
 		return nil, err
 	}
-	return c.Do(req)
+	return c.Do(req, false)
 }
 
 func ignoreClose(c io.Closer) {
 	_ = c.Close()
 }
+
+func (c *API) CreateHttpClient(timeout time.Duration, baseUrl string, streaming bool) *http.Client {
+	var httpClient *http.Client
+	if !streaming {
+		httpClient = &http.Client{Timeout: timeout}
+	} else { // Streaming doesn't have a timeout
+		httpClient = &http.Client{}
+	}
+
+	if strings.HasPrefix(baseUrl, "unix:") {
+		path := strings.TrimPrefix(baseUrl, "unix:")
+		httpClient.Transport = &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", path)
+			},
+		}
+	}
+
+	return httpClient
+}

api/container_delete.go

@@ -20,7 +20,7 @@ func (c *API) ContainerDelete(ctx context.Context, name string, force bool, dele
 
 	defer ignoreClose(res.Body)
 
-	if res.StatusCode == http.StatusNoContent {
+	if res.StatusCode == http.StatusNoContent || res.StatusCode == http.StatusOK {
 		return nil
 	}
 	return fmt.Errorf("cannot delete container, status code: %d", res.StatusCode)

config.go

@@ -10,6 +10,12 @@ import (
 )
 
 var (
+	// socketBodySpec is the hcl specification for the sockets in the driver object
+	socketBodySpec = hclspec.NewObject(map[string]*hclspec.Spec{
+		"name":        hclspec.NewAttr("name", "string", false), // If not specified == host_user
+		"socket_path": hclspec.NewAttr("socket_path", "string", true),
+	})
+
 	// configSpec is the hcl specification returned by the ConfigSchema RPC
 	configSpec = hclspec.NewObject(map[string]*hclspec.Spec{
 		// image registry authentication options
@@ -54,8 +60,11 @@ var (
 			options = {}
 		}`)),
 
+		// A list of sockets for the driver to manage
+		"socket": hclspec.NewBlockList("socket", socketBodySpec),
 		// the path to the podman api socket
 		"socket_path": hclspec.NewAttr("socket_path", "string", false),
+
 		// disable_log_collection indicates whether nomad should collect logs of podman
 		// task containers.  If true, logs are not forwarded to nomad.
 		"disable_log_collection": hclspec.NewAttr("disable_log_collection", "bool", false),
@@ -108,15 +117,19 @@ var (
 		"port_map":           hclspec.NewAttr("port_map", "list(map(number))", false),
 		"ports":              hclspec.NewAttr("ports", "list(string)", false),
 		"privileged":         hclspec.NewAttr("privileged", "bool", false),
-		"sysctl":             hclspec.NewAttr("sysctl", "list(map(string))", false),
-		"tmpfs":              hclspec.NewAttr("tmpfs", "list(string)", false),
-		"tty":                hclspec.NewAttr("tty", "bool", false),
-		"ulimit":             hclspec.NewAttr("ulimit", "list(map(string))", false),
-		"volumes":            hclspec.NewAttr("volumes", "list(string)", false),
-		"force_pull":         hclspec.NewAttr("force_pull", "bool", false),
-		"readonly_rootfs":    hclspec.NewAttr("readonly_rootfs", "bool", false),
-		"userns":             hclspec.NewAttr("userns", "string", false),
-		"shm_size":           hclspec.NewAttr("shm_size", "string", false),
+		"socket": hclspec.NewDefault(
+			hclspec.NewAttr("socket", "string", false),
+			hclspec.NewLiteral(`"default"`),
+		),
+		"sysctl":          hclspec.NewAttr("sysctl", "list(map(string))", false),
+		"tmpfs":           hclspec.NewAttr("tmpfs", "list(string)", false),
+		"tty":             hclspec.NewAttr("tty", "bool", false),
+		"ulimit":          hclspec.NewAttr("ulimit", "list(map(string))", false),
+		"volumes":         hclspec.NewAttr("volumes", "list(string)", false),
+		"force_pull":      hclspec.NewAttr("force_pull", "bool", false),
+		"readonly_rootfs": hclspec.NewAttr("readonly_rootfs", "bool", false),
+		"userns":          hclspec.NewAttr("userns", "string", false),
+		"shm_size":        hclspec.NewAttr("shm_size", "string", false),
 	})
 )
 
@@ -163,18 +176,24 @@ type PluginAuthConfig struct {
 	Helper     string `codec:"helper"`
 }
 
+type PluginSocketConfig struct {
+	Name       string `codec:"name"`
+	SocketPath string `codec:"socket_path"`
+}
+
 // PluginConfig is the driver configuration set by the SetConfig RPC call
 type PluginConfig struct {
-	Auth                 PluginAuthConfig `codec:"auth"`
-	Volumes              VolumeConfig     `codec:"volumes"`
-	GC                   GCConfig         `codec:"gc"`
-	RecoverStopped       bool             `codec:"recover_stopped"`
-	DisableLogCollection bool             `codec:"disable_log_collection"`
-	SocketPath           string           `codec:"socket_path"`
-	ClientHttpTimeout    string           `codec:"client_http_timeout"`
-	ExtraLabels          []string         `codec:"extra_labels"`
-	DNSServers           []string         `codec:"dns_servers"`
-	Logging              LoggingConfig    `codec:"logging"`
+	Auth                 PluginAuthConfig     `codec:"auth"`
+	Volumes              VolumeConfig         `codec:"volumes"`
+	GC                   GCConfig             `codec:"gc"`
+	RecoverStopped       bool                 `codec:"recover_stopped"`
+	DisableLogCollection bool                 `codec:"disable_log_collection"`
+	Socket               []PluginSocketConfig `codec:"socket"`
+	SocketPath           string               `codec:"socket_path"`
+	ClientHttpTimeout    string               `codec:"client_http_timeout"`
+	ExtraLabels          []string             `codec:"extra_labels"`
+	DNSServers           []string             `codec:"dns_servers"`
+	Logging              LoggingConfig        `codec:"logging"`
 }
 
 // LogWarnings will emit logs about known problematic configurations
@@ -214,6 +233,7 @@ type TaskConfig struct {
 	MemorySwappiness  int64              `codec:"memory_swappiness"`
 	PidsLimit         int64              `codec:"pids_limit"`
 	PortMap           hclutils.MapStrInt `codec:"port_map"`
+	Socket            string             `codec:"socket"`
 	Sysctl            hclutils.MapStrStr `codec:"sysctl"`
 	Ulimit            hclutils.MapStrStr `codec:"ulimit"`
 	CPUHardLimit      bool               `codec:"cpu_hard_limit"`

config_test.go

@@ -137,3 +137,18 @@ func TestConfig_ExtraHosts(t *testing.T) {
 	parser.ParseHCL(t, validHCL, &tc)
 	must.Eq(t, []string{"myhost:127.0.0.2", "example.com:10.0.0.1"}, tc.ExtraHosts)
 }
+
+func TestConfig_PodmanSocketDefaultIfNotGiven(t *testing.T) {
+	ci.Parallel(t)
+
+	parser := hclutils.NewConfigParser(taskConfigSpec)
+	validHCL := `
+	config {
+		image = "docker://redis"
+	}
+	`
+
+	var tc *TaskConfig
+	parser.ParseHCL(t, validHCL, &tc)
+	must.Eq(t, "default", tc.Socket)
+}