diff --git a/CHANGELOG.md b/CHANGELOG.md index f181921f409..59b48194c42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ IMPROVEMENTS: * csi: Return better error messages [[GH-7984](https://github.com/hashicorp/nomad/issues/7984)] [[GH-8030](https://github.com/hashicorp/nomad/issues/8030)] * csi: Move volume claim releases out of evaluation workers [[GH-8021](https://github.com/hashicorp/nomad/issues/8021)] * csi: Added support for `VolumeContext` and `VolumeParameters` [[GH-7957](https://github.com/hashicorp/nomad/issues/7957)] + * driver/docker: Added support for `memory_hard_limit` configuration in docker task driver [[GH-2093](https://github.com/hashicorp/nomad/issues/2093)] * logging: Remove spurious error log on task shutdown [[GH-8028](https://github.com/hashicorp/nomad/issues/8028)] * ui: Added filesystem browsing for allocations [[GH-5871](https://github.com/hashicorp/nomad/pull/7951)] diff --git a/drivers/docker/config.go b/drivers/docker/config.go index 44abc6a9ab2..a1555a57edd 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -319,7 +319,8 @@ var ( "driver": hclspec.NewAttr("driver", "string", false), "config": hclspec.NewAttr("config", "list(map(string))", false), })), - "mac_address": hclspec.NewAttr("mac_address", "string", false), + "mac_address": hclspec.NewAttr("mac_address", "string", false), + "memory_hard_limit": hclspec.NewAttr("memory_hard_limit", "number", false), "mounts": hclspec.NewBlockList("mounts", hclspec.NewObject(map[string]*hclspec.Spec{ "type": hclspec.NewDefault( hclspec.NewAttr("type", "string", false), @@ -408,6 +409,7 @@ type TaskConfig struct { LoadImage string `codec:"load"` Logging DockerLogging `codec:"logging"` MacAddress string `codec:"mac_address"` + MemoryHardLimit int64 `codec:"memory_hard_limit"` Mounts []DockerMount `codec:"mounts"` NetworkAliases []string `codec:"network_aliases"` NetworkMode string `codec:"network_mode"` diff --git a/drivers/docker/config_test.go b/drivers/docker/config_test.go index e1438fa9a35..75d387b513c 100644 --- a/drivers/docker/config_test.go +++ b/drivers/docker/config_test.go @@ -222,6 +222,7 @@ config { } } mac_address = "02:42:ac:11:00:02" + memory_hard_limit = 512 mounts = [ { type = "bind" @@ -349,7 +350,8 @@ config { "max-file": "3", "max-size": "10m", }}, - MacAddress: "02:42:ac:11:00:02", + MacAddress: "02:42:ac:11:00:02", + MemoryHardLimit: 512, Mounts: []DockerMount{ { Type: "bind", @@ -524,7 +526,6 @@ func TestConfig_InternalCapabilities(t *testing.T) { require.Equal(t, c.expected, d.InternalCapabilities()) }) } - } func TestConfig_DriverConfig_PullActivityTimeout(t *testing.T) { @@ -582,5 +583,4 @@ func TestConfig_DriverConfig_AllowRuntimes(t *testing.T) { require.Equal(t, c.expected, d.config.allowRuntimes) }) } - } diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 35e247ae5b1..75cde8ac8cc 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -724,6 +724,30 @@ func parseSecurityOpts(securityOpts []string) ([]string, error) { return securityOpts, nil } +// memoryLimits computes the memory and memory_reservation values passed along to +// the docker host config. These fields represent hard and soft memory limits from +// docker's perspective, respectively. +// +// The memory field on the task configuration can be interpreted as a hard or soft +// limit. Before Nomad v0.11.3, it was always a hard limit. Now, it is interpreted +// as a soft limit if the memory_hard_limit value is configured on the docker +// task driver configuration. When memory_hard_limit is set, the docker host +// config is configured such that the memory field is equal to memory_hard_limit +// value, and the memory_reservation field is set to the task driver memory value. +// +// If memory_hard_limit is not set (i.e. zero value), then the memory field of +// the task resource config is interpreted as a hard limit. In this case both the +// memory is set to the task resource memory value and memory_reservation is left +// unset. +// +// Returns (memory (hard), memory_reservation (soft)) values in bytes. +func (_ *Driver) memoryLimits(driverHardLimitMB, taskMemoryLimitBytes int64) (int64, int64) { + if driverHardLimitMB <= 0 { + return taskMemoryLimitBytes, 0 + } + return driverHardLimitMB * 1024 * 1024, taskMemoryLimitBytes +} + func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *TaskConfig, imageID string) (docker.CreateContainerOptions, error) { @@ -772,8 +796,12 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T return c, fmt.Errorf("requested runtime %q is not allowed", containerRuntime) } + memory, memoryReservation := d.memoryLimits(driverConfig.MemoryHardLimit, task.Resources.LinuxResources.MemoryLimitBytes) + hostConfig := &docker.HostConfig{ - Memory: task.Resources.LinuxResources.MemoryLimitBytes, + Memory: memory, // hard limit + MemoryReservation: memoryReservation, // soft limit + CPUShares: task.Resources.LinuxResources.CPUShares, // Binds are used to mount a host volume into the container. We mount a @@ -837,7 +865,8 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T } } - logger.Debug("configured resources", "memory", hostConfig.Memory, + logger.Debug("configured resources", + "memory", hostConfig.Memory, "memory_reservation", hostConfig.MemoryReservation, "cpu_shares", hostConfig.CPUShares, "cpu_quota", hostConfig.CPUQuota, "cpu_period", hostConfig.CPUPeriod) diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index 81ec357576b..b974fb010f9 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -1484,6 +1484,32 @@ func TestDockerDriver_DNS(t *testing.T) { require.Exactly(t, cfg.DNSOptions, container.HostConfig.DNSOptions) } +func TestDockerDriver_MemoryHardLimit(t *testing.T) { + if !tu.IsCI() { + t.Parallel() + } + testutil.DockerCompatible(t) + if runtime.GOOS == "windows" { + t.Skip("Windows does not support MemoryReservation") + } + + task, cfg, ports := dockerTask(t) + defer freeport.Return(ports) + + cfg.MemoryHardLimit = 300 + require.NoError(t, task.EncodeConcreteDriverConfig(cfg)) + + client, d, handle, cleanup := dockerSetup(t, task, nil) + defer cleanup() + require.NoError(t, d.WaitUntilStarted(task.ID, 5*time.Second)) + + container, err := client.InspectContainer(handle.containerID) + require.NoError(t, err) + + require.Equal(t, task.Resources.LinuxResources.MemoryLimitBytes, container.HostConfig.MemoryReservation) + require.Equal(t, cfg.MemoryHardLimit*1024*1024, container.HostConfig.Memory) +} + func TestDockerDriver_MACAddress(t *testing.T) { if !tu.IsCI() { t.Parallel() @@ -2681,3 +2707,19 @@ func TestDockerDriver_CreateContainerConfig_CPUHardLimit(t *testing.T) { require.NotZero(t, c.HostConfig.CPUQuota) require.NotZero(t, c.HostConfig.CPUPeriod) } + +func TestDockerDriver_memoryLimits(t *testing.T) { + t.Parallel() + + t.Run("driver hard limit not set", func(t *testing.T) { + memory, memoryReservation := new(Driver).memoryLimits(0, 256*1024*1024) + require.Equal(t, int64(256*1024*1024), memory) + require.Equal(t, int64(0), memoryReservation) + }) + + t.Run("driver hard limit is set", func(t *testing.T) { + memory, memoryReservation := new(Driver).memoryLimits(512, 256*1024*1024) + require.Equal(t, int64(512*1024*1024), memory) + require.Equal(t, int64(256*1024*1024), memoryReservation) + }) +} diff --git a/website/pages/docs/drivers/docker.mdx b/website/pages/docs/drivers/docker.mdx index 3b68ae23353..f261ba45c12 100644 --- a/website/pages/docs/drivers/docker.mdx +++ b/website/pages/docs/drivers/docker.mdx @@ -183,6 +183,14 @@ The `docker` driver supports the following configuration in the job spec. Only - `mac_address` - (Optional) The MAC address for the container to use (e.g. "02:68:b3:29:da:98"). +- `memory_hard_limit` - (Optional) The maximum allowable amount of memory used + (megabytes) by the container. If set, the [`memory`](/docs/job-specification/resources#memory) + parameter of the task resource configuration becomes a soft limit passed to the + docker driver as [`--memory_reservation`](https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory), + and `memory_hard_limit` is passed as the [`--memory`](https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory) + hard limit. When the host is under memory pressure, the behavior of soft limit + activation is governed by the [Kernel](https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt). + - `network_aliases` - (Optional) A list of network-scoped aliases, provide a way for a container to be discovered by an alternate name by any other container within the scope of a particular network. Network-scoped alias is supported only for