Skip to content

Commit

Permalink
Add cpu_hard_limit and cpu_cfs_period options
Browse files Browse the repository at this point in the history
Nomad's docker driver allows having nomad set cpu resources as hard
limits rather than soft limits via the `cpu_hard_limits` driver config
option. This is useful to restrict how much CPU time a nomad job is
permitted to consume, even if the host is under utilised, or when
predicatibility is more important than speed.

Default behaviour is unchanged. When `cpu_hard_limit` is set to `true`
the podman container is started with the equivalent API options to the
podman cli `--cpu-quota` and `--cpu-period`.

Nomad currently provides a `PercentTicks` value which is used to convert
the cpu resource from the specification into a number of microseconds
of cputime the job is allowed to consume for each `period` interval of time.
Use of this value is discouraged ouside of the docker driver, however the
desired long term replacement is to expose `cpu quota` and `cpu period` as
options in the task `resource` block, and these are not yet currently exposed.
Since this driver emulates the docker interface where possible, I think it's
acceptable to use the `PercentTicks` value in spite of the warning to the contrary.

The `cpu_cfs_period` defaults to 100,000 (microseconds) in both the linux
kernel, and this driver when not specified by the user. User may override
this between 1,000 and 1,000,000. Any value less than 1,000 is silently
increased to 1,000 to prevent
the job failing to run with an `Invalid Operation` error.

The minimum value for the quota is also 1,000. Since this value is
computed by this driver, it is silently increased to 1,000 if the
computed value is lower. For jobs with very small cpu resources in
the job spec, which equate to less than 1% of total available cpu,
the job may consume more cpu time than intended.
  • Loading branch information
optiz0r committed Jan 7, 2022
1 parent 7e3d018 commit a1d19d3
Show file tree
Hide file tree
Showing 5 changed files with 151 additions and 12 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ FEATURES:
* config: Map host devices into container. [[GH-41](https://github.com/hashicorp/nomad-driver-podman/pull/41)]
* config: Stream logs via API, support journald log driver. [[GH-99](https://github.com/hashicorp/nomad-driver-podman/pull/99)]
* config: Privileged containers.
* config: Add `cpu_hard_limit` and `cpu_cfs_period` options

BUG FIXES:
* log: Use error key context to log errors rather than Go err style. [[GH-126](https://github.com/hashicorp/nomad-driver-podman/pull/126)]
Expand Down
26 changes: 15 additions & 11 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,19 @@ var (
"username": hclspec.NewAttr("username", "string", false),
"password": hclspec.NewAttr("password", "string", false),
})),
"command": hclspec.NewAttr("command", "string", false),
"cap_add": hclspec.NewAttr("cap_add", "list(string)", false),
"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
"devices": hclspec.NewAttr("devices", "list(string)", false),
"entrypoint": hclspec.NewAttr("entrypoint", "string", false),
"working_dir": hclspec.NewAttr("working_dir", "string", false),
"hostname": hclspec.NewAttr("hostname", "string", false),
"image": hclspec.NewAttr("image", "string", true),
"init": hclspec.NewAttr("init", "bool", false),
"init_path": hclspec.NewAttr("init_path", "string", false),
"labels": hclspec.NewAttr("labels", "list(map(string))", false),
"command": hclspec.NewAttr("command", "string", false),
"cap_add": hclspec.NewAttr("cap_add", "list(string)", false),
"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
"cpu_hard_limit": hclspec.NewAttr("cpu_hard_limit", "bool", false),
"cpu_cfs_period": hclspec.NewAttr("cpu_cfs_period", "number", false),
"devices": hclspec.NewAttr("devices", "list(string)", false),
"entrypoint": hclspec.NewAttr("entrypoint", "string", false),
"working_dir": hclspec.NewAttr("working_dir", "string", false),
"hostname": hclspec.NewAttr("hostname", "string", false),
"image": hclspec.NewAttr("image", "string", true),
"init": hclspec.NewAttr("init", "bool", false),
"init_path": hclspec.NewAttr("init_path", "string", false),
"labels": hclspec.NewAttr("labels", "list(map(string))", false),
"logging": hclspec.NewBlock("logging", false, hclspec.NewObject(map[string]*hclspec.Spec{
"driver": hclspec.NewAttr("driver", "string", false),
"options": hclspec.NewAttr("options", "list(map(string))", false),
Expand Down Expand Up @@ -130,9 +132,11 @@ type TaskConfig struct {
MemoryReservation string `codec:"memory_reservation"`
MemorySwap string `codec:"memory_swap"`
NetworkMode string `codec:"network_mode"`
CPUCFSPeriod uint64 `codec:"cpu_cfs_period"`
MemorySwappiness int64 `codec:"memory_swappiness"`
PortMap hclutils.MapStrInt `codec:"port_map"`
Sysctl hclutils.MapStrStr `codec:"sysctl"`
CPUHardLimit bool `codec:"cpu_hard_limit"`
Init bool `codec:"init"`
Tty bool `codec:"tty"`
ForcePull bool `codec:"force_pull"`
Expand Down
17 changes: 17 additions & 0 deletions config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,20 @@ func TestConfig_ForcePull(t *testing.T) {
parser.ParseHCL(t, validHCL, &tc)
require.EqualValues(t, true, tc.ForcePull)
}

func TestConfig_CPUHardLimit(t *testing.T) {
parser := hclutils.NewConfigParser(taskConfigSpec)

validHCL := `
config {
image = "docker://redis"
cpu_hard_limit = true
cpu_cfs_period = 200000
}
`

var tc *TaskConfig
parser.ParseHCL(t, validHCL, &tc)
require.EqualValues(t, true, tc.CPUHardLimit)
require.EqualValues(t, 200000, tc.CPUCFSPeriod)
}
28 changes: 28 additions & 0 deletions driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -417,6 +418,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
CPU: &spec.LinuxCPU{},
}

err = setCPUResources(driverConfig.CPUHardLimit, driverConfig.CPUCFSPeriod, cfg.Resources.LinuxResources, createOpts.ContainerResourceConfig.ResourceLimits.CPU)
if err != nil {
return nil, nil, err
}

hard, soft, err := memoryLimits(cfg.Resources.NomadResources.Memory, driverConfig.MemoryReservation)
if err != nil {
return nil, nil, err
Expand Down Expand Up @@ -642,6 +648,28 @@ func memoryLimits(r drivers.MemoryResources, reservation string) (hard, soft *in
return nil, reserved, nil
}

func setCPUResources(hardLimit bool, period uint64, systemResources *drivers.LinuxResources, taskCPU *spec.LinuxCPU) error {
if hardLimit {
numCores := runtime.NumCPU()
if period > 1000000 {
return fmt.Errorf("invalid value for cpu_cfs_period")
}
if period == 0 {
period = 100000 // matches cgroup default
}
if period < 1000 {
period = 1000
}
quota := int64(systemResources.PercentTicks*float64(period)) * int64(numCores)
if quota < 1000 {
quota = 1000
}
taskCPU.Period = &period
taskCPU.Quota = &quota
}
return nil
}

func memoryInBytes(strmem string) (int64, error) {
l := len(strmem)
if l < 2 {
Expand Down
91 changes: 90 additions & 1 deletion driver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"io/ioutil"
"os"
"os/exec"
"runtime"
"strings"

"path/filepath"
Expand All @@ -25,6 +26,7 @@ import (
"github.com/hashicorp/nomad/plugins/drivers"
dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
tu "github.com/hashicorp/nomad/testutil"
spec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/require"
)

Expand All @@ -45,8 +47,11 @@ func createBasicResources() *drivers.Resources {
},
},
LinuxResources: &drivers.LinuxResources{
CPUShares: 512,
CPUPeriod: 100000,
CPUQuota: 100000,
CPUShares: 500,
MemoryLimitBytes: 256 * 1024 * 1024,
PercentTicks: float64(500) / float64(2000),
},
}
return &res
Expand Down Expand Up @@ -1820,6 +1825,90 @@ func createInspectImage(t *testing.T, image, reference string) {
require.Equal(t, idRef, idTest)
}

func Test_cpuLimits(t *testing.T) {
numCores := runtime.NumCPU()
cases := []struct {
name string
systemResources drivers.LinuxResources
hardLimit bool
period uint64
expectedQuota int64
expectedPeriod uint64
}{
{
name: "no hard limit",
systemResources: drivers.LinuxResources{
PercentTicks: 1.0,
CPUPeriod: 100000,
},
hardLimit: false,
expectedQuota: 100000,
expectedPeriod: 100000,
},
{
name: "hard limit max quota",
systemResources: drivers.LinuxResources{
PercentTicks: 1.0,
CPUPeriod: 100000,
},
hardLimit: true,
expectedQuota: 100000,
expectedPeriod: 100000,
},
{
name: "hard limit, half quota",
systemResources: drivers.LinuxResources{
PercentTicks: 0.5,
CPUPeriod: 100000,
},
hardLimit: true,
expectedQuota: 50000,
expectedPeriod: 100000,
},
{
name: "hard limit, custom period",
systemResources: drivers.LinuxResources{
PercentTicks: 1.0,
CPUPeriod: 100000,
},
hardLimit: true,
period: 20000,
expectedQuota: 20000,
expectedPeriod: 20000,
},
{
name: "hard limit, half quota, custom period",
systemResources: drivers.LinuxResources{
PercentTicks: 0.5,
CPUPeriod: 100000,
},
hardLimit: true,
period: 20000,
expectedQuota: 10000,
expectedPeriod: 20000,
},
}

for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
taskCPU := &spec.LinuxCPU{}
err := setCPUResources(c.hardLimit, c.period, &c.systemResources, taskCPU)
require.Nil(t, err)

if c.hardLimit {
require.NotNil(t, taskCPU.Quota)
require.Equal(t, c.expectedQuota*int64(numCores), *taskCPU.Quota)

require.NotNil(t, taskCPU.Period)
require.Equal(t, c.expectedPeriod, *taskCPU.Period)
} else {
require.Nil(t, taskCPU.Quota)
require.Nil(t, taskCPU.Period)
}
})
}
}

func Test_memoryLimits(t *testing.T) {
cases := []struct {
name string
Expand Down

0 comments on commit a1d19d3

Please sign in to comment.