diff --git a/CHANGELOG.md b/CHANGELOG.md index 472c60f7..b505799e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ FEATURES: * config: Map host devices into container. [[GH-41](https://github.com/hashicorp/nomad-driver-podman/pull/41)] * config: Stream logs via API, support journald log driver. [[GH-99](https://github.com/hashicorp/nomad-driver-podman/pull/99)] * config: Privileged containers. +* config: Add `cpu_hard_limit` and `cpu_cfs_period` options BUG FIXES: * log: Use error key context to log errors rather than Go err style. [[GH-126](https://github.com/hashicorp/nomad-driver-podman/pull/126)] diff --git a/config.go b/config.go index 3fe185cd..dfac4424 100644 --- a/config.go +++ b/config.go @@ -47,17 +47,19 @@ var ( "username": hclspec.NewAttr("username", "string", false), "password": hclspec.NewAttr("password", "string", false), })), - "command": hclspec.NewAttr("command", "string", false), - "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), - "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), - "devices": hclspec.NewAttr("devices", "list(string)", false), - "entrypoint": hclspec.NewAttr("entrypoint", "string", false), - "working_dir": hclspec.NewAttr("working_dir", "string", false), - "hostname": hclspec.NewAttr("hostname", "string", false), - "image": hclspec.NewAttr("image", "string", true), - "init": hclspec.NewAttr("init", "bool", false), - "init_path": hclspec.NewAttr("init_path", "string", false), - "labels": hclspec.NewAttr("labels", "list(map(string))", false), + "command": hclspec.NewAttr("command", "string", false), + "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), + "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), + "cpu_hard_limit": hclspec.NewAttr("cpu_hard_limit", "bool", false), + "cpu_cfs_period": hclspec.NewAttr("cpu_cfs_period", "number", false), + "devices": hclspec.NewAttr("devices", "list(string)", false), + "entrypoint": hclspec.NewAttr("entrypoint", "string", false), + "working_dir": hclspec.NewAttr("working_dir", "string", false), + "hostname": hclspec.NewAttr("hostname", "string", false), + "image": hclspec.NewAttr("image", "string", true), + "init": hclspec.NewAttr("init", "bool", false), + "init_path": hclspec.NewAttr("init_path", "string", false), + "labels": hclspec.NewAttr("labels", "list(map(string))", false), "logging": hclspec.NewBlock("logging", false, hclspec.NewObject(map[string]*hclspec.Spec{ "driver": hclspec.NewAttr("driver", "string", false), "options": hclspec.NewAttr("options", "list(map(string))", false), @@ -130,9 +132,11 @@ type TaskConfig struct { MemoryReservation string `codec:"memory_reservation"` MemorySwap string `codec:"memory_swap"` NetworkMode string `codec:"network_mode"` + CPUCFSPeriod uint64 `codec:"cpu_cfs_period"` MemorySwappiness int64 `codec:"memory_swappiness"` PortMap hclutils.MapStrInt `codec:"port_map"` Sysctl hclutils.MapStrStr `codec:"sysctl"` + CPUHardLimit bool `codec:"cpu_hard_limit"` Init bool `codec:"init"` Tty bool `codec:"tty"` ForcePull bool `codec:"force_pull"` diff --git a/config_test.go b/config_test.go index 4d5ecaf5..5fffffd1 100644 --- a/config_test.go +++ b/config_test.go @@ -81,3 +81,20 @@ func TestConfig_ForcePull(t *testing.T) { parser.ParseHCL(t, validHCL, &tc) require.EqualValues(t, true, tc.ForcePull) } + +func TestConfig_CPUHardLimit(t *testing.T) { + parser := hclutils.NewConfigParser(taskConfigSpec) + + validHCL := ` + config { + image = "docker://redis" + cpu_hard_limit = true + cpu_cfs_period = 200000 + } +` + + var tc *TaskConfig + parser.ParseHCL(t, validHCL, &tc) + require.EqualValues(t, true, tc.CPUHardLimit) + require.EqualValues(t, 200000, tc.CPUCFSPeriod) +} diff --git a/driver.go b/driver.go index 538d1271..fccade6b 100644 --- a/driver.go +++ b/driver.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "path/filepath" + "runtime" "strconv" "strings" "time" @@ -419,6 +420,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive CPU: &spec.LinuxCPU{}, } + err = setCPUResources(driverConfig.CPUHardLimit, driverConfig.CPUCFSPeriod, cfg.Resources.LinuxResources, createOpts.ContainerResourceConfig.ResourceLimits.CPU) + if err != nil { + return nil, nil, err + } + hard, soft, err := memoryLimits(cfg.Resources.NomadResources.Memory, driverConfig.MemoryReservation) if err != nil { return nil, nil, err @@ -644,6 +650,28 @@ func memoryLimits(r drivers.MemoryResources, reservation string) (hard, soft *in return nil, reserved, nil } +func setCPUResources(hardLimit bool, period uint64, systemResources *drivers.LinuxResources, taskCPU *spec.LinuxCPU) error { + if hardLimit { + numCores := runtime.NumCPU() + if period > 1000000 { + return fmt.Errorf("invalid value for cpu_cfs_period") + } + if period == 0 { + period = 100000 // matches cgroup default + } + if period < 1000 { + period = 1000 + } + quota := int64(systemResources.PercentTicks*float64(period)) * int64(numCores) + if quota < 1000 { + quota = 1000 + } + taskCPU.Period = &period + taskCPU.Quota = "a + } + return nil +} + func memoryInBytes(strmem string) (int64, error) { l := len(strmem) if l < 2 { diff --git a/driver_test.go b/driver_test.go index bf260860..a0cd618d 100644 --- a/driver_test.go +++ b/driver_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "os/exec" + "runtime" "strings" "path/filepath" @@ -25,6 +26,7 @@ import ( "github.com/hashicorp/nomad/plugins/drivers" dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" tu "github.com/hashicorp/nomad/testutil" + spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/require" ) @@ -45,8 +47,11 @@ func createBasicResources() *drivers.Resources { }, }, LinuxResources: &drivers.LinuxResources{ - CPUShares: 512, + CPUPeriod: 100000, + CPUQuota: 100000, + CPUShares: 500, MemoryLimitBytes: 256 * 1024 * 1024, + PercentTicks: float64(500) / float64(2000), }, } return &res @@ -1820,6 +1825,90 @@ func createInspectImage(t *testing.T, image, reference string) { require.Equal(t, idRef, idTest) } +func Test_cpuLimits(t *testing.T) { + numCores := runtime.NumCPU() + cases := []struct { + name string + systemResources drivers.LinuxResources + hardLimit bool + period uint64 + expectedQuota int64 + expectedPeriod uint64 + }{ + { + name: "no hard limit", + systemResources: drivers.LinuxResources{ + PercentTicks: 1.0, + CPUPeriod: 100000, + }, + hardLimit: false, + expectedQuota: 100000, + expectedPeriod: 100000, + }, + { + name: "hard limit max quota", + systemResources: drivers.LinuxResources{ + PercentTicks: 1.0, + CPUPeriod: 100000, + }, + hardLimit: true, + expectedQuota: 100000, + expectedPeriod: 100000, + }, + { + name: "hard limit, half quota", + systemResources: drivers.LinuxResources{ + PercentTicks: 0.5, + CPUPeriod: 100000, + }, + hardLimit: true, + expectedQuota: 50000, + expectedPeriod: 100000, + }, + { + name: "hard limit, custom period", + systemResources: drivers.LinuxResources{ + PercentTicks: 1.0, + CPUPeriod: 100000, + }, + hardLimit: true, + period: 20000, + expectedQuota: 20000, + expectedPeriod: 20000, + }, + { + name: "hard limit, half quota, custom period", + systemResources: drivers.LinuxResources{ + PercentTicks: 0.5, + CPUPeriod: 100000, + }, + hardLimit: true, + period: 20000, + expectedQuota: 10000, + expectedPeriod: 20000, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + taskCPU := &spec.LinuxCPU{} + err := setCPUResources(c.hardLimit, c.period, &c.systemResources, taskCPU) + require.Nil(t, err) + + if c.hardLimit { + require.NotNil(t, taskCPU.Quota) + require.Equal(t, c.expectedQuota*int64(numCores), *taskCPU.Quota) + + require.NotNil(t, taskCPU.Period) + require.Equal(t, c.expectedPeriod, *taskCPU.Period) + } else { + require.Nil(t, taskCPU.Quota) + require.Nil(t, taskCPU.Period) + } + }) + } +} + func Test_memoryLimits(t *testing.T) { cases := []struct { name string