Add cpu_hard_limit and cpu_cfs_period options

Nomad's docker driver allows having nomad set cpu resources as hard
limits rather than soft limits via the `cpu_hard_limits` driver config
option. This is useful to restrict how much CPU time a nomad job is
permitted to consume, even if the host is under utilised, or when
predicatibility is more important than speed.

Default behaviour is unchanged. When `cpu_hard_limit` is set to `true`
the podman container is started with the equivalent API options to the
podman cli `--cpu-quota` and `--cpu-period`.

Nomad currently provides a `PercentTicks` value which is used to convert
the cpu resource from the specification into a number of microseconds
of cputime the job is allowed to consume for each `period` interval of time.
Use of this value is discouraged ouside of the docker driver, however the
desired long term replacement is to expose `cpu quota` and `cpu period` as
options in the task `resource` block, and these are not yet currently exposed.
Since this driver emulates the docker interface where possible, I think it's
acceptable to use the `PercentTicks` value in spite of the warning to the contrary.

The `cpu_cfs_period` defaults to 100,000 (microseconds) in both the linux
kernel, and this driver when not specified by the user. User may override
this between 1,000 and 1,000,000. Any value less than 1,000 is silently
increased to 1,000 to prevent
the job failing to run with an `Invalid Operation` error.

The minimum value for the quota is also 1,000. Since this value is
computed by this driver, it is silently increased to 1,000 if the
computed value is lower. For jobs with very small cpu resources in
the job spec, which equate to less than 1% of total available cpu,
the job may consume more cpu time than intended.

CHANGELOG.md

@@ -4,6 +4,7 @@ FEATURES:
 * config: Map host devices into container. [[GH-41](https://github.com/hashicorp/nomad-driver-podman/pull/41)]
 * config: Stream logs via API, support journald log driver. [[GH-99](https://github.com/hashicorp/nomad-driver-podman/pull/99)]
 * config: Privileged containers.
+* config: Add `cpu_hard_limit` and `cpu_cfs_period` options
 
 BUG FIXES:
 * log: Use error key context to log errors rather than Go err style. [[GH-126](https://github.com/hashicorp/nomad-driver-podman/pull/126)]

config.go

@@ -47,17 +47,19 @@ var (
 			"username": hclspec.NewAttr("username", "string", false),
 			"password": hclspec.NewAttr("password", "string", false),
 		})),
-		"command":     hclspec.NewAttr("command", "string", false),
-		"cap_add":     hclspec.NewAttr("cap_add", "list(string)", false),
-		"cap_drop":    hclspec.NewAttr("cap_drop", "list(string)", false),
-		"devices":     hclspec.NewAttr("devices", "list(string)", false),
-		"entrypoint":  hclspec.NewAttr("entrypoint", "string", false),
-		"working_dir": hclspec.NewAttr("working_dir", "string", false),
-		"hostname":    hclspec.NewAttr("hostname", "string", false),
-		"image":       hclspec.NewAttr("image", "string", true),
-		"init":        hclspec.NewAttr("init", "bool", false),
-		"init_path":   hclspec.NewAttr("init_path", "string", false),
-		"labels":      hclspec.NewAttr("labels", "list(map(string))", false),
+		"command":        hclspec.NewAttr("command", "string", false),
+		"cap_add":        hclspec.NewAttr("cap_add", "list(string)", false),
+		"cap_drop":       hclspec.NewAttr("cap_drop", "list(string)", false),
+		"cpu_hard_limit": hclspec.NewAttr("cpu_hard_limit", "bool", false),
+		"cpu_cfs_period": hclspec.NewAttr("cpu_cfs_period", "number", false),
+		"devices":        hclspec.NewAttr("devices", "list(string)", false),
+		"entrypoint":     hclspec.NewAttr("entrypoint", "string", false),
+		"working_dir":    hclspec.NewAttr("working_dir", "string", false),
+		"hostname":       hclspec.NewAttr("hostname", "string", false),
+		"image":          hclspec.NewAttr("image", "string", true),
+		"init":           hclspec.NewAttr("init", "bool", false),
+		"init_path":      hclspec.NewAttr("init_path", "string", false),
+		"labels":         hclspec.NewAttr("labels", "list(map(string))", false),
 		"logging": hclspec.NewBlock("logging", false, hclspec.NewObject(map[string]*hclspec.Spec{
 			"driver":  hclspec.NewAttr("driver", "string", false),
 			"options": hclspec.NewAttr("options", "list(map(string))", false),
@@ -130,9 +132,11 @@ type TaskConfig struct {
 	MemoryReservation string             `codec:"memory_reservation"`
 	MemorySwap        string             `codec:"memory_swap"`
 	NetworkMode       string             `codec:"network_mode"`
+	CPUCFSPeriod      uint64             `codec:"cpu_cfs_period"`
 	MemorySwappiness  int64              `codec:"memory_swappiness"`
 	PortMap           hclutils.MapStrInt `codec:"port_map"`
 	Sysctl            hclutils.MapStrStr `codec:"sysctl"`
+	CPUHardLimit      bool               `codec:"cpu_hard_limit"`
 	Init              bool               `codec:"init"`
 	Tty               bool               `codec:"tty"`
 	ForcePull         bool               `codec:"force_pull"`

config_test.go

@@ -81,3 +81,20 @@ func TestConfig_ForcePull(t *testing.T) {
 	parser.ParseHCL(t, validHCL, &tc)
 	require.EqualValues(t, true, tc.ForcePull)
 }
+
+func TestConfig_CPUHardLimit(t *testing.T) {
+	parser := hclutils.NewConfigParser(taskConfigSpec)
+
+	validHCL := `
+  config {
+		image = "docker://redis"
+		cpu_hard_limit = true
+		cpu_cfs_period = 200000
+  }
+`
+
+	var tc *TaskConfig
+	parser.ParseHCL(t, validHCL, &tc)
+	require.EqualValues(t, true, tc.CPUHardLimit)
+	require.EqualValues(t, 200000, tc.CPUCFSPeriod)
+}

driver.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -419,6 +420,11 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		CPU:    &spec.LinuxCPU{},
 	}
 
+	err = setCPUResources(driverConfig.CPUHardLimit, driverConfig.CPUCFSPeriod, cfg.Resources.LinuxResources, createOpts.ContainerResourceConfig.ResourceLimits.CPU)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	hard, soft, err := memoryLimits(cfg.Resources.NomadResources.Memory, driverConfig.MemoryReservation)
 	if err != nil {
 		return nil, nil, err
@@ -644,6 +650,28 @@ func memoryLimits(r drivers.MemoryResources, reservation string) (hard, soft *in
 	return nil, reserved, nil
 }
 
+func setCPUResources(hardLimit bool, period uint64, systemResources *drivers.LinuxResources, taskCPU *spec.LinuxCPU) error {
+	if hardLimit {
+		numCores := runtime.NumCPU()
+		if period > 1000000 {
+			return fmt.Errorf("invalid value for cpu_cfs_period")
+		}
+		if period == 0 {
+			period = 100000 // matches cgroup default
+		}
+		if period < 1000 {
+			period = 1000
+		}
+		quota := int64(systemResources.PercentTicks*float64(period)) * int64(numCores)
+		if quota < 1000 {
+			quota = 1000
+		}
+		taskCPU.Period = &period
+		taskCPU.Quota = &quota
+	}
+	return nil
+}
+
 func memoryInBytes(strmem string) (int64, error) {
 	l := len(strmem)
 	if l < 2 {

driver_test.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"runtime"
 	"strings"
 
 	"path/filepath"
@@ -25,6 +26,7 @@ import (
 	"github.com/hashicorp/nomad/plugins/drivers"
 	dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
 	tu "github.com/hashicorp/nomad/testutil"
+	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/require"
 )
 
@@ -45,8 +47,11 @@ func createBasicResources() *drivers.Resources {
 			},
 		},
 		LinuxResources: &drivers.LinuxResources{
-			CPUShares:        512,
+			CPUPeriod:        100000,
+			CPUQuota:         100000,
+			CPUShares:        500,
 			MemoryLimitBytes: 256 * 1024 * 1024,
+			PercentTicks:     float64(500) / float64(2000),
 		},
 	}
 	return &res
@@ -1820,6 +1825,90 @@ func createInspectImage(t *testing.T, image, reference string) {
 	require.Equal(t, idRef, idTest)
 }
 
+func Test_cpuLimits(t *testing.T) {
+	numCores := runtime.NumCPU()
+	cases := []struct {
+		name            string
+		systemResources drivers.LinuxResources
+		hardLimit       bool
+		period          uint64
+		expectedQuota   int64
+		expectedPeriod  uint64
+	}{
+		{
+			name: "no hard limit",
+			systemResources: drivers.LinuxResources{
+				PercentTicks: 1.0,
+				CPUPeriod:    100000,
+			},
+			hardLimit:      false,
+			expectedQuota:  100000,
+			expectedPeriod: 100000,
+		},
+		{
+			name: "hard limit max quota",
+			systemResources: drivers.LinuxResources{
+				PercentTicks: 1.0,
+				CPUPeriod:    100000,
+			},
+			hardLimit:      true,
+			expectedQuota:  100000,
+			expectedPeriod: 100000,
+		},
+		{
+			name: "hard limit, half quota",
+			systemResources: drivers.LinuxResources{
+				PercentTicks: 0.5,
+				CPUPeriod:    100000,
+			},
+			hardLimit:      true,
+			expectedQuota:  50000,
+			expectedPeriod: 100000,
+		},
+		{
+			name: "hard limit, custom period",
+			systemResources: drivers.LinuxResources{
+				PercentTicks: 1.0,
+				CPUPeriod:    100000,
+			},
+			hardLimit:      true,
+			period:         20000,
+			expectedQuota:  20000,
+			expectedPeriod: 20000,
+		},
+		{
+			name: "hard limit, half quota, custom period",
+			systemResources: drivers.LinuxResources{
+				PercentTicks: 0.5,
+				CPUPeriod:    100000,
+			},
+			hardLimit:      true,
+			period:         20000,
+			expectedQuota:  10000,
+			expectedPeriod: 20000,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			taskCPU := &spec.LinuxCPU{}
+			err := setCPUResources(c.hardLimit, c.period, &c.systemResources, taskCPU)
+			require.Nil(t, err)
+
+			if c.hardLimit {
+				require.NotNil(t, taskCPU.Quota)
+				require.Equal(t, c.expectedQuota*int64(numCores), *taskCPU.Quota)
+
+				require.NotNil(t, taskCPU.Period)
+				require.Equal(t, c.expectedPeriod, *taskCPU.Period)
+			} else {
+				require.Nil(t, taskCPU.Quota)
+				require.Nil(t, taskCPU.Period)
+			}
+		})
+	}
+}
+
 func Test_memoryLimits(t *testing.T) {
 	cases := []struct {
 		name         string