From 8cfe123a3aabbd7c760c4284f3af026544b47871 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 7 Apr 2022 08:52:07 -0500 Subject: [PATCH 1/2] client: set environment variable indicating set of reserved cpu cores This PR injects the 'NOMAD_CPU_CORES' environment variable into tasks that have been allocated reserved cpu cores. The value uses normal cpuset notation, as found in cpuset.cpu cgroup interface files. Note this value is not necessiarly the same as the content of the actual cpuset.cpus interface file, which will also include shared cpu cores when using cgroups v2. This variable is a workaround for users who used to be able to read the reserved cgroup cpuset file, but lose the information about distinct reserved cores when using cgroups v2. Side discussion in: https://github.com/hashicorp/nomad/issues/12374 --- client/taskenv/env.go | 11 ++++++++++- client/taskenv/env_test.go | 8 +++++++- website/content/partials/envvars.mdx | 9 +++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/client/taskenv/env.go b/client/taskenv/env.go index 1b6d4ce788f..eaed60567a2 100644 --- a/client/taskenv/env.go +++ b/client/taskenv/env.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/nomad/helper" hargs "github.com/hashicorp/nomad/helper/args" + "github.com/hashicorp/nomad/lib/cpuset" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/drivers" "github.com/zclconf/go-cty/cty" @@ -40,6 +41,9 @@ const ( // CpuLimit is the environment variable with the tasks CPU limit in MHz. CpuLimit = "NOMAD_CPU_LIMIT" + // CpuCores is the environment variable for passing the task's reserved cpu cores + CpuCores = "NOMAD_CPU_CORES" + // AllocID is the environment variable for passing the allocation ID. AllocID = "NOMAD_ALLOC_ID" @@ -397,6 +401,7 @@ type Builder struct { // clientTaskSecretsDir is the secrets dir from the client's perspective; eg /secrets clientTaskSecretsDir string + cpuCores string cpuLimit int64 memLimit int64 memMaxLimit int64 @@ -493,6 +498,9 @@ func (b *Builder) buildEnv(allocDir, localDir, secretsDir string, if b.cpuLimit != 0 { envMap[CpuLimit] = strconv.FormatInt(b.cpuLimit, 10) } + if b.cpuCores != "" { + envMap[CpuCores] = b.cpuCores + } // Add the task metadata if b.allocId != "" { @@ -742,6 +750,7 @@ func (b *Builder) setAlloc(alloc *structs.Allocation) *Builder { // Populate task resources if tr, ok := alloc.AllocatedResources.Tasks[b.taskName]; ok { b.cpuLimit = tr.Cpu.CpuShares + b.cpuCores = cpuset.New(tr.Cpu.ReservedCores...).String() b.memLimit = tr.Memory.MemoryMB b.memMaxLimit = tr.Memory.MemoryMaxMB @@ -788,7 +797,7 @@ func (b *Builder) setAlloc(alloc *structs.Allocation) *Builder { } } - upstreams := []structs.ConsulUpstream{} + var upstreams []structs.ConsulUpstream for _, svc := range tg.Services { if svc.Connect.HasSidecar() && svc.Connect.SidecarService.HasUpstreams() { upstreams = append(upstreams, svc.Connect.SidecarService.Proxy.Upstreams...) diff --git a/client/taskenv/env_test.go b/client/taskenv/env_test.go index 32e6238169e..5d9bf75e09e 100644 --- a/client/taskenv/env_test.go +++ b/client/taskenv/env_test.go @@ -159,7 +159,10 @@ func TestEnvironment_AsList(t *testing.T) { a := mock.Alloc() a.Job.ParentID = fmt.Sprintf("mock-parent-service-%s", uuid.Generate()) a.AllocatedResources.Tasks["web"] = &structs.AllocatedTaskResources{ - Cpu: structs.AllocatedCpuResources{CpuShares: 500}, + Cpu: structs.AllocatedCpuResources{ + CpuShares: 500, + ReservedCores: []uint16{0, 5, 6, 7}, + }, Memory: structs.AllocatedMemoryResources{ MemoryMB: 256, MemoryMaxMB: 512, @@ -215,6 +218,7 @@ func TestEnvironment_AsList(t *testing.T) { "NOMAD_PORT_ssh_other=1234", "NOMAD_PORT_ssh_ssh=22", "NOMAD_CPU_LIMIT=500", + "NOMAD_CPU_CORES=0,5-7", "NOMAD_DC=dc1", "NOMAD_NAMESPACE=not-default", "NOMAD_REGION=global", @@ -260,6 +264,7 @@ func TestEnvironment_AllValues(t *testing.T) { MBits: 50, DynamicPorts: []structs.Port{{Label: "http", Value: 80}}, } + a.AllocatedResources.Tasks["web"].Cpu.ReservedCores = []uint16{0, 5, 6, 7} a.AllocatedResources.Tasks["ssh"] = &structs.AllocatedTaskResources{ Networks: []*structs.NetworkResource{ { @@ -378,6 +383,7 @@ func TestEnvironment_AllValues(t *testing.T) { "NOMAD_PORT_ssh_other": "1234", "NOMAD_PORT_ssh_ssh": "22", "NOMAD_CPU_LIMIT": "500", + "NOMAD_CPU_CORES": "0,5-7", "NOMAD_DC": "dc1", "NOMAD_PARENT_CGROUP": "abc.slice", "NOMAD_NAMESPACE": "default", diff --git a/website/content/partials/envvars.mdx b/website/content/partials/envvars.mdx index 019603d741a..9edca49a950 100644 --- a/website/content/partials/envvars.mdx +++ b/website/content/partials/envvars.mdx @@ -54,6 +54,15 @@ CPU limit in MHz for the task + + + NOMAD_CPU_CORES + + + The specific CPU cores reserved for the task in cpuset list notation. + Omitted if the the task does not request cpu cores. E.g. 0-2,7,12-14 + + NOMAD_ALLOC_ID From 5c5607a0009ced12c88dde01e82636e9c5eaf41a Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 7 Apr 2022 10:02:00 -0500 Subject: [PATCH 2/2] docs: update cl --- .changelog/12496.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/12496.txt diff --git a/.changelog/12496.txt b/.changelog/12496.txt new file mode 100644 index 00000000000..ed5ca473c14 --- /dev/null +++ b/.changelog/12496.txt @@ -0,0 +1,3 @@ +```release-note:improvement +client: set NOMAD_CPU_CORES environment variable when reserving cpu cores +```