diff --git a/.changelog/19515.txt b/.changelog/19515.txt new file mode 100644 index 00000000000..0237b257b9f --- /dev/null +++ b/.changelog/19515.txt @@ -0,0 +1,3 @@ +```release-note:bug +rawexec: Fixed a bug where oom_score_adj would be inherited from Nomad client +``` diff --git a/drivers/shared/executor/executor_universal_linux.go b/drivers/shared/executor/executor_universal_linux.go index 4aab9199d22..32c4bd90295 100644 --- a/drivers/shared/executor/executor_universal_linux.go +++ b/drivers/shared/executor/executor_universal_linux.go @@ -5,6 +5,7 @@ package executor import ( "fmt" + "os" "os/exec" "strconv" "syscall" @@ -112,6 +113,11 @@ func (e *UniversalExecutor) statCG(cgroup string) (int, func(), error) { func (e *UniversalExecutor) configureResourceContainer(command *ExecCommand, pid int) (func(), error) { cgroup := command.StatsCgroup() + // ensure tasks do not inherit Nomad agent oom_score_adj value + if err := e.setOomAdj(); err != nil { + return nil, err + } + // cgCleanup will be called after the task has been launched // v1: remove the executor process from the task's cgroups // v2: let go of the file descriptor of the task's cgroup @@ -244,6 +250,14 @@ func (e *UniversalExecutor) configureCG2(cgroup string, command *ExecCommand) { _ = ed.Write("cpuset.cpus", cpusetCpus) } +func (e *UniversalExecutor) setOomAdj() error { + // children should not inherit Nomad agent oom_score_adj value + // + // /proc/self/oom_score_adj should work on both cgroups v1 and v2 systems + // range is -1000 to 1000; 0 is the default + return os.WriteFile("/proc/self/oom_score_adj", []byte("0"), 0644) +} + func (*UniversalExecutor) computeCPU(command *ExecCommand) uint64 { cpuShares := command.Resources.LinuxResources.CPUShares cpuWeight := cgroups.ConvertCPUSharesToCgroupV2Value(uint64(cpuShares)) diff --git a/e2e/rawexec/doc.go b/e2e/rawexec/doc.go new file mode 100644 index 00000000000..dac71294eef --- /dev/null +++ b/e2e/rawexec/doc.go @@ -0,0 +1,5 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +// Package rawexec tests the raw_exec task driver. +package rawexec diff --git a/e2e/rawexec/input/oomadj.hcl b/e2e/rawexec/input/oomadj.hcl new file mode 100644 index 00000000000..bc6abe51547 --- /dev/null +++ b/e2e/rawexec/input/oomadj.hcl @@ -0,0 +1,32 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +job "oomadj" { + type = "batch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "group" { + + reschedule { + attempts = 0 + unlimited = false + } + + restart { + attempts = 0 + mode = "fail" + } + + task "cat" { + driver = "raw_exec" + config { + command = "cat" + args = ["/proc/self/oom_score_adj"] + } + } + } +} diff --git a/e2e/rawexec/rawexec_test.go b/e2e/rawexec/rawexec_test.go new file mode 100644 index 00000000000..8907e797e16 --- /dev/null +++ b/e2e/rawexec/rawexec_test.go @@ -0,0 +1,29 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package rawexec + +import ( + "testing" + + "github.com/hashicorp/nomad/e2e/v3/cluster3" + "github.com/hashicorp/nomad/e2e/v3/jobs3" + "github.com/shoenig/test/must" +) + +func TestRawExec(t *testing.T) { + cluster3.Establish(t, + cluster3.Leader(), + cluster3.LinuxClients(1), + ) + + t.Run("testOomAdj", testOomAdj) +} + +func testOomAdj(t *testing.T) { + job, cleanup := jobs3.Submit(t, "./input/oomadj.hcl") + t.Cleanup(cleanup) + + logs := job.TaskLogs("group", "cat") + must.StrContains(t, logs.Stdout, "0") +} diff --git a/e2e/terraform/etc/nomad.d/nomad-client.service b/e2e/terraform/etc/nomad.d/nomad-client.service index e37b995c09e..ef6a95e14a2 100644 --- a/e2e/terraform/etc/nomad.d/nomad-client.service +++ b/e2e/terraform/etc/nomad.d/nomad-client.service @@ -17,6 +17,7 @@ LimitNPROC=infinity TasksMax=infinity Restart=on-failure RestartSec=2 +OOMScoreAdjust=-999 [Install] WantedBy=multi-user.target