From 1876fc2b108cbdab19c3870863dd727bdbc89da6 Mon Sep 17 00:00:00 2001 From: Jay Oster Date: Mon, 1 Aug 2016 19:58:32 -0700 Subject: [PATCH 1/6] Add config field to specify chroot mapping for exec driver - Same format as used by the internal chroot mapping - Map: source_path -> dest_path - Example HCL: client { chroot_env { "/etc" = "/etc" "/lib" = "/lib" "/opt/projects/foo/bin" = "/usr/bin" } } --- client/config/config.go | 4 ++++ client/driver/exec.go | 1 + client/driver/executor/executor.go | 4 ++++ client/driver/executor/executor_linux.go | 7 ++++++- command/agent/agent.go | 1 + command/agent/config.go | 12 ++++++++++++ command/agent/config_parse.go | 16 ++++++++++++++++ 7 files changed, 44 insertions(+), 1 deletion(-) diff --git a/client/config/config.go b/client/config/config.go index 127926115cd..f18559428dd 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -99,6 +99,10 @@ type Config struct { // devices and IPs. GloballyReservedPorts []int + // A mapping of directories on the host OS to attempt to embed inside each + // task's chroot. + ChrootEnv map[string]string + // Options provides arbitrary key-value configuration for nomad internals, // like fingerprinters and drivers. The format is: // diff --git a/client/driver/exec.go b/client/driver/exec.go index ab534b82071..b346f387a81 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -132,6 +132,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, Cmd: command, Args: driverConfig.Args, FSIsolation: true, + ChrootEnv: d.config.ChrootEnv, ResourceLimits: true, User: getExecutorUser(task), }, executorCtx) diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 7685bfdf25c..17984b22629 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -126,6 +126,10 @@ type ExecCommand struct { // FSIsolation determines whether the command would be run in a chroot. FSIsolation bool + // A mapping of directories on the host OS to attempt to embed inside each + // task's chroot. + ChrootEnv map[string]string + // User is the user which the executor uses to run the command. User string diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go index 1f6fef13cc4..6b1d1b44538 100644 --- a/client/driver/executor/executor_linux.go +++ b/client/driver/executor/executor_linux.go @@ -227,7 +227,12 @@ func (e *UniversalExecutor) configureChroot() error { return err } - if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil { + chroot := chrootEnv + if e.command.ChrootEnv != nil && len(e.command.ChrootEnv) > 0 { + chroot = e.command.ChrootEnv + } + + if err := allocDir.Embed(e.ctx.Task.Name, chroot); err != nil { return err } diff --git a/command/agent/agent.go b/command/agent/agent.go index 2f9701ce294..f1558f1b481 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -274,6 +274,7 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) { if a.config.Client.NetworkInterface != "" { conf.NetworkInterface = a.config.Client.NetworkInterface } + conf.ChrootEnv = a.config.Client.ChrootEnv conf.Options = a.config.Client.Options // Logging deprecation messages about consul related configuration in client // options diff --git a/command/agent/config.go b/command/agent/config.go index b7c820bceb9..142db652d91 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -156,6 +156,10 @@ type ClientConfig struct { // Metadata associated with the node Meta map[string]string `mapstructure:"meta"` + // A mapping of directories on the host OS to attempt to embed inside each + // task's chroot. + ChrootEnv map[string]string `mapstructure:"chroot_env"` + // Interface to use for network fingerprinting NetworkInterface string `mapstructure:"network_interface"` @@ -718,6 +722,14 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig { result.Meta[k] = v } + // Add the chroot_env map values + if result.ChrootEnv == nil { + result.ChrootEnv = make(map[string]string) + } + for k, v := range b.ChrootEnv { + result.ChrootEnv[k] = v + } + return &result } diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index 6abacfb10fd..462b61cdf0c 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -315,6 +315,7 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error { "node_class", "options", "meta", + "chroot_env", "network_interface", "network_speed", "max_kill_timeout", @@ -334,6 +335,7 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error { delete(m, "options") delete(m, "meta") + delete(m, "chroot_env") delete(m, "reserved") delete(m, "stats") @@ -370,6 +372,20 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error { } } + // Parse out chroot_env fields. These are in HCL as a list so we need to + // iterate over them and merge them. + if chrootEnvO := listVal.Filter("chroot_env"); len(chrootEnvO.Items) > 0 { + for _, o := range chrootEnvO.Elem().Items { + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + if err := mapstructure.WeakDecode(m, &config.ChrootEnv); err != nil { + return err + } + } + } + // Parse reserved config if o := listVal.Filter("reserved"); len(o.Items) > 0 { if err := parseReserved(&config.Reserved, o); err != nil { From 09f2fc9f8faa18ca7014210491e77c1311769ee0 Mon Sep 17 00:00:00 2001 From: Jay Oster Date: Thu, 4 Aug 2016 11:14:15 -0700 Subject: [PATCH 2/6] Add chroot_env to Java driver (which uses the exec driver internally) --- client/driver/java.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/driver/java.go b/client/driver/java.go index 532199433f4..4488a5e338e 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -219,6 +219,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, Cmd: absPath, Args: args, FSIsolation: true, + ChrootEnv: d.config.ChrootEnv, ResourceLimits: true, User: getExecutorUser(task), }, executorCtx) From 593669ada2e50eec1505ee68cea8e7d534d83f4a Mon Sep 17 00:00:00 2001 From: Jay Oster Date: Thu, 4 Aug 2016 15:03:56 -0700 Subject: [PATCH 3/6] Address review comments - Simplify map length check in Linux Executor - Added a `chroot_env` test for config parser - Moved `ChrootEnv` field from ExecutorCommand to ExecutorContext - Added a test for `chroot_env` functionality --- client/driver/exec.go | 12 ++++---- client/driver/executor/executor.go | 8 ++--- client/driver/executor/executor_linux.go | 4 +-- client/driver/executor/executor_linux_test.go | 29 +++++++++++++++++-- client/driver/java.go | 12 ++++---- command/agent/config-test-fixtures/basic.hcl | 4 +++ command/agent/config_parse_test.go | 4 +++ 7 files changed, 52 insertions(+), 21 deletions(-) diff --git a/client/driver/exec.go b/client/driver/exec.go index b346f387a81..728ed3f5f74 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -121,18 +121,18 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, err } executorCtx := &executor.ExecutorContext{ - TaskEnv: d.taskEnv, - Driver: "exec", - AllocDir: ctx.AllocDir, - AllocID: ctx.AllocID, - Task: task, + TaskEnv: d.taskEnv, + Driver: "exec", + AllocDir: ctx.AllocDir, + AllocID: ctx.AllocID, + ChrootEnv: d.config.ChrootEnv, + Task: task, } ps, err := exec.LaunchCmd(&executor.ExecCommand{ Cmd: command, Args: driverConfig.Args, FSIsolation: true, - ChrootEnv: d.config.ChrootEnv, ResourceLimits: true, User: getExecutorUser(task), }, executorCtx) diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 17984b22629..7fc973e6307 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -102,6 +102,10 @@ type ExecutorContext struct { // AllocID is the allocation id to which the task belongs AllocID string + // A mapping of directories on the host OS to attempt to embed inside each + // task's chroot. + ChrootEnv map[string]string + // Driver is the name of the driver that invoked the executor Driver string @@ -126,10 +130,6 @@ type ExecCommand struct { // FSIsolation determines whether the command would be run in a chroot. FSIsolation bool - // A mapping of directories on the host OS to attempt to embed inside each - // task's chroot. - ChrootEnv map[string]string - // User is the user which the executor uses to run the command. User string diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go index 6b1d1b44538..c673555cc5f 100644 --- a/client/driver/executor/executor_linux.go +++ b/client/driver/executor/executor_linux.go @@ -228,8 +228,8 @@ func (e *UniversalExecutor) configureChroot() error { } chroot := chrootEnv - if e.command.ChrootEnv != nil && len(e.command.ChrootEnv) > 0 { - chroot = e.command.ChrootEnv + if len(e.ctx.ChrootEnv) > 0 { + chroot = e.ctx.ChrootEnv } if err := allocDir.Embed(e.ctx.Task.Name, chroot); err != nil { diff --git a/client/driver/executor/executor_linux_test.go b/client/driver/executor/executor_linux_test.go index 9af75dec51d..fcc0e088a3f 100644 --- a/client/driver/executor/executor_linux_test.go +++ b/client/driver/executor/executor_linux_test.go @@ -9,15 +9,38 @@ import ( "strings" "testing" + "github.com/hashicorp/nomad/client/driver/env" cstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/client/testutil" + "github.com/hashicorp/nomad/nomad/mock" ) +func testExecutorContextWithChroot(t *testing.T) *ExecutorContext { + taskEnv := env.NewTaskEnvironment(mock.Node()) + task, allocDir := mockAllocDir(t) + ctx := &ExecutorContext{ + TaskEnv: taskEnv, + Task: task, + AllocDir: allocDir, + ChrootEnv: map[string]string{ + "/etc/ld.so.cache": "/etc/ld.so.cache", + "/etc/ld.so.conf": "/etc/ld.so.conf", + "/etc/ld.so.conf.d": "/etc/ld.so.conf.d", + "/lib": "/lib", + "/lib64": "/lib64", + "/usr/lib": "/usr/lib", + "/bin/ls": "/bin/ls", + "/foobar": "/does/not/exist", + }, + } + return ctx +} + func TestExecutor_IsolationAndConstraints(t *testing.T) { testutil.ExecCompatible(t) - execCmd := ExecCommand{Cmd: "/bin/echo", Args: []string{"hello world"}} - ctx := testExecutorContext(t) + execCmd := ExecCommand{Cmd: "/bin/ls", Args: []string{"-F", "/"}} + ctx := testExecutorContextWithChroot(t) defer ctx.AllocDir.Destroy() execCmd.FSIsolation = true @@ -58,7 +81,7 @@ func TestExecutor_IsolationAndConstraints(t *testing.T) { t.Fatalf("file %v hasn't been removed", memLimits) } - expected := "hello world" + expected := "bin/\netc/\nlib/\nlib64/\nusr/" file := filepath.Join(ctx.AllocDir.LogDir(), "web.stdout.0") output, err := ioutil.ReadFile(file) if err != nil { diff --git a/client/driver/java.go b/client/driver/java.go index 4488a5e338e..24e6ce6a8fd 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -203,11 +203,12 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, err } executorCtx := &executor.ExecutorContext{ - TaskEnv: d.taskEnv, - Driver: "java", - AllocDir: ctx.AllocDir, - AllocID: ctx.AllocID, - Task: task, + TaskEnv: d.taskEnv, + Driver: "java", + AllocDir: ctx.AllocDir, + AllocID: ctx.AllocID, + ChrootEnv: d.config.ChrootEnv, + Task: task, } absPath, err := GetAbsolutePath("java") @@ -219,7 +220,6 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, Cmd: absPath, Args: args, FSIsolation: true, - ChrootEnv: d.config.ChrootEnv, ResourceLimits: true, User: getExecutorUser(task), }, executorCtx) diff --git a/command/agent/config-test-fixtures/basic.hcl b/command/agent/config-test-fixtures/basic.hcl index 1663d7d7072..61d9577a927 100644 --- a/command/agent/config-test-fixtures/basic.hcl +++ b/command/agent/config-test-fixtures/basic.hcl @@ -33,6 +33,10 @@ client { foo = "bar" baz = "zip" } + chroot_env { + "/opt/myapp/etc" = "/etc" + "/opt/myapp/bin" = "/bin" + } network_interface = "eth0" network_speed = 100 reserved { diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index 04d00c43c21..9dd7593e5e1 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -53,6 +53,10 @@ func TestConfig_Parse(t *testing.T) { "foo": "bar", "baz": "zip", }, + ChrootEnv: map[string]string{ + "/opt/myapp/etc": "/etc", + "/opt/myapp/bin": "/bin", + }, NetworkInterface: "eth0", NetworkSpeed: 100, MaxKillTimeout: "10s", From 62015883ed2dd2e635f4147d2eb4c7a574984328 Mon Sep 17 00:00:00 2001 From: Jay Oster Date: Fri, 5 Aug 2016 15:59:06 -0700 Subject: [PATCH 4/6] Add documentation for chroot_env client configuration --- website/source/docs/agent/config.html.md | 28 ++++++++++++++++++++++++ website/source/docs/drivers/exec.html.md | 5 ++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/website/source/docs/agent/config.html.md b/website/source/docs/agent/config.html.md index 371c1137de7..c28fe1b89ab 100644 --- a/website/source/docs/agent/config.html.md +++ b/website/source/docs/agent/config.html.md @@ -421,6 +421,9 @@ configured on server nodes. * `options`: This is a key/value mapping of internal configuration for clients, such as for driver configuration. Please see [here](#options_map) for a description of available options. + * `chroot_env`: This is a key/value mapping that + defines the chroot environment for jobs using the Exec and Java drivers. + Please see [here](#chroot_env_map) for an example and further information. * `network_interface`: This is a string to force network fingerprinting to use a specific network interface * `network_speed`: This is an int that sets the @@ -496,6 +499,31 @@ documentation [here](/docs/drivers/index.html) If specified, fingerprinters not in the whitelist will be disabled. If the whitelist is empty, all fingerprinters are used. +### Client ChrootEnv Map + +Drivers based on [Isolated Fork/Exec](/docs/drivers/exec.html) implement file +system isolation using chroot on Linux. The `chroot_env` map allows the chroot +environment to be configured using source paths on the host operating system. +The mapping format is: `source_path -> dest_path`. + +The following example specifies a chroot which contains just enough to run the +`ls` utility, and not much else: + +``` +chroot_env { + "/bin/ls" = "/bin/ls" + "/etc/ld.so.cache" = "/etc/ld.so.cache" + "/etc/ld.so.conf" = "/etc/ld.so.conf" + "/etc/ld.so.conf.d" = "/etc/ld.so.conf.d" + "/lib" = "/lib" + "/lib64" = "/lib64" +} +``` + +When `chroot_env` is unspecified, the `exec` driver will use a default chroot +environment with the most commonly used parts of the operating system. See +`exec` documentation for the full list [here](/docs/drivers/exec.html#chroot). + ## Command-line Options A subset of the available Nomad agent configuration can optionally be passed in diff --git a/website/source/docs/drivers/exec.html.md b/website/source/docs/drivers/exec.html.md index 4477c860ec7..20dfea5f765 100644 --- a/website/source/docs/drivers/exec.html.md +++ b/website/source/docs/drivers/exec.html.md @@ -96,9 +96,12 @@ the client and the configuration. On Linux, Nomad will use cgroups, and a chroot to isolate the resources of a process and as such the Nomad agent must be run as root. -### Chroot +### Chroot The chroot is populated with data in the following folders from the host machine: `["/bin", "/etc", "/lib", "/lib32", "/lib64", "/run/resolvconf", "/sbin", "/usr"]` + +This list is configurable through the agent client +[configuration file](/docs/agent/config.html#chroot_env). From bd0363b6a6b3475d9ac1757ef0ec31c9728d8de0 Mon Sep 17 00:00:00 2001 From: Jay Oster Date: Mon, 8 Aug 2016 14:04:09 -0700 Subject: [PATCH 5/6] Fix Linux executor isolation test - Properly expects the hard-coded mounts (alloc, dev, and proc) and hardcoded local directories (local and tmp) - Also verifies that etc contains only the requested paths --- client/driver/executor/executor_linux_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/driver/executor/executor_linux_test.go b/client/driver/executor/executor_linux_test.go index fcc0e088a3f..c3006071a4f 100644 --- a/client/driver/executor/executor_linux_test.go +++ b/client/driver/executor/executor_linux_test.go @@ -39,7 +39,7 @@ func testExecutorContextWithChroot(t *testing.T) *ExecutorContext { func TestExecutor_IsolationAndConstraints(t *testing.T) { testutil.ExecCompatible(t) - execCmd := ExecCommand{Cmd: "/bin/ls", Args: []string{"-F", "/"}} + execCmd := ExecCommand{Cmd: "/bin/ls", Args: []string{"-F", "/", "/etc/"}} ctx := testExecutorContextWithChroot(t) defer ctx.AllocDir.Destroy() @@ -81,7 +81,7 @@ func TestExecutor_IsolationAndConstraints(t *testing.T) { t.Fatalf("file %v hasn't been removed", memLimits) } - expected := "bin/\netc/\nlib/\nlib64/\nusr/" + expected := "/:\nalloc/\nbin/\ndev/\netc/\nlib/\nlib64/\nlocal/\nproc/\ntmp/\nusr/\n\n/etc/:\nld.so.cache\nld.so.conf\nld.so.conf.d/" file := filepath.Join(ctx.AllocDir.LogDir(), "web.stdout.0") output, err := ioutil.ReadFile(file) if err != nil { From 95c1d765445b429dc96d71ede03bc948be6ea870 Mon Sep 17 00:00:00 2001 From: Jay Oster Date: Mon, 8 Aug 2016 18:58:49 -0700 Subject: [PATCH 6/6] Fix ConfigMerge test with chroot config --- command/agent/config_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 8306012288e..b83454bb15b 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -139,6 +139,7 @@ func TestConfig_Merge(t *testing.T) { "foo": "bar", "baz": "zip", }, + ChrootEnv: map[string]string{}, ClientMaxPort: 20000, ClientMinPort: 22000, NetworkSpeed: 105,