From 1098e562fcecc2829c7f52fce66c493e800b166a Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 7 Oct 2015 19:00:05 -0700 Subject: [PATCH 1/5] Privileged exec driver --- client/driver/driver.go | 3 +- client/driver/exec_test.go | 3 +- client/driver/pexec.go | 197 ++++++++++++++++++++++++++++++++ client/driver/pexec_test.go | 216 ++++++++++++++++++++++++++++++++++++ 4 files changed, 416 insertions(+), 3 deletions(-) create mode 100644 client/driver/pexec.go create mode 100644 client/driver/pexec_test.go diff --git a/client/driver/driver.go b/client/driver/driver.go index 46b18538c1e..9025b952aac 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -17,6 +17,7 @@ import ( var BuiltinDrivers = map[string]Factory{ "docker": NewDockerDriver, "exec": NewExecDriver, + "pexec": NewPrivilegedExecDriver, "java": NewJavaDriver, "qemu": NewQemuDriver, "rkt": NewRktDriver, @@ -112,7 +113,7 @@ func TaskEnvironmentVariables(ctx *ExecContext, task *structs.Task) environment. env.SetMeta(task.Meta) if ctx.AllocDir != nil { - env.SetAllocDir(ctx.AllocDir.AllocDir) + env.SetAllocDir(ctx.AllocDir.SharedDir) } if task.Resources != nil { diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go index 4a05d5891ff..feba89ae36d 100644 --- a/client/driver/exec_test.go +++ b/client/driver/exec_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/environment" "github.com/hashicorp/nomad/nomad/structs" @@ -159,7 +158,7 @@ func TestExecDriver_Start_Wait_AllocDir(t *testing.T) { } // Check that data was written to the shared alloc directory. - outputFile := filepath.Join(ctx.AllocDir.AllocDir, allocdir.SharedAllocName, file) + outputFile := filepath.Join(ctx.AllocDir.SharedDir, file) act, err := ioutil.ReadFile(outputFile) if err != nil { t.Fatalf("Couldn't read expected output: %v", err) diff --git a/client/driver/pexec.go b/client/driver/pexec.go new file mode 100644 index 00000000000..948b3e538d9 --- /dev/null +++ b/client/driver/pexec.go @@ -0,0 +1,197 @@ +package driver + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "github.com/hashicorp/nomad/client/allocdir" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/args" + "github.com/hashicorp/nomad/nomad/structs" +) + +const ( + // The option that enables this driver in the Config.Options map. + pexecConfigOption = "driver.pexec.enable" + + // Null files to use as stdin. + unixNull = "/dev/null" + windowsNull = "nul" +) + +// The PexecDriver is a privileged version of the exec driver. It provides no +// resource isolation and just fork/execs. The Exec driver should be preferred +// and this should only be used when explicitly needed. +type PexecDriver struct { + DriverContext +} + +// pexecHandle is returned from Start/Open as a handle to the PID +type pexecHandle struct { + proc *os.Process + waitCh chan error + doneCh chan struct{} +} + +// NewPrivilegedExecDriver is used to create a new privileged exec driver +func NewPrivilegedExecDriver(ctx *DriverContext) Driver { + return &PexecDriver{*ctx} +} + +func (d *PexecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { + // Check that the user has explicitly enabled this executor. + enabled := strings.ToLower(cfg.ReadDefault(pexecConfigOption, "false")) + if enabled == "1" || enabled == "true" { + d.logger.Printf("[WARN] driver.Pexec: privileged exec is enabled. Only enable if needed") + node.Attributes["driver.pexec"] = "1" + return true, nil + } + + return false, nil +} + +func (d *PexecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { + // Get the command + command, ok := task.Config["command"] + if !ok || command == "" { + return nil, fmt.Errorf("missing command for pexec driver") + } + + // Get the tasks local directory. + taskName := d.DriverContext.taskName + taskDir, ok := ctx.AllocDir.TaskDirs[taskName] + if !ok { + return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) + } + taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) + + // Get the environment variables. + envVars := TaskEnvironmentVariables(ctx, task) + + // Look for arguments + var cmdArgs []string + if argRaw, ok := task.Config["args"]; ok { + parsed, err := args.ParseAndReplace(argRaw, envVars.Map()) + if err != nil { + return nil, err + } + cmdArgs = append(cmdArgs, parsed...) + } + + // Setup the command + cmd := exec.Command(command, cmdArgs...) + cmd.Dir = taskDir + cmd.Env = envVars.List() + + // Capture the stdout/stderr and redirect stdin to /dev/null + stdoutFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stdout", taskName)) + stderrFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stderr", taskName)) + stdinFilename := unixNull + if runtime.GOOS == "windows" { + stdinFilename = windowsNull + } + + stdo, err := os.OpenFile(stdoutFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stdout: %v", err) + } + + stde, err := os.OpenFile(stderrFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stderr: %v", err) + } + + stdi, err := os.OpenFile(stdinFilename, os.O_CREATE|os.O_RDONLY, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stdin: %v", err) + } + + cmd.Stdout = stdo + cmd.Stderr = stde + cmd.Stdin = stdi + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start command: %v", err) + } + + // Return a driver handle + h := &pexecHandle{ + proc: cmd.Process, + doneCh: make(chan struct{}), + waitCh: make(chan error, 1), + } + go h.run() + return h, nil +} + +func (d *PexecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { + // Split the handle + pidStr := strings.TrimPrefix(handleID, "PID:") + pid, err := strconv.Atoi(pidStr) + if err != nil { + return nil, fmt.Errorf("failed to parse handle '%s': %v", handleID, err) + } + + // Find the process + proc, err := os.FindProcess(pid) + if proc == nil || err != nil { + return nil, fmt.Errorf("failed to find PID %d: %v", pid, err) + } + + // Return a driver handle + h := &pexecHandle{ + proc: proc, + doneCh: make(chan struct{}), + waitCh: make(chan error, 1), + } + go h.run() + return h, nil +} + +func (h *pexecHandle) ID() string { + // Return a handle to the PID + return fmt.Sprintf("PID:%d", h.proc.Pid) +} + +func (h *pexecHandle) WaitCh() chan error { + return h.waitCh +} + +func (h *pexecHandle) Update(task *structs.Task) error { + // Update is not possible + return nil +} + +// Kill is used to terminate the task. We send an Interrupt +// and then provide a 5 second grace period before doing a Kill on supported +// OS's, otherwise we kill immediately. +func (h *pexecHandle) Kill() error { + if runtime.GOOS == "windows" { + return h.proc.Kill() + } + + h.proc.Signal(os.Interrupt) + select { + case <-h.doneCh: + return nil + case <-time.After(5 * time.Second): + return h.proc.Kill() + } +} + +func (h *pexecHandle) run() { + ps, err := h.proc.Wait() + close(h.doneCh) + if err != nil { + h.waitCh <- err + } else if !ps.Success() { + h.waitCh <- fmt.Errorf("task exited with error") + } + close(h.waitCh) +} diff --git a/client/driver/pexec_test.go b/client/driver/pexec_test.go new file mode 100644 index 00000000000..719ace02892 --- /dev/null +++ b/client/driver/pexec_test.go @@ -0,0 +1,216 @@ +package driver + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/environment" + "github.com/hashicorp/nomad/nomad/structs" +) + +func TestPexecDriver_Fingerprint(t *testing.T) { + d := NewPrivilegedExecDriver(testDriverContext("")) + node := &structs.Node{ + Attributes: make(map[string]string), + } + + // Disable privileged exec. + cfg := &config.Config{Options: map[string]string{pexecConfigOption: "false"}} + + apply, err := d.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if apply { + t.Fatalf("should not apply") + } + if node.Attributes["driver.pexec"] != "" { + t.Fatalf("driver incorrectly enabled") + } + + // Enable privileged exec. + cfg.Options[pexecConfigOption] = "true" + apply, err = d.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if !apply { + t.Fatalf("should apply") + } + if node.Attributes["driver.pexec"] != "1" { + t.Fatalf("driver not enabled") + } +} + +func TestPexecDriver_StartOpen_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "2", + }, + } + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewPrivilegedExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Attempt to open + handle2, err := d.Open(ctx, handle.ID()) + handle2.(*pexecHandle).waitCh = make(chan error, 1) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle2 == nil { + t.Fatalf("missing handle") + } + + // Task should terminate quickly + select { + case err := <-handle2.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatalf("timeout") + } +} + +func TestPexecDriver_Start_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "1", + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewPrivilegedExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Update should be a no-op + err = handle.Update(task) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } +} + +func TestPexecDriver_Start_Wait_AllocDir(t *testing.T) { + exp := []byte{'w', 'i', 'n'} + file := "output.txt" + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/bash", + "args": fmt.Sprintf(`-c "sleep 1; echo -n %s > $%s/%s"`, string(exp), environment.AllocDir, file), + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewPrivilegedExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } + + // Check that data was written to the shared alloc directory. + outputFile := filepath.Join(ctx.AllocDir.SharedDir, file) + act, err := ioutil.ReadFile(outputFile) + if err != nil { + t.Fatalf("Couldn't read expected output: %v", err) + } + + if !reflect.DeepEqual(act, exp) { + t.Fatalf("Command outputted %v; want %v", act, exp) + } +} + +func TestPexecDriver_Start_Kill_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "1", + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewPrivilegedExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + go func() { + time.Sleep(100 * time.Millisecond) + err := handle.Kill() + if err != nil { + t.Fatalf("err: %v", err) + } + }() + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err == nil { + t.Fatal("should err") + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } +} From 7caa30b85939408754bb33fb8d8fc5e58352bf6b Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 8 Oct 2015 11:34:42 -0700 Subject: [PATCH 2/5] Change name from pexec to raw_exec; hamming distance one seemed like a bad idea --- client/driver/driver.go | 12 +- client/driver/pexec.go | 197 -------------------------------- client/driver/pexec_test.go | 216 ------------------------------------ 3 files changed, 6 insertions(+), 419 deletions(-) delete mode 100644 client/driver/pexec.go delete mode 100644 client/driver/pexec_test.go diff --git a/client/driver/driver.go b/client/driver/driver.go index 9025b952aac..31986c32131 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -15,12 +15,12 @@ import ( // BuiltinDrivers contains the built in registered drivers // which are available for allocation handling var BuiltinDrivers = map[string]Factory{ - "docker": NewDockerDriver, - "exec": NewExecDriver, - "pexec": NewPrivilegedExecDriver, - "java": NewJavaDriver, - "qemu": NewQemuDriver, - "rkt": NewRktDriver, + "docker": NewDockerDriver, + "exec": NewExecDriver, + "raw_exec": NewRawExecDriver, + "java": NewJavaDriver, + "qemu": NewQemuDriver, + "rkt": NewRktDriver, } // NewDriver is used to instantiate and return a new driver diff --git a/client/driver/pexec.go b/client/driver/pexec.go deleted file mode 100644 index 948b3e538d9..00000000000 --- a/client/driver/pexec.go +++ /dev/null @@ -1,197 +0,0 @@ -package driver - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - "strconv" - "strings" - "time" - - "github.com/hashicorp/nomad/client/allocdir" - "github.com/hashicorp/nomad/client/config" - "github.com/hashicorp/nomad/client/driver/args" - "github.com/hashicorp/nomad/nomad/structs" -) - -const ( - // The option that enables this driver in the Config.Options map. - pexecConfigOption = "driver.pexec.enable" - - // Null files to use as stdin. - unixNull = "/dev/null" - windowsNull = "nul" -) - -// The PexecDriver is a privileged version of the exec driver. It provides no -// resource isolation and just fork/execs. The Exec driver should be preferred -// and this should only be used when explicitly needed. -type PexecDriver struct { - DriverContext -} - -// pexecHandle is returned from Start/Open as a handle to the PID -type pexecHandle struct { - proc *os.Process - waitCh chan error - doneCh chan struct{} -} - -// NewPrivilegedExecDriver is used to create a new privileged exec driver -func NewPrivilegedExecDriver(ctx *DriverContext) Driver { - return &PexecDriver{*ctx} -} - -func (d *PexecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { - // Check that the user has explicitly enabled this executor. - enabled := strings.ToLower(cfg.ReadDefault(pexecConfigOption, "false")) - if enabled == "1" || enabled == "true" { - d.logger.Printf("[WARN] driver.Pexec: privileged exec is enabled. Only enable if needed") - node.Attributes["driver.pexec"] = "1" - return true, nil - } - - return false, nil -} - -func (d *PexecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { - // Get the command - command, ok := task.Config["command"] - if !ok || command == "" { - return nil, fmt.Errorf("missing command for pexec driver") - } - - // Get the tasks local directory. - taskName := d.DriverContext.taskName - taskDir, ok := ctx.AllocDir.TaskDirs[taskName] - if !ok { - return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) - } - taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) - - // Get the environment variables. - envVars := TaskEnvironmentVariables(ctx, task) - - // Look for arguments - var cmdArgs []string - if argRaw, ok := task.Config["args"]; ok { - parsed, err := args.ParseAndReplace(argRaw, envVars.Map()) - if err != nil { - return nil, err - } - cmdArgs = append(cmdArgs, parsed...) - } - - // Setup the command - cmd := exec.Command(command, cmdArgs...) - cmd.Dir = taskDir - cmd.Env = envVars.List() - - // Capture the stdout/stderr and redirect stdin to /dev/null - stdoutFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stdout", taskName)) - stderrFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stderr", taskName)) - stdinFilename := unixNull - if runtime.GOOS == "windows" { - stdinFilename = windowsNull - } - - stdo, err := os.OpenFile(stdoutFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) - if err != nil { - return nil, fmt.Errorf("Error opening file to redirect stdout: %v", err) - } - - stde, err := os.OpenFile(stderrFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) - if err != nil { - return nil, fmt.Errorf("Error opening file to redirect stderr: %v", err) - } - - stdi, err := os.OpenFile(stdinFilename, os.O_CREATE|os.O_RDONLY, 0666) - if err != nil { - return nil, fmt.Errorf("Error opening file to redirect stdin: %v", err) - } - - cmd.Stdout = stdo - cmd.Stderr = stde - cmd.Stdin = stdi - - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("failed to start command: %v", err) - } - - // Return a driver handle - h := &pexecHandle{ - proc: cmd.Process, - doneCh: make(chan struct{}), - waitCh: make(chan error, 1), - } - go h.run() - return h, nil -} - -func (d *PexecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { - // Split the handle - pidStr := strings.TrimPrefix(handleID, "PID:") - pid, err := strconv.Atoi(pidStr) - if err != nil { - return nil, fmt.Errorf("failed to parse handle '%s': %v", handleID, err) - } - - // Find the process - proc, err := os.FindProcess(pid) - if proc == nil || err != nil { - return nil, fmt.Errorf("failed to find PID %d: %v", pid, err) - } - - // Return a driver handle - h := &pexecHandle{ - proc: proc, - doneCh: make(chan struct{}), - waitCh: make(chan error, 1), - } - go h.run() - return h, nil -} - -func (h *pexecHandle) ID() string { - // Return a handle to the PID - return fmt.Sprintf("PID:%d", h.proc.Pid) -} - -func (h *pexecHandle) WaitCh() chan error { - return h.waitCh -} - -func (h *pexecHandle) Update(task *structs.Task) error { - // Update is not possible - return nil -} - -// Kill is used to terminate the task. We send an Interrupt -// and then provide a 5 second grace period before doing a Kill on supported -// OS's, otherwise we kill immediately. -func (h *pexecHandle) Kill() error { - if runtime.GOOS == "windows" { - return h.proc.Kill() - } - - h.proc.Signal(os.Interrupt) - select { - case <-h.doneCh: - return nil - case <-time.After(5 * time.Second): - return h.proc.Kill() - } -} - -func (h *pexecHandle) run() { - ps, err := h.proc.Wait() - close(h.doneCh) - if err != nil { - h.waitCh <- err - } else if !ps.Success() { - h.waitCh <- fmt.Errorf("task exited with error") - } - close(h.waitCh) -} diff --git a/client/driver/pexec_test.go b/client/driver/pexec_test.go deleted file mode 100644 index 719ace02892..00000000000 --- a/client/driver/pexec_test.go +++ /dev/null @@ -1,216 +0,0 @@ -package driver - -import ( - "fmt" - "io/ioutil" - "path/filepath" - "reflect" - "testing" - "time" - - "github.com/hashicorp/nomad/client/config" - "github.com/hashicorp/nomad/client/driver/environment" - "github.com/hashicorp/nomad/nomad/structs" -) - -func TestPexecDriver_Fingerprint(t *testing.T) { - d := NewPrivilegedExecDriver(testDriverContext("")) - node := &structs.Node{ - Attributes: make(map[string]string), - } - - // Disable privileged exec. - cfg := &config.Config{Options: map[string]string{pexecConfigOption: "false"}} - - apply, err := d.Fingerprint(cfg, node) - if err != nil { - t.Fatalf("err: %v", err) - } - if apply { - t.Fatalf("should not apply") - } - if node.Attributes["driver.pexec"] != "" { - t.Fatalf("driver incorrectly enabled") - } - - // Enable privileged exec. - cfg.Options[pexecConfigOption] = "true" - apply, err = d.Fingerprint(cfg, node) - if err != nil { - t.Fatalf("err: %v", err) - } - if !apply { - t.Fatalf("should apply") - } - if node.Attributes["driver.pexec"] != "1" { - t.Fatalf("driver not enabled") - } -} - -func TestPexecDriver_StartOpen_Wait(t *testing.T) { - task := &structs.Task{ - Name: "sleep", - Config: map[string]string{ - "command": "/bin/sleep", - "args": "2", - }, - } - driverCtx := testDriverContext(task.Name) - ctx := testDriverExecContext(task, driverCtx) - defer ctx.AllocDir.Destroy() - - d := NewPrivilegedExecDriver(driverCtx) - handle, err := d.Start(ctx, task) - if err != nil { - t.Fatalf("err: %v", err) - } - if handle == nil { - t.Fatalf("missing handle") - } - - // Attempt to open - handle2, err := d.Open(ctx, handle.ID()) - handle2.(*pexecHandle).waitCh = make(chan error, 1) - if err != nil { - t.Fatalf("err: %v", err) - } - if handle2 == nil { - t.Fatalf("missing handle") - } - - // Task should terminate quickly - select { - case err := <-handle2.WaitCh(): - if err != nil { - t.Fatalf("err: %v", err) - } - case <-time.After(3 * time.Second): - t.Fatalf("timeout") - } -} - -func TestPexecDriver_Start_Wait(t *testing.T) { - task := &structs.Task{ - Name: "sleep", - Config: map[string]string{ - "command": "/bin/sleep", - "args": "1", - }, - } - - driverCtx := testDriverContext(task.Name) - ctx := testDriverExecContext(task, driverCtx) - defer ctx.AllocDir.Destroy() - - d := NewPrivilegedExecDriver(driverCtx) - handle, err := d.Start(ctx, task) - if err != nil { - t.Fatalf("err: %v", err) - } - if handle == nil { - t.Fatalf("missing handle") - } - - // Update should be a no-op - err = handle.Update(task) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Task should terminate quickly - select { - case err := <-handle.WaitCh(): - if err != nil { - t.Fatalf("err: %v", err) - } - case <-time.After(2 * time.Second): - t.Fatalf("timeout") - } -} - -func TestPexecDriver_Start_Wait_AllocDir(t *testing.T) { - exp := []byte{'w', 'i', 'n'} - file := "output.txt" - task := &structs.Task{ - Name: "sleep", - Config: map[string]string{ - "command": "/bin/bash", - "args": fmt.Sprintf(`-c "sleep 1; echo -n %s > $%s/%s"`, string(exp), environment.AllocDir, file), - }, - } - - driverCtx := testDriverContext(task.Name) - ctx := testDriverExecContext(task, driverCtx) - defer ctx.AllocDir.Destroy() - - d := NewPrivilegedExecDriver(driverCtx) - handle, err := d.Start(ctx, task) - if err != nil { - t.Fatalf("err: %v", err) - } - if handle == nil { - t.Fatalf("missing handle") - } - - // Task should terminate quickly - select { - case err := <-handle.WaitCh(): - if err != nil { - t.Fatalf("err: %v", err) - } - case <-time.After(2 * time.Second): - t.Fatalf("timeout") - } - - // Check that data was written to the shared alloc directory. - outputFile := filepath.Join(ctx.AllocDir.SharedDir, file) - act, err := ioutil.ReadFile(outputFile) - if err != nil { - t.Fatalf("Couldn't read expected output: %v", err) - } - - if !reflect.DeepEqual(act, exp) { - t.Fatalf("Command outputted %v; want %v", act, exp) - } -} - -func TestPexecDriver_Start_Kill_Wait(t *testing.T) { - task := &structs.Task{ - Name: "sleep", - Config: map[string]string{ - "command": "/bin/sleep", - "args": "1", - }, - } - - driverCtx := testDriverContext(task.Name) - ctx := testDriverExecContext(task, driverCtx) - defer ctx.AllocDir.Destroy() - - d := NewPrivilegedExecDriver(driverCtx) - handle, err := d.Start(ctx, task) - if err != nil { - t.Fatalf("err: %v", err) - } - if handle == nil { - t.Fatalf("missing handle") - } - - go func() { - time.Sleep(100 * time.Millisecond) - err := handle.Kill() - if err != nil { - t.Fatalf("err: %v", err) - } - }() - - // Task should terminate quickly - select { - case err := <-handle.WaitCh(): - if err == nil { - t.Fatal("should err") - } - case <-time.After(2 * time.Second): - t.Fatalf("timeout") - } -} From 60346ae8ec91d2192d617e8ecb7f2e3b6cdf4cbd Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 8 Oct 2015 11:36:22 -0700 Subject: [PATCH 3/5] Actually add the files --- client/driver/raw_exec.go | 197 +++++++++++++++++++++ client/driver/raw_exec_test.go | 216 +++++++++++++++++++++++ website/source/docs/drivers/exec.html.md | 2 +- 3 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 client/driver/raw_exec.go create mode 100644 client/driver/raw_exec_test.go diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go new file mode 100644 index 00000000000..ae8e6ab636b --- /dev/null +++ b/client/driver/raw_exec.go @@ -0,0 +1,197 @@ +package driver + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "github.com/hashicorp/nomad/client/allocdir" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/args" + "github.com/hashicorp/nomad/nomad/structs" +) + +const ( + // The option that enables this driver in the Config.Options map. + rawExecConfigOption = "driver.raw_exec.enable" + + // Null files to use as stdin. + unixNull = "/dev/null" + windowsNull = "nul" +) + +// The RawExecDriver is a privileged version of the exec driver. It provides no +// resource isolation and just fork/execs. The Exec driver should be preferred +// and this should only be used when explicitly needed. +type RawExecDriver struct { + DriverContext +} + +// rawExecHandle is returned from Start/Open as a handle to the PID +type rawExecHandle struct { + proc *os.Process + waitCh chan error + doneCh chan struct{} +} + +// NewRawExecDriver is used to create a new raw exec driver +func NewRawExecDriver(ctx *DriverContext) Driver { + return &RawExecDriver{*ctx} +} + +func (d *RawExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { + // Check that the user has explicitly enabled this executor. + enabled := strings.ToLower(cfg.ReadDefault(rawExecConfigOption, "false")) + if enabled == "1" || enabled == "true" { + d.logger.Printf("[WARN] driver.raw_exec: raw exec is enabled. Only enable if needed") + node.Attributes["driver.raw_exec"] = "1" + return true, nil + } + + return false, nil +} + +func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { + // Get the command + command, ok := task.Config["command"] + if !ok || command == "" { + return nil, fmt.Errorf("missing command for raw_exec driver") + } + + // Get the tasks local directory. + taskName := d.DriverContext.taskName + taskDir, ok := ctx.AllocDir.TaskDirs[taskName] + if !ok { + return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) + } + taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) + + // Get the environment variables. + envVars := TaskEnvironmentVariables(ctx, task) + + // Look for arguments + var cmdArgs []string + if argRaw, ok := task.Config["args"]; ok { + parsed, err := args.ParseAndReplace(argRaw, envVars.Map()) + if err != nil { + return nil, err + } + cmdArgs = append(cmdArgs, parsed...) + } + + // Setup the command + cmd := exec.Command(command, cmdArgs...) + cmd.Dir = taskDir + cmd.Env = envVars.List() + + // Capture the stdout/stderr and redirect stdin to /dev/null + stdoutFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stdout", taskName)) + stderrFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stderr", taskName)) + stdinFilename := unixNull + if runtime.GOOS == "windows" { + stdinFilename = windowsNull + } + + stdo, err := os.OpenFile(stdoutFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stdout: %v", err) + } + + stde, err := os.OpenFile(stderrFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stderr: %v", err) + } + + stdi, err := os.OpenFile(stdinFilename, os.O_CREATE|os.O_RDONLY, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stdin: %v", err) + } + + cmd.Stdout = stdo + cmd.Stderr = stde + cmd.Stdin = stdi + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start command: %v", err) + } + + // Return a driver handle + h := &rawExecHandle{ + proc: cmd.Process, + doneCh: make(chan struct{}), + waitCh: make(chan error, 1), + } + go h.run() + return h, nil +} + +func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { + // Split the handle + pidStr := strings.TrimPrefix(handleID, "PID:") + pid, err := strconv.Atoi(pidStr) + if err != nil { + return nil, fmt.Errorf("failed to parse handle '%s': %v", handleID, err) + } + + // Find the process + proc, err := os.FindProcess(pid) + if proc == nil || err != nil { + return nil, fmt.Errorf("failed to find PID %d: %v", pid, err) + } + + // Return a driver handle + h := &rawExecHandle{ + proc: proc, + doneCh: make(chan struct{}), + waitCh: make(chan error, 1), + } + go h.run() + return h, nil +} + +func (h *rawExecHandle) ID() string { + // Return a handle to the PID + return fmt.Sprintf("PID:%d", h.proc.Pid) +} + +func (h *rawExecHandle) WaitCh() chan error { + return h.waitCh +} + +func (h *rawExecHandle) Update(task *structs.Task) error { + // Update is not possible + return nil +} + +// Kill is used to terminate the task. We send an Interrupt +// and then provide a 5 second grace period before doing a Kill on supported +// OS's, otherwise we kill immediately. +func (h *rawExecHandle) Kill() error { + if runtime.GOOS == "windows" { + return h.proc.Kill() + } + + h.proc.Signal(os.Interrupt) + select { + case <-h.doneCh: + return nil + case <-time.After(5 * time.Second): + return h.proc.Kill() + } +} + +func (h *rawExecHandle) run() { + ps, err := h.proc.Wait() + close(h.doneCh) + if err != nil { + h.waitCh <- err + } else if !ps.Success() { + h.waitCh <- fmt.Errorf("task exited with error") + } + close(h.waitCh) +} diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go new file mode 100644 index 00000000000..8d06a7de8ca --- /dev/null +++ b/client/driver/raw_exec_test.go @@ -0,0 +1,216 @@ +package driver + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/environment" + "github.com/hashicorp/nomad/nomad/structs" +) + +func TestRawExecDriver_Fingerprint(t *testing.T) { + d := NewRawExecDriver(testDriverContext("")) + node := &structs.Node{ + Attributes: make(map[string]string), + } + + // Disable raw exec. + cfg := &config.Config{Options: map[string]string{rawExecConfigOption: "false"}} + + apply, err := d.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if apply { + t.Fatalf("should not apply") + } + if node.Attributes["driver.raw_exec"] != "" { + t.Fatalf("driver incorrectly enabled") + } + + // Enable raw exec. + cfg.Options[rawExecConfigOption] = "true" + apply, err = d.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if !apply { + t.Fatalf("should apply") + } + if node.Attributes["driver.raw_exec"] != "1" { + t.Fatalf("driver not enabled") + } +} + +func TestRawExecDriver_StartOpen_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "2", + }, + } + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewRawExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Attempt to open + handle2, err := d.Open(ctx, handle.ID()) + handle2.(*rawExecHandle).waitCh = make(chan error, 1) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle2 == nil { + t.Fatalf("missing handle") + } + + // Task should terminate quickly + select { + case err := <-handle2.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatalf("timeout") + } +} + +func TestRawExecDriver_Start_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "1", + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewRawExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Update should be a no-op + err = handle.Update(task) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } +} + +func TestRawExecDriver_Start_Wait_AllocDir(t *testing.T) { + exp := []byte{'w', 'i', 'n'} + file := "output.txt" + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/bash", + "args": fmt.Sprintf(`-c "sleep 1; echo -n %s > $%s/%s"`, string(exp), environment.AllocDir, file), + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewRawExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } + + // Check that data was written to the shared alloc directory. + outputFile := filepath.Join(ctx.AllocDir.SharedDir, file) + act, err := ioutil.ReadFile(outputFile) + if err != nil { + t.Fatalf("Couldn't read expected output: %v", err) + } + + if !reflect.DeepEqual(act, exp) { + t.Fatalf("Command outputted %v; want %v", act, exp) + } +} + +func TestRawExecDriver_Start_Kill_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "1", + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewRawExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + go func() { + time.Sleep(100 * time.Millisecond) + err := handle.Kill() + if err != nil { + t.Fatalf("err: %v", err) + } + }() + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err == nil { + t.Fatal("should err") + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } +} diff --git a/website/source/docs/drivers/exec.html.md b/website/source/docs/drivers/exec.html.md index e480f38bda4..dd30af74a9f 100644 --- a/website/source/docs/drivers/exec.html.md +++ b/website/source/docs/drivers/exec.html.md @@ -6,7 +6,7 @@ description: |- The Exec task driver is used to run binaries using OS isolation primitives. --- -# Fork/Exec Driver +# Isolated Fork/Exec Driver Name: `exec` From c4e48618894149032d42da3d79edbb2c428bdba7 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 8 Oct 2015 12:18:44 -0700 Subject: [PATCH 4/5] Documentation --- website/source/docs/agent/config.html.md | 4 +- website/source/docs/drivers/exec.html.md | 7 +-- website/source/docs/drivers/raw_exec.html.md | 47 ++++++++++++++++++++ website/source/layouts/docs.erb | 10 ++++- 4 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 website/source/docs/drivers/raw_exec.html.md diff --git a/website/source/docs/agent/config.html.md b/website/source/docs/agent/config.html.md index 56532f921e4..26151475263 100644 --- a/website/source/docs/agent/config.html.md +++ b/website/source/docs/agent/config.html.md @@ -207,8 +207,8 @@ configured on server nodes. option is not required and has no default. * `meta`: This is a key/value mapping of metadata pairs. This is a free-form map and can contain any string values. - * `options`: This is a key/value mapping of internal configuration for clients, - such as for driver configuration. + * `options`: This is a key/value mapping of internal + configuration for clients, such as for driver configuration. * `network_interface`: This is a string to force network fingerprinting to use a specific network interface * `network_speed`: This is an int that sets the diff --git a/website/source/docs/drivers/exec.html.md b/website/source/docs/drivers/exec.html.md index dd30af74a9f..1f3e50935c0 100644 --- a/website/source/docs/drivers/exec.html.md +++ b/website/source/docs/drivers/exec.html.md @@ -11,9 +11,10 @@ description: |- Name: `exec` The `exec` driver is used to simply execute a particular command for a task. -This is the simplest driver and is extremely flexible. In particlar, because -it can invoke any command, it can be used to call scripts or other wrappers -which provide higher level features. +However unlike [`raw_exec`](raw_exec.html) it uses the underlying isolation +primitives of the operating system to limit the tasks access to resources. While +simple, since the `exec` driver can invoke any command, it can be used to call +scripts or other wrappers which provide higher level features. ## Task Configuration diff --git a/website/source/docs/drivers/raw_exec.html.md b/website/source/docs/drivers/raw_exec.html.md new file mode 100644 index 00000000000..fdbcb956c4f --- /dev/null +++ b/website/source/docs/drivers/raw_exec.html.md @@ -0,0 +1,47 @@ +--- +layout: "docs" +page_title: "Drivers: Raw Exec" +sidebar_current: "docs-drivers-raw-exec" +description: |- + The Raw Exec task driver simply fork/execs and provides no isolation. +--- + +# Raw Fork/Exec Driver + +Name: `raw_exec` + +The `raw_exec` driver is used to execute a command for a task without any +resource isolation. As such, it should be used with extreme care and is disabled +by default. + +## Task Configuration + +The `raw_exec` driver supports the following configuration in the job spec: + +* `command` - The command to execute. Must be provided. + +* `args` - The argument list to the command, space seperated. Optional. + +## Client Requirements + +The `raw_exec` driver can run on all supported operating systems. It is however +disabled by default. In order to be enabled, the Nomad client configuration must +explicitly enable the `raw_exec` driver in the +[options](../agent/config.html#options) field: + +``` +options = { + driver.raw_exec.enable = "1" +} +``` + +## Client Attributes + +The `raw_exec` driver will set the following client attributes: + +* `driver.raw_exec` - This will be set to "1", indicating the + driver is available. + +## Resource Isolation + +The `raw_exec` driver provides no isolation. diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 62d1dddca90..e428cb593cf 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -49,7 +49,11 @@ > - Fork/Exec + Isolated Fork/Exec + + + > + Raw Fork/Exec > @@ -60,6 +64,10 @@ Qemu + > + Rkt + + > Custom From bbdceca358978b7c9b048a4e633b8d756a0b8aa2 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Fri, 9 Oct 2015 11:29:59 -0700 Subject: [PATCH 5/5] Better parsing of raw_exec option and updated docs --- client/driver/raw_exec.go | 8 ++++++-- website/source/docs/drivers/raw_exec.html.md | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index ae8e6ab636b..cdc41e676ad 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -46,8 +46,12 @@ func NewRawExecDriver(ctx *DriverContext) Driver { func (d *RawExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { // Check that the user has explicitly enabled this executor. - enabled := strings.ToLower(cfg.ReadDefault(rawExecConfigOption, "false")) - if enabled == "1" || enabled == "true" { + enabled, err := strconv.ParseBool(cfg.ReadDefault(rawExecConfigOption, "false")) + if err != nil { + return false, fmt.Errorf("Failed to parse %v option: %v", rawExecConfigOption, err) + } + + if enabled { d.logger.Printf("[WARN] driver.raw_exec: raw exec is enabled. Only enable if needed") node.Attributes["driver.raw_exec"] = "1" return true, nil diff --git a/website/source/docs/drivers/raw_exec.html.md b/website/source/docs/drivers/raw_exec.html.md index fdbcb956c4f..35b0c95bf76 100644 --- a/website/source/docs/drivers/raw_exec.html.md +++ b/website/source/docs/drivers/raw_exec.html.md @@ -11,8 +11,8 @@ description: |- Name: `raw_exec` The `raw_exec` driver is used to execute a command for a task without any -resource isolation. As such, it should be used with extreme care and is disabled -by default. +isolation. Further, the task is started as the same user as the Nomad process. +As such, it should be used with extreme care and is disabled by default. ## Task Configuration