From db88c38bc7239b42c51961fcf4ccf0cfbe9753ce Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 15 Oct 2015 16:40:07 -0700 Subject: [PATCH 1/5] Bind alloc dir and task local dir to docker containers and parse args correctly --- client/alloc_runner.go | 6 ++- client/driver/docker.go | 76 ++++++++++++++++++++++++------- client/driver/environment/vars.go | 8 ++++ client/fingerprint/host.go | 2 +- scripts/build.sh | 1 + 5 files changed, 74 insertions(+), 19 deletions(-) diff --git a/client/alloc_runner.go b/client/alloc_runner.go index f4613ad67cd..f41be4558ce 100644 --- a/client/alloc_runner.go +++ b/client/alloc_runner.go @@ -282,7 +282,11 @@ func (r *AllocRunner) Run() { // Create the execution context if r.ctx == nil { allocDir := allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID)) - allocDir.Build(tg.Tasks) + if err := allocDir.Build(tg.Tasks); err != nil { + r.logger.Printf("[WARN] client: failed to build task directories: %v", err) + r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("failed to build task dirs for '%s'", alloc.TaskGroup)) + return + } r.ctx = driver.NewExecContext(allocDir) } diff --git a/client/driver/docker.go b/client/driver/docker.go index 8ff6b93f820..1100061d435 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -4,12 +4,15 @@ import ( "encoding/json" "fmt" "log" + "path/filepath" "strconv" "strings" docker "github.com/fsouza/go-dockerclient" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/args" "github.com/hashicorp/nomad/nomad/structs" ) @@ -97,12 +100,38 @@ func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool return true, nil } -// We have to call this when we create the container AND when we start it so -// we'll make a function. -func createHostConfig(task *structs.Task) *docker.HostConfig { - // hostConfig holds options for the docker container that are unique to this - // machine, such as resource limits and port mappings - return &docker.HostConfig{ +func containerBinds(alloc *allocdir.AllocDir, task *structs.Task) ([]string, error) { + shared := alloc.SharedDir + local, ok := alloc.TaskDirs[task.Name] + if !ok { + fmt.Println("ALLOCDIR: ", alloc) + fmt.Println("TASK DIRS: ", alloc.TaskDirs) + for task, dir := range alloc.TaskDirs { + fmt.Printf("%v -> %v\n", task, dir) + } + return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) + } + + return []string{ + fmt.Sprintf("%s:%s", shared, allocdir.SharedAllocName), + fmt.Sprintf("%s:%s", local, allocdir.TaskLocal), + }, nil +} + +// createContainer initializes a struct needed to call docker.client.CreateContainer() +func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) (docker.CreateContainerOptions, error) { + var c docker.CreateContainerOptions + if task.Resources == nil { + logger.Printf("[ERR] driver.docker: task.Resources is empty") + return c, fmt.Errorf("task.Resources is nil and we can't constrain resource usage. We shouldn't have been able to schedule this in the first place.") + } + + binds, err := containerBinds(ctx.AllocDir, task) + if err != nil { + return c, err + } + + hostConfig := &docker.HostConfig{ // Convert MB to bytes. This is an absolute value. // // This value represents the total amount of memory a process can use. @@ -131,20 +160,16 @@ func createHostConfig(task *structs.Task) *docker.HostConfig { // - https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt // - https://www.kernel.org/doc/Documentation/scheduler/sched-design-CFS.txt CPUShares: int64(task.Resources.CPU), - } -} -// createContainer initializes a struct needed to call docker.client.CreateContainer() -func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) (docker.CreateContainerOptions, error) { - var c docker.CreateContainerOptions - if task.Resources == nil { - logger.Printf("[ERR] driver.docker: task.Resources is empty") - return c, fmt.Errorf("task.Resources is nil and we can't constrain resource usage. We shouldn't have been able to schedule this in the first place.") + // Binds are used to mount a host volume into the container. We mount a + // local directory for storage and a shared alloc directory that can be + // used to share data between different tasks in the same task group. + Binds: binds, } - hostConfig := createHostConfig(task) logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"]) logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"]) + logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"]) mode, ok := task.Config["network_mode"] if !ok || mode == "" { @@ -198,14 +223,31 @@ func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) ( hostConfig.PortBindings = dockerPorts } + // Create environment variables. + env := TaskEnvironmentVariables(ctx, task) + env.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)) + env.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)) + config := &docker.Config{ - Env: TaskEnvironmentVariables(ctx, task).List(), + Env: env.List(), Image: task.Config["image"], } + rawArgs, hasArgs := task.Config["args"] + parsedArgs, err := args.ParseAndReplace(rawArgs, env.Map()) + if err != nil { + return c, err + } + // If the user specified a custom command to run, we'll inject it here. if command, ok := task.Config["command"]; ok { - config.Cmd = strings.Split(command, " ") + cmd := []string{command} + if hasArgs { + cmd = append(cmd, parsedArgs...) + } + config.Cmd = cmd + } else if hasArgs { + logger.Println("[DEBUG] driver.docker: ignoring args because command not specified") } return docker.CreateContainerOptions{ diff --git a/client/driver/environment/vars.go b/client/driver/environment/vars.go index 8e2d698ed2a..587252c8787 100644 --- a/client/driver/environment/vars.go +++ b/client/driver/environment/vars.go @@ -12,6 +12,10 @@ const ( // group. AllocDir = "NOMAD_ALLOC_DIR" + // The path to the tasks local directory where it can store data that is + // persisted to the alloc is removed. + TaskLocalDir = "NOMAD_TASK_DIR" + // The tasks memory limit in MBs. MemLimit = "NOMAD_MEMORY_LIMIT" @@ -70,6 +74,10 @@ func (t TaskEnvironment) SetAllocDir(dir string) { t[AllocDir] = dir } +func (t TaskEnvironment) SetTaskLocalDir(dir string) { + t[TaskLocalDir] = dir +} + func (t TaskEnvironment) SetMemLimit(limit int) { t[MemLimit] = strconv.Itoa(limit) } diff --git a/client/fingerprint/host.go b/client/fingerprint/host.go index 6e5244bf992..ac7a347f2ba 100644 --- a/client/fingerprint/host.go +++ b/client/fingerprint/host.go @@ -5,7 +5,7 @@ import ( "log" "os/exec" "runtime" - "strings" + "strings" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" diff --git a/scripts/build.sh b/scripts/build.sh index 36566413a2d..4a79e8351e2 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -44,6 +44,7 @@ gox \ -arch="${XC_ARCH}" \ -osarch="!linux/arm !darwin/386" \ -ldflags "-X main.GitCommit ${GIT_COMMIT}${GIT_DIRTY}" \ + -cgo \ -output "pkg/{{.OS}}_{{.Arch}}/nomad" \ . From 5bf2ad02613a043ee775950aa9b788d3d1f4c34f Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 15 Oct 2015 16:59:08 -0700 Subject: [PATCH 2/5] Docker alloc dir tests and test fixes --- client/driver/docker.go | 2 - client/driver/docker_test.go | 78 +++++++++++++++++++++++++++++++++++- client/driver/exec_test.go | 4 +- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 1100061d435..449eb1ab753 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -104,8 +104,6 @@ func containerBinds(alloc *allocdir.AllocDir, task *structs.Task) ([]string, err shared := alloc.SharedDir local, ok := alloc.TaskDirs[task.Name] if !ok { - fmt.Println("ALLOCDIR: ", alloc) - fmt.Println("TASK DIRS: ", alloc.TaskDirs) for task, dir := range alloc.TaskDirs { fmt.Printf("%v -> %v\n", task, dir) } diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index 028738f3500..53052bd7e70 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -1,11 +1,16 @@ package driver import ( + "fmt" + "io/ioutil" "os/exec" + "path/filepath" + "reflect" "testing" "time" "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/environment" "github.com/hashicorp/nomad/nomad/structs" ) @@ -102,7 +107,8 @@ func TestDockerDriver_Start_Wait(t *testing.T) { Name: "redis-demo", Config: map[string]string{ "image": "redis", - "command": "redis-server -v", + "command": "redis-server", + "args": "-v", }, Resources: &structs.Resources{ MemoryMB: 256, @@ -140,6 +146,61 @@ func TestDockerDriver_Start_Wait(t *testing.T) { } } +func TestDockerDriver_Start_Wait_AllocDir(t *testing.T) { + if !dockerLocated() { + t.SkipNow() + } + + exp := []byte{'w', 'i', 'n'} + file := "output.txt" + task := &structs.Task{ + Name: "redis-demo", + Config: map[string]string{ + "image": "redis", + "command": "/bin/bash", + "args": fmt.Sprintf(`-c "sleep 1; echo -n %s > $%s/%s"`, string(exp), environment.AllocDir, file), + }, + Resources: &structs.Resources{ + MemoryMB: 256, + CPU: 512, + }, + } + + driverCtx := testDockerDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewDockerDriver(driverCtx) + + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + defer handle.Kill() + + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatalf("timeout") + } + + // Check that data was written to the shared alloc directory. + outputFile := filepath.Join(ctx.AllocDir.SharedDir, file) + act, err := ioutil.ReadFile(outputFile) + if err != nil { + t.Fatalf("Couldn't read expected output: %v", err) + } + + if !reflect.DeepEqual(act, exp) { + t.Fatalf("Command outputted %v; want %v", act, exp) + } +} + func TestDockerDriver_Start_Kill_Wait(t *testing.T) { if !dockerLocated() { t.SkipNow() @@ -149,7 +210,8 @@ func TestDockerDriver_Start_Kill_Wait(t *testing.T) { Name: "redis-demo", Config: map[string]string{ "image": "redis", - "command": "sleep 10", + "command": "/bin/sleep", + "args": "10", }, Resources: basicResources, } @@ -188,6 +250,7 @@ func TestDockerDriver_Start_Kill_Wait(t *testing.T) { func taskTemplate() *structs.Task { return &structs.Task{ + Name: "redis-demo", Config: map[string]string{ "image": "redis", }, @@ -242,6 +305,11 @@ func TestDocker_StartN(t *testing.T) { t.Log("==> All tasks are started. Terminating...") for idx, handle := range handles { + if handle == nil { + t.Errorf("Bad handle for task #%d", idx+1) + continue + } + err := handle.Kill() if err != nil { t.Errorf("Failed stopping task #%d: %s", idx+1, err) @@ -291,6 +359,11 @@ func TestDocker_StartNVersions(t *testing.T) { t.Log("==> All tasks are started. Terminating...") for idx, handle := range handles { + if handle == nil { + t.Errorf("Bad handle for task #%d", idx+1) + continue + } + err := handle.Kill() if err != nil { t.Errorf("Failed stopping task #%d: %s", idx+1, err) @@ -306,6 +379,7 @@ func TestDockerHostNet(t *testing.T) { } task := &structs.Task{ + Name: "redis-demo", Config: map[string]string{ "image": "redis", "network_mode": "host", diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go index feba89ae36d..d2c2886524e 100644 --- a/client/driver/exec_test.go +++ b/client/driver/exec_test.go @@ -85,7 +85,7 @@ func TestExecDriver_Start_Wait(t *testing.T) { Name: "sleep", Config: map[string]string{ "command": "/bin/sleep", - "args": "1", + "args": "2", }, Resources: basicResources, } @@ -115,7 +115,7 @@ func TestExecDriver_Start_Wait(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } - case <-time.After(2 * time.Second): + case <-time.After(4 * time.Second): t.Fatalf("timeout") } } From f66abc0ad7ef40400c8fc8952448d43e916567e2 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 15 Oct 2015 17:28:25 -0700 Subject: [PATCH 3/5] Documentation --- website/source/docs/drivers/docker.html.md | 3 +++ .../source/docs/jobspec/environment.html.md | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index bb6bdb9e62d..02ead445d28 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -23,6 +23,9 @@ The `docker` driver supports the following configuration in the job specificatio * `command` - (Optional) The command to run when starting the container. +* `args` - (Optional) Arguments to the optional `command`. If no `command` is + present, `args` are ignored. + * `network_mode` - (Optional) The network mode to be used for the container. Valid options are `default`, `bridge`, `host` or `none`. If nothing is specified, the container will start in `bridge` mode. The `container` diff --git a/website/source/docs/jobspec/environment.html.md b/website/source/docs/jobspec/environment.html.md index a9d865e7b30..a807598436b 100644 --- a/website/source/docs/jobspec/environment.html.md +++ b/website/source/docs/jobspec/environment.html.md @@ -56,6 +56,27 @@ exported as environment variables for consistency, e.g. `NOMAD_PORT_5000`. Please see the relevant driver documentation for details. +### Task Directories + +Nomad makes the following two directories available to tasks: + +* `alloc/`: This directory is shared across all tasks in a task group and can be + used to store data that needs to be used by multiple tasks, such as a log + shipper. +* `local/`: This directory is private to each task. It can be used to store + arbitrary data that shouldn't be shared by tasks in the task group. + +Both these directories are persisted until the allocation is removed, which +occurs hours after all the tasks in the task group enter terminal states. This +gives time to view the data produced by tasks. + +Depending on the driver and operating system being targeted, the directories are +made available in various ways. For example, on `docker` the directories are +binded to the container, while on `exec` on Linux the directories are mounted into the +chroot. Regardless of how the directories are made available, the path to the +directories can be read through the following environment variables: +`NOMAD_ALLOC_DIR` and `NOMAD_TASK_DIR`. + ## Meta The job specification also allows you to specify a `meta` block to supply arbitrary From 87ae6c6be6d63fa4cd281cf4aeed275d20231360 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 15 Oct 2015 17:30:40 -0700 Subject: [PATCH 4/5] Remove debug lines --- client/driver/docker.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 449eb1ab753..bff9c055cbd 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -104,9 +104,6 @@ func containerBinds(alloc *allocdir.AllocDir, task *structs.Task) ([]string, err shared := alloc.SharedDir local, ok := alloc.TaskDirs[task.Name] if !ok { - for task, dir := range alloc.TaskDirs { - fmt.Printf("%v -> %v\n", task, dir) - } return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) } From 70f4fc5b158e3a27ca2978f4890b5780f8b20dc6 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 15 Oct 2015 17:47:14 -0700 Subject: [PATCH 5/5] Change two helper functions to be methods --- client/driver/docker.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index bff9c055cbd..bbd52a9d8a2 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -100,7 +100,7 @@ func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool return true, nil } -func containerBinds(alloc *allocdir.AllocDir, task *structs.Task) ([]string, error) { +func (d *DockerDriver) containerBinds(alloc *allocdir.AllocDir, task *structs.Task) ([]string, error) { shared := alloc.SharedDir local, ok := alloc.TaskDirs[task.Name] if !ok { @@ -114,14 +114,14 @@ func containerBinds(alloc *allocdir.AllocDir, task *structs.Task) ([]string, err } // createContainer initializes a struct needed to call docker.client.CreateContainer() -func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) (docker.CreateContainerOptions, error) { +func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task) (docker.CreateContainerOptions, error) { var c docker.CreateContainerOptions if task.Resources == nil { - logger.Printf("[ERR] driver.docker: task.Resources is empty") + d.logger.Printf("[ERR] driver.docker: task.Resources is empty") return c, fmt.Errorf("task.Resources is nil and we can't constrain resource usage. We shouldn't have been able to schedule this in the first place.") } - binds, err := containerBinds(ctx.AllocDir, task) + binds, err := d.containerBinds(ctx.AllocDir, task) if err != nil { return c, err } @@ -162,23 +162,23 @@ func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) ( Binds: binds, } - logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"]) - logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"]) - logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"]) + d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"]) + d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"]) + d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"]) mode, ok := task.Config["network_mode"] if !ok || mode == "" { // docker default - logger.Printf("[WARN] driver.docker: no mode specified for networking, defaulting to bridge") + d.logger.Printf("[WARN] driver.docker: no mode specified for networking, defaulting to bridge") mode = "bridge" } // Ignore the container mode for now switch mode { case "default", "bridge", "none", "host": - logger.Printf("[DEBUG] driver.docker: using %s as network mode", mode) + d.logger.Printf("[DEBUG] driver.docker: using %s as network mode", mode) default: - logger.Printf("[ERR] driver.docker: invalid setting for network mode: %s", mode) + d.logger.Printf("[ERR] driver.docker: invalid setting for network mode: %s", mode) return c, fmt.Errorf("Invalid setting for network mode: %s", mode) } hostConfig.NetworkMode = mode @@ -186,7 +186,7 @@ func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) ( // Setup port mapping (equivalent to -p on docker CLI). Ports must already be // exposed in the container. if len(task.Resources.Networks) == 0 { - logger.Print("[WARN] driver.docker: No networks are available for port mapping") + d.logger.Print("[WARN] driver.docker: No networks are available for port mapping") } else { network := task.Resources.Networks[0] dockerPorts := map[docker.Port][]docker.PortBinding{} @@ -194,7 +194,7 @@ func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) ( for _, port := range network.ListStaticPorts() { dockerPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} dockerPorts[docker.Port(strconv.Itoa(port)+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} - logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)\n", network.IP, port, port) + d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)\n", network.IP, port, port) } for label, port := range network.MapDynamicPorts() { @@ -208,11 +208,11 @@ func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) ( if _, err := strconv.Atoi(label); err == nil { dockerPorts[docker.Port(label+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} dockerPorts[docker.Port(label+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} - logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %s (mapped)", network.IP, port, label) + d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %s (mapped)", network.IP, port, label) } else { dockerPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} dockerPorts[docker.Port(strconv.Itoa(port)+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} - logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d for label %s\n", network.IP, port, port, label) + d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d for label %s\n", network.IP, port, port, label) } } hostConfig.PortBindings = dockerPorts @@ -242,7 +242,7 @@ func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) ( } config.Cmd = cmd } else if hasArgs { - logger.Println("[DEBUG] driver.docker: ignoring args because command not specified") + d.logger.Println("[DEBUG] driver.docker: ignoring args because command not specified") } return docker.CreateContainerOptions{ @@ -322,7 +322,7 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle d.logger.Printf("[DEBUG] driver.docker: using image %s", dockerImage.ID) d.logger.Printf("[INFO] driver.docker: identified image %s as %s", image, dockerImage.ID) - config, err := createContainer(ctx, task, d.logger) + config, err := d.createContainer(ctx, task) if err != nil { d.logger.Printf("[ERR] driver.docker: %s", err) return nil, fmt.Errorf("Failed to create container config for image %s", image)