From f59a6f7362174b4d7ad640bf2ded1cab2bb30267 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Tue, 16 Aug 2016 10:49:06 +0900 Subject: [PATCH 1/5] driver.rkt: support port mapping with net and port options --- client/driver/rkt.go | 81 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index a59b61c2c05..140a617fa81 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -11,6 +11,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" "syscall" "time" @@ -60,14 +61,18 @@ type RktDriver struct { } type RktDriverConfig struct { - ImageName string `mapstructure:"image"` - Command string `mapstructure:"command"` - Args []string `mapstructure:"args"` - TrustPrefix string `mapstructure:"trust_prefix"` - DNSServers []string `mapstructure:"dns_servers"` // DNS Server for containers - DNSSearchDomains []string `mapstructure:"dns_search_domains"` // DNS Search domains for containers - Debug bool `mapstructure:"debug"` // Enable debug option for rkt command - Volumes []string `mapstructure:"volumes"` // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container + ImageName string `mapstructure:"image"` + Command string `mapstructure:"command"` + Args []string `mapstructure:"args"` + TrustPrefix string `mapstructure:"trust_prefix"` + DNSServers []string `mapstructure:"dns_servers"` // DNS Server for containers + DNSSearchDomains []string `mapstructure:"dns_search_domains"` // DNS Search domains for containers + Net []string `mapstructure:"net"` // Networks for the containers + PortMapRaw []map[string]string `mapstructure:"port_map"` // + PortMap map[string]string `mapstructure:"-"` // A map of host port and the port name defined in the image manifest file + Volumes []string `mapstructure:"volumes"` // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container + + Debug bool `mapstructure:"debug"` // Enable debug option for rkt command } // rktHandle is returned from Start/Open as a handle to the PID @@ -122,6 +127,12 @@ func (d *RktDriver) Validate(config map[string]interface{}) error { "dns_search_domains": &fields.FieldSchema{ Type: fields.TypeArray, }, + "net": &fields.FieldSchema{ + Type: fields.TypeArray, + }, + "port_map": &fields.FieldSchema{ + Type: fields.TypeArray, + }, "debug": &fields.FieldSchema{ Type: fields.TypeBool, }, @@ -199,6 +210,8 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e return nil, err } + driverConfig.PortMap = mapMergeStrStr(driverConfig.PortMapRaw...) + // ACI image img := driverConfig.ImageName @@ -308,6 +321,58 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e cmdArgs = append(cmdArgs, fmt.Sprintf("--dns-search=%s", domain)) } + // set network + network := strings.Join(driverConfig.Net, ",") + if network != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("--net=%s", network)) + } + + // Setup port mapping and exposed ports + if len(task.Resources.Networks) == 0 { + d.logger.Println("[DEBUG] driver.rkt: No network interfaces are available") + if len(driverConfig.PortMap) > 0 { + return nil, fmt.Errorf("Trying to map ports but no network interface is available") + } + } else { + // TODO add support for more than one network + network := task.Resources.Networks[0] + for _, port := range network.ReservedPorts { + var containerPort string + + if mapped, ok := driverConfig.PortMap[port.Label]; ok { + containerPort = mapped + } else { + // If the user doesn't have mapped a port using port_map, driver stops running container. + return nil, fmt.Errorf("port_map is not set. When you defined port in the resources, you need to configure port_map.") + } + + hostPortStr := strconv.Itoa(port.Value) + + d.logger.Printf("[DEBUG] driver.rkt: exposed port %s", containerPort) + // Add port option to rkt run arguments. rkt allows multiple port args + cmdArgs = append(cmdArgs, fmt.Sprintf("--port=%s:%s", containerPort, hostPortStr)) + } + + for _, port := range network.DynamicPorts { + // By default we will map the allocated port 1:1 to the container + var containerPort string + + if mapped, ok := driverConfig.PortMap[port.Label]; ok { + containerPort = mapped + } else { + // If the user doesn't have mapped a port using port_map, driver stops running container. + return nil, fmt.Errorf("port_map is not set. When you defined port in the resources, you need to configure port_map.") + } + + hostPortStr := strconv.Itoa(port.Value) + + d.logger.Printf("[DEBUG] driver.rkt: exposed port %s", containerPort) + // Add port option to rkt run arguments. rkt allows multiple port args + cmdArgs = append(cmdArgs, fmt.Sprintf("--port=%s:%s", containerPort, hostPortStr)) + } + + } + // Add user passed arguments. if len(driverConfig.Args) != 0 { parsed := d.taskEnv.ParseAndReplace(driverConfig.Args) From 82c1e7914e40bfe3efc530f7eb593947200565f3 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Fri, 19 Aug 2016 00:07:11 +0900 Subject: [PATCH 2/5] driver.rkt: add rkt drriver port mapping test --- client/driver/rkt_test.go | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index 36a494c83f9..b3a324b0022 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -397,3 +397,49 @@ func TestRktTaskValidate(t *testing.T) { t.Fatalf("Validation error in TaskConfig : '%v'", err) } } + +// TODO: Port Mapping test should be ran with proper ACI image and test the port access. +func TestRktDriver_PortsMapping(t *testing.T) { + if os.Getenv("NOMAD_TEST_RKT") == "" { + t.Skip("skipping rkt tests") + } + + ctestutils.RktCompatible(t) + task := &structs.Task{ + Name: "etcd", + Config: map[string]interface{}{ + "trust_prefix": "coreos.com/etcd", + "image": "coreos.com/etcd:v2.0.4", + "command": "/etcd", + "args": []string{"--version"}, + "port_map": []map[string]string{ + map[string]string{ + "main": "8080-tcp", + }, + }, + }, + } + + driverCtx, execCtx := testDriverContexts(task) + defer execCtx.AllocDir.Destroy() + + d := NewRktDriver(driverCtx) + + handle, err := d.Start(execCtx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + defer handle.Kill() + + // Attempt to open + handle2, err := d.Open(execCtx, handle.ID()) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle2 == nil { + t.Fatalf("missing handle") + } +} From a2397f09a49f4b6209e9f06067efffdedf1891c3 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Fri, 19 Aug 2016 01:00:21 +0900 Subject: [PATCH 3/5] driver.rkt: add documentation about Networking --- website/source/docs/drivers/rkt.html.md | 57 +++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/website/source/docs/drivers/rkt.html.md b/website/source/docs/drivers/rkt.html.md index 2056c76bcfa..34a916b9ea9 100644 --- a/website/source/docs/drivers/rkt.html.md +++ b/website/source/docs/drivers/rkt.html.md @@ -73,6 +73,19 @@ The `rkt` driver supports the following configuration in the job spec: * `dns_search_domains` - (Optional) A list of DNS search domains to be used in the containers. +* `net` - (Optional) A list of networks to be used by the containers + +* `port_map` - (Optional) A key/value map of port to be used by the container. + port name in the image manifest file needs to be specified for the value. For example: + + ``` + port_map { + app = "8080-tcp" + } + ``` + + See below for more details. + * `debug` - (Optional) Enable rkt command debug option. * `volumes` - (Optional) A list of `host_path:container_path` strings to bind @@ -84,6 +97,50 @@ The `rkt` driver supports the following configuration in the job spec: } ``` +## Networking + +The `rkt` can specify `--net` and `--port` for the rkt client. Hence, there are two ways to use host ports by +using `--net=host` or `--port=PORT` with your network. + +Example: + +``` + task "etcd" { + # Use Docker to run the task. + driver = "rkt" + + config { + image = "docker://my_image" + net = ["containers"] + port_map { + app = "8080-tcp" + } + } + + service { + port = "app" + } + + resources { + network { + mbits = 10 + port "app" { + static = 12345 + } + } + } + } + +``` + +### Allocating Ports + +You can allocate ports to your task using the port syntax described on the +[networking page](/docs/jobspec/networking.html). + +When you use port allocation, the image manifest needs to declare public ports and host has configured network. +For more information, please refer to [rkt Networking](https://coreos.com/rkt/docs/latest/networking/overview.html). + ## Client Requirements The `rkt` driver requires rkt to be installed and in your system's `$PATH`. From 294b7b306d84ca0727b27ef93f804ceebfb50082 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 25 Oct 2016 16:27:35 -0700 Subject: [PATCH 4/5] Fix tests for rkt port map --- client/driver/rkt.go | 8 ++++---- client/driver/rkt_test.go | 37 +++++++++++++++++++++++++------------ 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 140a617fa81..8c8c0666280 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -339,12 +339,12 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e for _, port := range network.ReservedPorts { var containerPort string - if mapped, ok := driverConfig.PortMap[port.Label]; ok { - containerPort = mapped - } else { - // If the user doesn't have mapped a port using port_map, driver stops running container. + mapped, ok := driverConfig.PortMap[port.Label] + if !ok { + // If the user doesn't have a mapped port using port_map, driver stops running container. return nil, fmt.Errorf("port_map is not set. When you defined port in the resources, you need to configure port_map.") } + containerPort = mapped hostPortStr := strconv.Itoa(port.Value) diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go index b3a324b0022..ffdbdc1e42a 100644 --- a/client/driver/rkt_test.go +++ b/client/driver/rkt_test.go @@ -408,13 +408,26 @@ func TestRktDriver_PortsMapping(t *testing.T) { task := &structs.Task{ Name: "etcd", Config: map[string]interface{}{ - "trust_prefix": "coreos.com/etcd", - "image": "coreos.com/etcd:v2.0.4", - "command": "/etcd", - "args": []string{"--version"}, + "image": "docker://redis:latest", + "args": []string{"--version"}, "port_map": []map[string]string{ map[string]string{ - "main": "8080-tcp", + "main": "6379-tcp", + }, + }, + "debug": "true", + }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, + Resources: &structs.Resources{ + MemoryMB: 256, + CPU: 512, + Networks: []*structs.NetworkResource{ + &structs.NetworkResource{ + IP: "127.0.0.1", + ReservedPorts: []structs.Port{{"main", 8080}}, }, }, }, @@ -434,12 +447,12 @@ func TestRktDriver_PortsMapping(t *testing.T) { } defer handle.Kill() - // Attempt to open - handle2, err := d.Open(execCtx, handle.ID()) - if err != nil { - t.Fatalf("err: %v", err) - } - if handle2 == nil { - t.Fatalf("missing handle") + select { + case res := <-handle.WaitCh(): + if !res.Successful() { + t.Fatalf("err: %v", res) + } + case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second): + t.Fatalf("timeout") } } From edac5e322dddf1594cc35759a4c6abb4c9dc0beb Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 25 Oct 2016 17:11:01 -0700 Subject: [PATCH 5/5] Use rkt port example that Just Works --- website/source/docs/drivers/rkt.html.md | 43 ++++++++++++------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/website/source/docs/drivers/rkt.html.md b/website/source/docs/drivers/rkt.html.md index 34a916b9ea9..48630572f22 100644 --- a/website/source/docs/drivers/rkt.html.md +++ b/website/source/docs/drivers/rkt.html.md @@ -105,32 +105,31 @@ using `--net=host` or `--port=PORT` with your network. Example: ``` - task "etcd" { - # Use Docker to run the task. - driver = "rkt" - - config { - image = "docker://my_image" - net = ["containers"] - port_map { - app = "8080-tcp" - } - } +task "redis" { + # Use rkt to run the task. + driver = "rkt" + + config { + # Use docker image with port defined + image = "docker://redis:latest" + port_map { + app = "6379-tcp" + } + } - service { - port = "app" - } + service { + port = "app" + } - resources { - network { - mbits = 10 - port "app" { - static = 12345 - } - } + resources { + network { + mbits = 10 + port "app" { + static = 12345 } } - + } +} ``` ### Allocating Ports