From 67391d639504dc57ea302132726f1f38206c1b09 Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Mon, 25 Jan 2021 15:03:12 +0100 Subject: [PATCH 01/14] add extra Docker labels --- drivers/docker/driver.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 36a917407a7..9a382d71de9 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -70,7 +70,10 @@ var ( ) const ( - dockerLabelAllocID = "com.hashicorp.nomad.alloc_id" + dockerLabelAllocID = "com.hashicorp.nomad.alloc_id" + dockerLabelJobName = "com.hashicorp.nomad.job_name" + dockerLabelTaskGroupName = "com.hashicorp.nomad.task_group_name" + dockerLabelTaskName = "com.hashicorp.nomad.task_name" ) type Driver struct { @@ -1115,6 +1118,10 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T labels[k] = v } labels[dockerLabelAllocID] = task.AllocID + labels[dockerLabelJobName] = task.JobName + labels[dockerLabelTaskGroupName] = task.TaskGroupName + labels[dockerLabelTaskName] = task.Name + config.Labels = labels logger.Debug("applied labels on the container", "labels", config.Labels) From b61ae091cccc188515bf9101d7836ab7f7b64a32 Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Mon, 25 Jan 2021 16:48:39 +0100 Subject: [PATCH 02/14] fix tests around Docker labels --- drivers/docker/driver_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index a3d9cda38bb..05ce029d98d 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -799,8 +799,8 @@ func TestDockerDriver_Labels(t *testing.T) { t.Fatalf("err: %v", err) } - // expect to see 1 additional standard labels - require.Equal(t, len(cfg.Labels)+1, len(container.Config.Labels)) + // expect to see 4 additional standard labels (allocID, jobName, TaskGroupName and TaskName) + require.Equal(t, len(cfg.Labels)+4, len(container.Config.Labels)) for k, v := range cfg.Labels { require.Equal(t, v, container.Config.Labels[k]) } @@ -1066,7 +1066,10 @@ func TestDockerDriver_CreateContainerConfig_Labels(t *testing.T) { // user provided labels "user_label": "user_value", // default labels - "com.hashicorp.nomad.alloc_id": task.AllocID, + "com.hashicorp.nomad.alloc_id": task.AllocID, + "com.hashicorp.nomad.job_name": task.JobName, + "com.hashicorp.nomad.task_group_name": task.TaskGroupName, + "com.hashicorp.nomad.task_name": task.Name, } require.Equal(t, expectedLabels, c.Config.Labels) From 45034b0f2cef83a7ee467af24c182b0f79685246 Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Wed, 24 Feb 2021 19:03:00 +0100 Subject: [PATCH 03/14] add extra arguments in task runner for node name&id and namespace --- client/allocrunner/taskrunner/task_runner.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go index 9b94e1c7539..f5dc3be415e 100644 --- a/client/allocrunner/taskrunner/task_runner.go +++ b/client/allocrunner/taskrunner/task_runner.go @@ -980,7 +980,11 @@ func (tr *TaskRunner) buildTaskConfig() *drivers.TaskConfig { ID: fmt.Sprintf("%s/%s/%s", alloc.ID, task.Name, invocationid), Name: task.Name, JobName: alloc.Job.Name, + JobID: alloc.Job.ID, TaskGroupName: alloc.TaskGroup, + Namespace: alloc.Namespace, + NodeName: alloc.NodeName, + NodeID: alloc.NodeID, Resources: &drivers.Resources{ NomadResources: taskResources, LinuxResources: &drivers.LinuxResources{ From 0b2f59faca87bd3e0b100cb305c60f0b441078f2 Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Wed, 24 Feb 2021 19:04:01 +0100 Subject: [PATCH 04/14] add extra labels via glob matching --- drivers/docker/driver.go | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 9a382d71de9..7459c619e1d 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -28,6 +28,7 @@ import ( "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" + "github.com/ryanuber/go-glob" ) var ( @@ -72,8 +73,12 @@ var ( const ( dockerLabelAllocID = "com.hashicorp.nomad.alloc_id" dockerLabelJobName = "com.hashicorp.nomad.job_name" + dockerLabelJobID = "com.hashicorp.nomad.job_id" dockerLabelTaskGroupName = "com.hashicorp.nomad.task_group_name" dockerLabelTaskName = "com.hashicorp.nomad.task_name" + dockerLabelNamespace = "com.hashicorp.nomad.namespace" + dockerLabelNodeName = "com.hashicorp.nomad.node_name" + dockerLabelNodeID = "com.hashicorp.nomad.node_id" ) type Driver struct { @@ -1117,10 +1122,33 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T for k, v := range driverConfig.Labels { labels[k] = v } + // main mandatory label labels[dockerLabelAllocID] = task.AllocID - labels[dockerLabelJobName] = task.JobName - labels[dockerLabelTaskGroupName] = task.TaskGroupName - labels[dockerLabelTaskName] = task.Name + + //optional labels, as configured in plugin configuration + for _, configurationExtraLabel := range d.config.ExtraLabels { + if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelJobName, ".")[3]) { + labels[dockerLabelJobName] = task.JobName + } + if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelJobID, ".")[3]) { + labels[dockerLabelJobID] = task.JobID + } + if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelTaskGroupName, ".")[3]) { + labels[dockerLabelTaskGroupName] = task.TaskGroupName + } + if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelTaskName, ".")[3]) { + labels[dockerLabelTaskName] = task.Name + } + if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelNamespace, ".")[3]) { + labels[dockerLabelNamespace] = task.Namespace + } + if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelNodeName, ".")[3]) { + labels[dockerLabelNodeName] = task.NodeName + } + if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelNodeID, ".")[3]) { + labels[dockerLabelNodeID] = task.NodeID + } + } config.Labels = labels logger.Debug("applied labels on the container", "labels", config.Labels) From 6242d17776499100f3dfc98ec00c88392145f6d1 Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Wed, 24 Feb 2021 19:04:27 +0100 Subject: [PATCH 05/14] add extra data in TaskConfig for extra Docker labels --- plugins/drivers/driver.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/drivers/driver.go b/plugins/drivers/driver.go index c2d040eedd5..8c586036e65 100644 --- a/plugins/drivers/driver.go +++ b/plugins/drivers/driver.go @@ -237,8 +237,12 @@ func (c *DNSConfig) Copy() *DNSConfig { type TaskConfig struct { ID string JobName string + JobID string TaskGroupName string Name string + Namespace string + NodeName string + NodeID string Env map[string]string DeviceEnv map[string]string Resources *Resources From 05b6b2681e91a0d269bec012c3e11f80fde5d0af Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Wed, 24 Feb 2021 19:06:58 +0100 Subject: [PATCH 06/14] start adding extra_labels configuration option to docker plugin --- drivers/docker/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/docker/config.go b/drivers/docker/config.go index a62607d3b31..01f7d1c34f0 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -612,6 +612,7 @@ type DriverConfig struct { DisableLogCollection bool `codec:"disable_log_collection"` PullActivityTimeout string `codec:"pull_activity_timeout"` pullActivityTimeoutDuration time.Duration `codec:"-"` + ExtraLabels []string `codec:"extra_labels"` AllowRuntimesList []string `codec:"allow_runtimes"` allowRuntimes map[string]struct{} `codec:"-"` From 5f03424590d4ab845cbdb6bfacb054050f9f3314 Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Tue, 2 Mar 2021 18:23:00 +0100 Subject: [PATCH 07/14] add config parsing for docker plugin extra_labels --- drivers/docker/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/docker/config.go b/drivers/docker/config.go index 01f7d1c34f0..9f956c90532 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -203,6 +203,9 @@ var ( "ca": hclspec.NewAttr("ca", "string", false), })), + // extra docker labels, globs supported + "extra_labels": hclspec.NewAttr("extra_labels", "list(string)", false), + // garbage collection options // default needed for both if the gc {...} block is not set and // if the default fields are missing From 325ff1d23d59845975bf442105615a92a2b55c4d Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Tue, 2 Mar 2021 18:46:13 +0100 Subject: [PATCH 08/14] 'fix' TestDockerDriver_Labels - there's only one label by default --- drivers/docker/driver_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index 05ce029d98d..7e4176e7e84 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -799,8 +799,8 @@ func TestDockerDriver_Labels(t *testing.T) { t.Fatalf("err: %v", err) } - // expect to see 4 additional standard labels (allocID, jobName, TaskGroupName and TaskName) - require.Equal(t, len(cfg.Labels)+4, len(container.Config.Labels)) + // expect to see 1 additional standard labels (allocID) + require.Equal(t, len(cfg.Labels)+1, len(container.Config.Labels)) for k, v := range cfg.Labels { require.Equal(t, v, container.Config.Labels[k]) } From 4e7c415c19c09bdf2802462044b31eeb041d517d Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Tue, 2 Mar 2021 18:46:30 +0100 Subject: [PATCH 09/14] add docs entry with extra_labels --- website/content/docs/drivers/docker.mdx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/website/content/docs/drivers/docker.mdx b/website/content/docs/drivers/docker.mdx index fbd803f7044..974c97a9f5e 100644 --- a/website/content/docs/drivers/docker.mdx +++ b/website/content/docs/drivers/docker.mdx @@ -774,6 +774,8 @@ plugin "docker" { ca = "/etc/nomad/nomad.cert" } + extra_labels = ["job_name", "job_id", "task_group_name", "task_name", "namespace", "node_name", "node_id"] + gc { image = true image_delay = "3m" @@ -866,6 +868,10 @@ plugin "docker" { capabilities and exclusively use host based log aggregation, you may consider this option to disable nomad log collection overhead. +- `extra_labels` - Extra labels to add to Docker containers. + Available options are `job_name`, `job_id`, `task_group_name`, `task_name`, + `namespace`, `node_name`, `node_id`. Globs are supported ( e.g. `task*`) + - `gc` stanza: - `image` - Defaults to `true`. Changing this to `false` will prevent Nomad From 414d29df037d6bb9aadd3c7eb329389304890226 Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Tue, 2 Mar 2021 18:56:52 +0100 Subject: [PATCH 10/14] fix TestDockerDriver_CreateContainerConfig_Labels to expect only one hashicorp default label --- drivers/docker/driver_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index 7e4176e7e84..d2bea70ccf3 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -1065,11 +1065,8 @@ func TestDockerDriver_CreateContainerConfig_Labels(t *testing.T) { expectedLabels := map[string]string{ // user provided labels "user_label": "user_value", - // default labels + // default label "com.hashicorp.nomad.alloc_id": task.AllocID, - "com.hashicorp.nomad.job_name": task.JobName, - "com.hashicorp.nomad.task_group_name": task.TaskGroupName, - "com.hashicorp.nomad.task_name": task.Name, } require.Equal(t, expectedLabels, c.Config.Labels) From 4ce99c8cb4579bd36653f1f6c526f8a81f77e088 Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Fri, 5 Mar 2021 15:30:52 +0100 Subject: [PATCH 11/14] Update website/content/docs/drivers/docker.mdx Co-authored-by: Tim Gross --- website/content/docs/drivers/docker.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/drivers/docker.mdx b/website/content/docs/drivers/docker.mdx index 974c97a9f5e..024715952ee 100644 --- a/website/content/docs/drivers/docker.mdx +++ b/website/content/docs/drivers/docker.mdx @@ -870,7 +870,7 @@ plugin "docker" { - `extra_labels` - Extra labels to add to Docker containers. Available options are `job_name`, `job_id`, `task_group_name`, `task_name`, - `namespace`, `node_name`, `node_id`. Globs are supported ( e.g. `task*`) + `namespace`, `node_name`, `node_id`. Globs are supported (e.g. `task*`) - `gc` stanza: From a2b486b268a960d243d7489445e358f2bc0349e3 Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Fri, 5 Mar 2021 15:32:24 +0100 Subject: [PATCH 12/14] fix extra whitespace --- drivers/docker/driver_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index d2bea70ccf3..f7e7972f85d 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -1066,7 +1066,7 @@ func TestDockerDriver_CreateContainerConfig_Labels(t *testing.T) { // user provided labels "user_label": "user_value", // default label - "com.hashicorp.nomad.alloc_id": task.AllocID, + "com.hashicorp.nomad.alloc_id": task.AllocID, } require.Equal(t, expectedLabels, c.Config.Labels) From a140e3badd8bb1a2f0e49860c9062a2983f278dc Mon Sep 17 00:00:00 2001 From: Adrian Todorov Date: Fri, 5 Mar 2021 17:03:09 +0100 Subject: [PATCH 13/14] add docker extra_labels test --- drivers/docker/driver_test.go | 37 +++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index f7e7972f85d..8f810580a35 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -806,6 +806,43 @@ func TestDockerDriver_Labels(t *testing.T) { } } +func TestDockerDriver_ExtraLabels(t *testing.T) { + if !tu.IsCI() { + t.Parallel() + } + testutil.DockerCompatible(t) + + task, cfg, ports := dockerTask(t) + defer freeport.Return(ports) + + require.NoError(t, task.EncodeConcreteDriverConfig(cfg)) + + dockerClientConfig := make(map[string]interface{}) + + dockerClientConfig["extra_labels"] = []string{"task*", "job_name"} + client, d, handle, cleanup := dockerSetup(t, task, dockerClientConfig) + defer cleanup() + require.NoError(t, d.WaitUntilStarted(task.ID, 5*time.Second)) + + container, err := client.InspectContainer(handle.containerID) + if err != nil { + t.Fatalf("err: %v", err) + } + + expectedLabels := map[string]string{ + "com.hashicorp.nomad.alloc_id": task.AllocID, + "com.hashicorp.nomad.task_name": task.Name, + "com.hashicorp.nomad.task_group_name": task.TaskGroupName, + "com.hashicorp.nomad.job_name": task.JobName, + } + + // expect to see 4 labels (allocID by default, task_name and task_group_name due to task*, and job_name) + require.Equal(t, 4, len(container.Config.Labels)) + for k, v := range expectedLabels { + require.Equal(t, v, container.Config.Labels[k]) + } +} + func TestDockerDriver_ForcePull(t *testing.T) { if !tu.IsCI() { t.Parallel() From 29d3460b6fa26bc988d3131ac8bc5668a4111b2b Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 8 Mar 2021 08:40:08 -0500 Subject: [PATCH 14/14] reduce string manipulation --- drivers/docker/driver.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 7459c619e1d..2fa3cc24db7 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -1127,25 +1127,25 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T //optional labels, as configured in plugin configuration for _, configurationExtraLabel := range d.config.ExtraLabels { - if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelJobName, ".")[3]) { + if glob.Glob(configurationExtraLabel, "job_name") { labels[dockerLabelJobName] = task.JobName } - if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelJobID, ".")[3]) { + if glob.Glob(configurationExtraLabel, "job_id") { labels[dockerLabelJobID] = task.JobID } - if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelTaskGroupName, ".")[3]) { + if glob.Glob(configurationExtraLabel, "task_group_name") { labels[dockerLabelTaskGroupName] = task.TaskGroupName } - if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelTaskName, ".")[3]) { + if glob.Glob(configurationExtraLabel, "task_name") { labels[dockerLabelTaskName] = task.Name } - if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelNamespace, ".")[3]) { + if glob.Glob(configurationExtraLabel, "namespace") { labels[dockerLabelNamespace] = task.Namespace } - if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelNodeName, ".")[3]) { + if glob.Glob(configurationExtraLabel, "node_name") { labels[dockerLabelNodeName] = task.NodeName } - if glob.Glob(configurationExtraLabel, strings.Split(dockerLabelNodeID, ".")[3]) { + if glob.Glob(configurationExtraLabel, "node_id") { labels[dockerLabelNodeID] = task.NodeID } }