From d6be784e2d090861868d9603e6716df26b8e5f0d Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Fri, 16 Aug 2024 15:47:19 +0200 Subject: [PATCH] namespaces: add allowed network modes to capabilities. (#23813) --- .changelog/23813.txt | 3 + api/namespace.go | 6 +- command/namespace_status.go | 14 ++- nomad/job_endpoint_validators.go | 46 +++++++ nomad/job_endpoint_validators_test.go | 119 +++++++++++++++++- nomad/structs/structs.go | 14 ++- .../content/docs/commands/namespace/apply.mdx | 6 +- .../docs/commands/namespace/status.mdx | 12 +- .../docs/other-specifications/namespace.mdx | 12 +- 9 files changed, 216 insertions(+), 16 deletions(-) create mode 100644 .changelog/23813.txt diff --git a/.changelog/23813.txt b/.changelog/23813.txt new file mode 100644 index 00000000000..bd483aeed76 --- /dev/null +++ b/.changelog/23813.txt @@ -0,0 +1,3 @@ +```release-note:improvement +namespaces: Allow enabling/disabling allowed network modes per namespace +``` diff --git a/api/namespace.go b/api/namespace.go index bc12ec77d86..6cde45346d5 100644 --- a/api/namespace.go +++ b/api/namespace.go @@ -85,8 +85,10 @@ type Namespace struct { // NamespaceCapabilities represents a set of capabilities allowed for this // namespace, to be checked at job submission time. type NamespaceCapabilities struct { - EnabledTaskDrivers []string `hcl:"enabled_task_drivers"` - DisabledTaskDrivers []string `hcl:"disabled_task_drivers"` + EnabledTaskDrivers []string `hcl:"enabled_task_drivers"` + DisabledTaskDrivers []string `hcl:"disabled_task_drivers"` + EnabledNetworkModes []string `hcl:"enabled_network_modes"` + DisabledNetworkModes []string `hcl:"disabled_network_modes"` } // NamespaceNodePoolConfiguration stores configuration about node pools for a diff --git a/command/namespace_status.go b/command/namespace_status.go index a2706d62029..003d6bf6f6e 100644 --- a/command/namespace_status.go +++ b/command/namespace_status.go @@ -201,6 +201,8 @@ func (c *NamespaceStatusCommand) Run(args []string) int { func formatNamespaceBasics(ns *api.Namespace) string { enabled_drivers := "*" disabled_drivers := "" + enabled_network_modes := "*" + disabled_network_modes := "" if ns.Capabilities != nil { if len(ns.Capabilities.EnabledTaskDrivers) != 0 { enabled_drivers = strings.Join(ns.Capabilities.EnabledTaskDrivers, ",") @@ -208,13 +210,21 @@ func formatNamespaceBasics(ns *api.Namespace) string { if len(ns.Capabilities.DisabledTaskDrivers) != 0 { disabled_drivers = strings.Join(ns.Capabilities.DisabledTaskDrivers, ",") } + if len(ns.Capabilities.EnabledNetworkModes) != 0 { + enabled_network_modes = strings.Join(ns.Capabilities.EnabledNetworkModes, ",") + } + if len(ns.Capabilities.DisabledNetworkModes) != 0 { + disabled_network_modes = strings.Join(ns.Capabilities.DisabledNetworkModes, ",") + } } basic := []string{ fmt.Sprintf("Name|%s", ns.Name), fmt.Sprintf("Description|%s", ns.Description), fmt.Sprintf("Quota|%s", ns.Quota), - fmt.Sprintf("EnabledDrivers|%s", enabled_drivers), - fmt.Sprintf("DisabledDrivers|%s", disabled_drivers), + fmt.Sprintf("Enabled Drivers|%s", enabled_drivers), + fmt.Sprintf("Disabled Drivers|%s", disabled_drivers), + fmt.Sprintf("Enabled Network Modes|%s", enabled_network_modes), + fmt.Sprintf("Disabled Network Modes|%s", disabled_network_modes), } return formatKV(basic) diff --git a/nomad/job_endpoint_validators.go b/nomad/job_endpoint_validators.go index 1285ea2bfaf..b4f6c292bf0 100644 --- a/nomad/job_endpoint_validators.go +++ b/nomad/job_endpoint_validators.go @@ -47,9 +47,55 @@ func (c jobNamespaceConstraintCheckHook) Validate(job *structs.Job) (warnings [] ) } } + + var disallowedNetworkModes []string + for _, tg := range job.TaskGroups { + for _, network := range tg.Networks { + if allowed, network_mode := taskValidateNetworkMode(network, ns); !allowed { + disallowedNetworkModes = append(disallowedNetworkModes, network_mode) + } + } + } + if len(disallowedNetworkModes) > 0 { + if len(disallowedNetworkModes) == 1 { + return nil, fmt.Errorf( + "used group network mode %q is not allowed in namespace %q", disallowedNetworkModes[0], ns.Name, + ) + + } else { + return nil, fmt.Errorf( + "used group network modes %q are not allowed in namespace %q", disallowedNetworkModes, ns.Name, + ) + } + } + return nil, nil } +func taskValidateNetworkMode(network *structs.NetworkResource, ns *structs.Namespace) (bool, string) { + network_mode := "host" + if len(network.Mode) > 0 { + network_mode = network.Mode + } + if ns.Capabilities == nil { + return true, network_mode + } + allow := len(ns.Capabilities.EnabledNetworkModes) == 0 + for _, m := range ns.Capabilities.EnabledNetworkModes { + if network_mode == m { + allow = true + break + } + } + for _, m := range ns.Capabilities.DisabledNetworkModes { + if network_mode == m { + allow = false + break + } + } + return allow, network_mode +} + func taskValidateDriver(task *structs.Task, ns *structs.Namespace) bool { if ns.Capabilities == nil { return true diff --git a/nomad/job_endpoint_validators_test.go b/nomad/job_endpoint_validators_test.go index 500dbcaac9e..a0b9b0b0dee 100644 --- a/nomad/job_endpoint_validators_test.go +++ b/nomad/job_endpoint_validators_test.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test/must" "github.com/stretchr/testify/require" ) @@ -88,7 +89,7 @@ func TestJobNamespaceConstraintCheckHook_taskValidateDriver(t *testing.T) { } } -func TestJobNamespaceConstraintCheckHook_validate(t *testing.T) { +func TestJobNamespaceConstraintCheckHook_validate_drivers(t *testing.T) { ci.Parallel(t) s1, cleanupS1 := TestServer(t, nil) defer cleanupS1() @@ -120,3 +121,119 @@ func TestJobNamespaceConstraintCheckHook_validate(t *testing.T) { _, err = hook.Validate(job) require.Equal(t, err.Error(), "used task drivers [\"exec\" \"raw_exec\"] are not allowed in namespace \"default\"") } + +func TestJobNamespaceConstraintCheckHook_taskValidateNetworkMode(t *testing.T) { + ci.Parallel(t) + + cases := []struct { + description string + mode string + ns *structs.Namespace + result bool + }{ + { + "No capabilities set, allow all", + "bridge", + &structs.Namespace{}, + true, + }, + { + "No drivers enabled/disabled, allow all", + "bridge", + &structs.Namespace{Capabilities: &structs.NamespaceCapabilities{}}, + true, + }, + { + "No mode set and only host allowed", + "", + &structs.Namespace{ + Capabilities: &structs.NamespaceCapabilities{ + EnabledNetworkModes: []string{"host"}}, + }, + true, + }, + { + "Only bridge and cni/custom are allowed 1/2", + "bridge", + &structs.Namespace{ + Capabilities: &structs.NamespaceCapabilities{ + EnabledNetworkModes: []string{"bridge", "cni/custom"}}, + }, + true, + }, + { + "Only bridge and cni/custom are allowed 2/2", + "host", + &structs.Namespace{ + Capabilities: &structs.NamespaceCapabilities{ + EnabledNetworkModes: []string{"bridge", "cni/custom"}}, + }, + false, + }, + { + "disable takes precedence over enable", + "bridge", + &structs.Namespace{ + Capabilities: &structs.NamespaceCapabilities{ + EnabledNetworkModes: []string{"bridge"}, + DisabledNetworkModes: []string{"bridge"}}, + }, + false, + }, + { + "All modes but host are allowed 1/2", + "host", + &structs.Namespace{ + Capabilities: &structs.NamespaceCapabilities{ + DisabledNetworkModes: []string{"host"}}, + }, + false, + }, + { + "All modes but host are allowed 2/2", + "bridge", + &structs.Namespace{ + Capabilities: &structs.NamespaceCapabilities{ + DisabledNetworkModes: []string{"host"}}, + }, + true, + }, + } + + for _, c := range cases { + var network = &structs.NetworkResource{Mode: c.mode} + allowed, _ := taskValidateNetworkMode(network, c.ns) + must.Eq(t, c.result, allowed, must.Sprint(c.description)) + } +} + +func TestJobNamespaceConstraintCheckHook_validate_network_modes(t *testing.T) { + ci.Parallel(t) + s1, cleanupS1 := TestServer(t, nil) + defer cleanupS1() + testutil.WaitForLeader(t, s1.RPC) + + // Create a namespace + ns := mock.Namespace() + ns.Name = "default" // fix the name + ns.Capabilities = &structs.NamespaceCapabilities{ + EnabledNetworkModes: []string{"bridge", "cni/allowed"}, + DisabledNetworkModes: []string{"host", "cni/forbidden"}, + } + must.NoError(t, s1.fsm.State().UpsertNamespaces(1000, []*structs.Namespace{ns})) + + hook := jobNamespaceConstraintCheckHook{srv: s1} + job := mock.LifecycleJob() + job.TaskGroups[0].Networks = append(job.TaskGroups[0].Networks, &structs.NetworkResource{}) + _, err := hook.Validate(job) + must.EqError(t, err, "used group network mode \"host\" is not allowed in namespace \"default\"") + + job.TaskGroups[0].Networks[0].Mode = "bridge" + _, err = hook.Validate(job) + must.NoError(t, err) + + job.TaskGroups[0].Networks[0].Mode = "host" + job.TaskGroups[0].Networks = append(job.TaskGroups[0].Networks, &structs.NetworkResource{Mode: "cni/forbidden"}) + _, err = hook.Validate(job) + must.EqError(t, err, "used group network modes [\"host\" \"cni/forbidden\"] are not allowed in namespace \"default\"") +} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index aad6b42a16b..18a5de39c4f 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -5479,8 +5479,10 @@ type Namespace struct { // NamespaceCapabilities represents a set of capabilities allowed for this // namespace, to be checked at job submission time. type NamespaceCapabilities struct { - EnabledTaskDrivers []string - DisabledTaskDrivers []string + EnabledTaskDrivers []string + DisabledTaskDrivers []string + EnabledNetworkModes []string + DisabledNetworkModes []string } // NamespaceNodePoolConfiguration stores configuration about node pools for a @@ -5570,6 +5572,12 @@ func (n *Namespace) SetHash() []byte { for _, driver := range n.Capabilities.DisabledTaskDrivers { _, _ = hash.Write([]byte(driver)) } + for _, mode := range n.Capabilities.EnabledNetworkModes { + _, _ = hash.Write([]byte(mode)) + } + for _, mode := range n.Capabilities.DisabledNetworkModes { + _, _ = hash.Write([]byte(mode)) + } } if n.NodePoolConfiguration != nil { _, _ = hash.Write([]byte(n.NodePoolConfiguration.Default)) @@ -5630,6 +5638,8 @@ func (n *Namespace) Copy() *Namespace { *c = *n.Capabilities c.EnabledTaskDrivers = slices.Clone(n.Capabilities.EnabledTaskDrivers) c.DisabledTaskDrivers = slices.Clone(n.Capabilities.DisabledTaskDrivers) + c.EnabledNetworkModes = slices.Clone(n.Capabilities.EnabledNetworkModes) + c.DisabledNetworkModes = slices.Clone(n.Capabilities.DisabledNetworkModes) nc.Capabilities = c } if n.NodePoolConfiguration != nil { diff --git a/website/content/docs/commands/namespace/apply.mdx b/website/content/docs/commands/namespace/apply.mdx index c1ac6e347fe..d3f37d14629 100644 --- a/website/content/docs/commands/namespace/apply.mdx +++ b/website/content/docs/commands/namespace/apply.mdx @@ -66,8 +66,10 @@ name = "dev" description = "Namespace for developers" capabilities { - enabled_task_drivers = ["docker", "exec"] - disabled_task_drivers = ["raw_exec"] + enabled_task_drivers = ["docker", "exec"] + disabled_task_drivers = ["raw_exec"] + enabled_network_modes = ["bridge", "cni/custom"] + disabled_network_modes = ["host"] } meta { diff --git a/website/content/docs/commands/namespace/status.mdx b/website/content/docs/commands/namespace/status.mdx index 1a6ea780e8a..a9f28d5369f 100644 --- a/website/content/docs/commands/namespace/status.mdx +++ b/website/content/docs/commands/namespace/status.mdx @@ -41,11 +41,13 @@ View the status of a namespace: ```shell-session $ nomad namespace status default -Name = api-prod -Description = Prod API servers -Quota = prod -EnabledDrivers = docker,exec -DisabledDrivers = raw_exec +Name = api-prod +Description = Prod API servers +Quota = prod +EnabledDrivers = docker,exec +DisabledDrivers = raw_exec +EnabledNetworkModes = bridge,cni/custom +DisabledNetworkModes = host Metadata contact = platform-eng@example.com diff --git a/website/content/docs/other-specifications/namespace.mdx b/website/content/docs/other-specifications/namespace.mdx index 1d3d5d7b7de..20e711ea6c8 100644 --- a/website/content/docs/other-specifications/namespace.mdx +++ b/website/content/docs/other-specifications/namespace.mdx @@ -37,8 +37,10 @@ meta { } capabilities { - enabled_task_drivers = ["java", "docker"] - disabled_task_drivers = ["raw_exec"] + enabled_task_drivers = ["java", "docker"] + disabled_task_drivers = ["raw_exec"] + enabled_network_modes = ["bridge", "cni/custom"] + disabled_network_modes = ["host"] } # Node Pool configuration is a Nomad Enterprise feature. @@ -98,6 +100,12 @@ consul { - `disabled_task_drivers` `(array: [])` - List of task drivers disabled in the namespace. +- `enabled_network_modes` `(array: [])` - List of network modes allowed + in the namespace. If empty all network modes are allowed. + +- `disabled_network_modes` `(array: [])` - List of network modes disabled + in the namespace. + ### `node_pool_config` Parameters - `default` `(string: "default")` - Specifies the node pool to use for jobs in