cni: allow users to set CNI args in job spec (#23538)

.changelog/23538.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cni: allow users to input CNI args in job specification
+```

api/resources.go

@@ -143,6 +143,9 @@ type DNSConfig struct {
 	Searches []string `mapstructure:"searches" hcl:"searches,optional"`
 	Options  []string `mapstructure:"options" hcl:"options,optional"`
 }
+type CNIConfig struct {
+	Args map[string]string `hcl:"args,optional"`
+}
 
 // NetworkResource is used to describe required network
 // resources of a given task.
@@ -160,7 +163,8 @@ type NetworkResource struct {
 	// XXX Deprecated. Please do not use. The field will be removed in Nomad
 	// 0.13 and is only being kept to allow any references to be removed before
 	// then.
-	MBits *int `hcl:"mbits,optional"`
+	MBits *int       `hcl:"mbits,optional"`
+	CNI   *CNIConfig `hcl:"cni,block"`
 }
 
 // COMPAT(0.13)

client/allocrunner/networking_cni.go

@@ -102,6 +102,18 @@ const (
 	ConsulIPTablesConfigEnvVar = "CONSUL_IPTABLES_CONFIG"
 )
 
+// Adds user inputted custom CNI args to cniArgs map
+func addCustomCNIArgs(networks []*structs.NetworkResource, cniArgs map[string]string) {
+	for _, net := range networks {
+		if net.CNI == nil {
+			continue
+		}
+		for k, v := range net.CNI.Args {
+			cniArgs[k] = v
+		}
+	}
+}
+
 // Setup calls the CNI plugins with the add action
 func (c *cniNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) (*structs.AllocNetworkStatus, error) {
 	if err := c.ensureCNIInitialized(); err != nil {
@@ -114,6 +126,10 @@ func (c *cniNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Alloc
 		"IgnoreUnknown": "true",
 	}
 
+	tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
+
+	addCustomCNIArgs(tg.Networks, cniArgs)
+
 	portMaps := getPortMapping(alloc, c.ignorePortMappingHostIP)
 
 	tproxyArgs, err := c.setupTransparentProxyArgs(alloc, spec, portMaps)

client/allocrunner/networking_cni_test.go

@@ -205,6 +205,53 @@ func TestCNI_cniToAllocNet_Invalid(t *testing.T) {
 	require.Nil(t, allocNet)
 }
 
+func TestCNI_addCustomCNIArgs(t *testing.T) {
+	ci.Parallel(t)
+	cniArgs := map[string]string{
+		"default": "yup",
+	}
+
+	networkWithArgs := []*structs.NetworkResource{{
+		CNI: &structs.CNIConfig{
+			Args: map[string]string{
+				"first_arg": "example",
+				"new_arg":   "example_2",
+			},
+		},
+	}}
+	networkWithoutArgs := []*structs.NetworkResource{{
+		Mode: "bridge",
+	}}
+	testCases := []struct {
+		name      string
+		network   []*structs.NetworkResource
+		expectMap map[string]string
+	}{
+		{
+			name:    "cni args not specified",
+			network: networkWithoutArgs,
+			expectMap: map[string]string{
+				"default": "yup",
+			},
+		}, {
+			name:    "cni args specified",
+			network: networkWithArgs,
+			expectMap: map[string]string{
+				"default":   "yup",
+				"first_arg": "example",
+				"new_arg":   "example_2",
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			addCustomCNIArgs(tc.network, cniArgs)
+			test.Eq(t, tc.expectMap, cniArgs)
+
+		})
+	}
+}
+
 func TestCNI_setupTproxyArgs(t *testing.T) {
 	ci.Parallel(t)
 

client/taskenv/network.go

@@ -13,6 +13,7 @@ import (
 // Current interoperable fields:
 //   - Hostname
 //   - DNS
+//   - CNI
 func InterpolateNetworks(taskEnv *TaskEnv, networks structs.Networks) structs.Networks {
 
 	// Guard against not having a valid taskEnv. This can be the case if the
@@ -32,6 +33,12 @@ func InterpolateNetworks(taskEnv *TaskEnv, networks structs.Networks) structs.Ne
 			interpolated[i].DNS.Searches = taskEnv.ParseAndReplace(interpolated[i].DNS.Searches)
 			interpolated[i].DNS.Options = taskEnv.ParseAndReplace(interpolated[i].DNS.Options)
 		}
+		if interpolated[i].CNI != nil {
+			for k, v := range interpolated[i].CNI.Args {
+				interpolated[i].CNI.Args[k] = taskEnv.ReplaceEnv(v)
+
+			}
+		}
 	}
 
 	return interpolated

client/taskenv/network_test.go

@@ -84,6 +84,30 @@ func Test_InterpolateNetworks(t *testing.T) {
 			},
 			name: "interpolated dns servers",
 		},
+		{
+			inputTaskEnv: testEnv,
+			inputNetworks: structs.Networks{
+				{
+					CNI: &structs.CNIConfig{
+						Args: map[string]string{
+							"static": "example",
+							"second": "${foo}-opt",
+						},
+					},
+				},
+			},
+			expectedOutputNetworks: structs.Networks{
+				{
+					CNI: &structs.CNIConfig{
+						Args: map[string]string{
+							"static": "example",
+							"second": "bar-opt",
+						},
+					},
+				},
+			},
+			name: "interpolated and non-interpolated cni args",
+		},
 	}
 
 	for _, tc := range testCases {

command/agent/job_endpoint.go

@@ -1555,6 +1555,11 @@ func ApiNetworkResourceToStructs(in []*api.NetworkResource) []*structs.NetworkRe
 				Options:  nw.DNS.Options,
 			}
 		}
+		if nw.CNI != nil {
+			out[i].CNI = &structs.CNIConfig{
+				Args: nw.CNI.Args,
+			}
+		}
 
 		if l := len(nw.DynamicPorts); l != 0 {
 			out[i].DynamicPorts = make([]structs.Port, l)

nomad/structs/cni_config.go

@@ -0,0 +1,32 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package structs
+
+import (
+	"maps"
+)
+
+type CNIConfig struct {
+	Args map[string]string
+}
+
+func (d *CNIConfig) Copy() *CNIConfig {
+	if d == nil {
+		return nil
+	}
+	newMap := make(map[string]string)
+	for k, v := range d.Args {
+		newMap[k] = v
+	}
+	return &CNIConfig{
+		Args: newMap,
+	}
+}
+
+func (d *CNIConfig) Equal(o *CNIConfig) bool {
+	if d == nil || o == nil {
+		return d == o
+	}
+	return maps.Equal(d.Args, o.Args)
+}

nomad/structs/cni_config_test.go

@@ -0,0 +1,28 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package structs
+
+import (
+	"github.com/hashicorp/nomad/ci"
+	"github.com/shoenig/test/must"
+	"testing"
+)
+
+func TestCNIConfig_Equal(t *testing.T) {
+	ci.Parallel(t)
+
+	must.Equal[*CNIConfig](t, nil, nil)
+	must.NotEqual[*CNIConfig](t, nil, new(CNIConfig))
+	must.NotEqual[*CNIConfig](t, nil, &CNIConfig{Args: map[string]string{"first": "second"}})
+
+	must.StructEqual(t, &CNIConfig{
+		Args: map[string]string{
+			"arg":     "example_1",
+			"new_arg": "example_2",
+		},
+	}, []must.Tweak[*CNIConfig]{{
+		Field: "Args",
+		Apply: func(c *CNIConfig) { c.Args = map[string]string{"different": "arg"} },
+	}})
+}

nomad/structs/diff.go

@@ -2663,6 +2663,10 @@ func (n *NetworkResource) Diff(other *NetworkResource, contextual bool) *ObjectD
 		diff.Objects = append(diff.Objects, dnsDiff)
 	}
 
+	if cniDiff := n.CNI.Diff(other.CNI, contextual); cniDiff != nil {
+		diff.Objects = append(diff.Objects, cniDiff)
+	}
+
 	return diff
 }
 
@@ -2706,6 +2710,21 @@ func (d *DNSConfig) Diff(other *DNSConfig, contextual bool) *ObjectDiff {
 	return diff
 }
 
+// Diff returns a diff of two CNIConfig structs
+func (d *CNIConfig) Diff(other *CNIConfig, contextual bool) *ObjectDiff {
+	if d == nil {
+		d = &CNIConfig{}
+	}
+	if other == nil {
+		other = &CNIConfig{}
+	}
+	if d.Equal(other) {
+		return nil
+	}
+
+	return primitiveObjectDiff(d.Args, other.Args, nil, "CNIConfig", contextual)
+}
+
 func disconectStrategyDiffs(old, new *DisconnectStrategy, contextual bool) *ObjectDiff {
 	diff := &ObjectDiff{Type: DiffTypeNone, Name: "Disconnect"}
 	var oldDisconnectFlat, newDisconnectFlat map[string]string