feat(#23388): Add a preferred address family option for network-interface

.changelog/23389.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+client: add a preferred_address_family config to prefer ipv4 or ipv6 when deducing IP from network interface
+```

client/config/config.go

@@ -116,6 +116,9 @@ type Config struct {
 	// Network interface to be used in network fingerprinting
 	NetworkInterface string
 
+	// Preferred address family to be used in network fingerprinting
+	PreferredAddressFamily structs.NodeNetworkAF
+
 	// Network speed is the default speed of network interfaces if they can not
 	// be determined dynamically.
 	NetworkSpeed int

client/fingerprint/network.go

@@ -6,6 +6,7 @@ package fingerprint
 import (
 	"fmt"
 	"net"
+	"sort"
 	"strings"
 
 	log "github.com/hashicorp/go-hclog"
@@ -100,7 +101,7 @@ func (f *NetworkFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpr
 
 	// Create the network resources from the interface
 	disallowLinkLocal := cfg.ReadBoolDefault(networkDisallowLinkLocalOption, networkDisallowLinkLocalDefault)
-	nwResources, err := f.createNetworkResources(mbits, intf, disallowLinkLocal)
+	nwResources, err := f.createNetworkResources(mbits, intf, disallowLinkLocal, cfg.PreferredAddressFamily)
 	if err != nil {
 		return err
 	}
@@ -169,6 +170,7 @@ func (f *NetworkFingerprint) createNodeNetworkResources(ifaces []net.Interface,
 			} else {
 				family = structs.NodeNetworkAF_IPv6
 			}
+
 			for _, alias := range deriveAddressAliases(iface, ip, conf) {
 				newAddr := structs.NodeNetworkAddress{
 					Address: ip.String(),
@@ -190,6 +192,9 @@ func (f *NetworkFingerprint) createNodeNetworkResources(ifaces []net.Interface,
 			}
 		}
 
+		sortNodeNetworkAddresses(networkAddrs, conf.PreferredAddressFamily)
+		sortNodeNetworkAddresses(linkLocalAddrs, conf.PreferredAddressFamily)
+
 		if len(networkAddrs) == 0 && len(linkLocalAddrs) > 0 {
 			if disallowLinkLocal {
 				f.logger.Debug("ignoring detected link-local address on interface", "interface", iface.Name)
@@ -204,6 +209,9 @@ func (f *NetworkFingerprint) createNodeNetworkResources(ifaces []net.Interface,
 			nets = append(nets, newNetwork)
 		}
 	}
+
+	sortNodeNetworkResources(nets, conf.PreferredAddressFamily)
+
 	return nets, nil
 }
 
@@ -257,7 +265,7 @@ func deriveAddressAliases(iface net.Interface, addr net.IP, config *config.Confi
 }
 
 // createNetworkResources creates network resources for every IP
-func (f *NetworkFingerprint) createNetworkResources(throughput int, intf *net.Interface, disallowLinkLocal bool) ([]*structs.NetworkResource, error) {
+func (f *NetworkFingerprint) createNetworkResources(throughput int, intf *net.Interface, disallowLinkLocal bool, preferredAF structs.NodeNetworkAF) ([]*structs.NetworkResource, error) {
 	// Find the interface with the name
 	addrs, err := f.interfaceDetector.Addrs(intf)
 	if err != nil {
@@ -301,6 +309,9 @@ func (f *NetworkFingerprint) createNetworkResources(throughput int, intf *net.In
 		nwResources = append(nwResources, newNetwork)
 	}
 
+	sortNetworkResources(nwResources, preferredAF)
+	sortNetworkResources(linkLocals, preferredAF)
+
 	if len(nwResources) == 0 && len(linkLocals) != 0 {
 		if disallowLinkLocal {
 			f.logger.Debug("ignoring detected link-local address on interface", "interface", intf.Name)
@@ -335,3 +346,77 @@ func (f *NetworkFingerprint) findInterface(deviceName string) (*net.Interface, e
 
 	return f.interfaceDetector.InterfaceByName(deviceName)
 }
+
+// Define a type for the comparison function
+type LessFunc[T any] func(a, b T) bool
+
+// Generic sort function
+func sortResources[T any](res []T, less LessFunc[T]) {
+	sort.Slice(res, func(i, j int) bool {
+		return less(res[i], res[j])
+	})
+}
+
+// Less functions for each resource type and address family
+func lessNetworkResourceIPv4(a, b *structs.NetworkResource) bool {
+	return net.ParseIP(a.IP).To4() != nil && net.ParseIP(b.IP).To4() == nil
+}
+
+func lessNetworkResourceIPv6(a, b *structs.NetworkResource) bool {
+	return net.ParseIP(a.IP).To4() == nil && net.ParseIP(b.IP).To4() != nil
+}
+
+func lessNodeNetworkResourceIPv4(a, b *structs.NodeNetworkResource) bool {
+	if len(a.Addresses) == 0 && len(b.Addresses) == 0 {
+		return false
+	} else if len(a.Addresses) == 0 {
+		return false
+	} else if len(b.Addresses) == 0 {
+		return true
+	} else if a.Addresses[0].Family == structs.NodeNetworkAF_IPv4 && b.Addresses[0].Family == structs.NodeNetworkAF_IPv6 {
+		return true
+	}
+	return false
+}
+
+func lessNodeNetworkResourceIPv6(a, b *structs.NodeNetworkResource) bool {
+	if len(a.Addresses) == 0 {
+		return false
+	} else if len(b.Addresses) == 0 {
+		return true
+	}
+	return a.Addresses[0].Family == structs.NodeNetworkAF_IPv6 && b.Addresses[0].Family == structs.NodeNetworkAF_IPv4
+}
+
+func lessNodeNetworkAddressIPv4(a, b structs.NodeNetworkAddress) bool {
+	return a.Family == structs.NodeNetworkAF_IPv4 && b.Family == structs.NodeNetworkAF_IPv6
+}
+
+func lessNodeNetworkAddressIPv6(a, b structs.NodeNetworkAddress) bool {
+	return a.Family == structs.NodeNetworkAF_IPv6 && b.Family == structs.NodeNetworkAF_IPv4
+}
+
+// Sorting functions for different resource types and address families
+func sortNetworkResources(res []*structs.NetworkResource, preferredAF structs.NodeNetworkAF) {
+	if preferredAF == structs.NodeNetworkAF_IPv4 {
+		sortResources(res, lessNetworkResourceIPv4)
+	} else if preferredAF == structs.NodeNetworkAF_IPv6 {
+		sortResources(res, lessNetworkResourceIPv6)
+	}
+}
+
+func sortNodeNetworkResources(res []*structs.NodeNetworkResource, preferredAF structs.NodeNetworkAF) {
+	if preferredAF == structs.NodeNetworkAF_IPv4 {
+		sortResources(res, lessNodeNetworkResourceIPv4)
+	} else if preferredAF == structs.NodeNetworkAF_IPv6 {
+		sortResources(res, lessNodeNetworkResourceIPv6)
+	}
+}
+
+func sortNodeNetworkAddresses(res []structs.NodeNetworkAddress, preferredAF structs.NodeNetworkAF) {
+	if preferredAF == structs.NodeNetworkAF_IPv4 {
+		sortResources(res, lessNodeNetworkAddressIPv4)
+	} else if preferredAF == structs.NodeNetworkAF_IPv6 {
+		sortResources(res, lessNodeNetworkAddressIPv6)
+	}
+}

client/fingerprint/network_test.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"os"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/hashicorp/nomad/ci"
@@ -73,6 +74,13 @@ var (
 	}
 )
 
+const (
+	loIPv4   = "127.0.0.1"
+	loCIDRv4 = loIPv4 + "/8"
+	loIPv6   = "2001:DB8::"
+	loCIDRv6 = loIPv6 + "/48"
+)
+
 // A fake network detector which returns no devices
 type NetworkInterfaceDetectorNoDevices struct {
 }
@@ -107,8 +115,8 @@ func (n *NetworkInterfaceDetectorOnlyLo) InterfaceByName(name string) (*net.Inte
 
 func (n *NetworkInterfaceDetectorOnlyLo) Addrs(intf *net.Interface) ([]net.Addr, error) {
 	if intf.Name == "lo" {
-		_, ipnet1, _ := net.ParseCIDR("127.0.0.1/8")
-		_, ipnet2, _ := net.ParseCIDR("2001:DB8::/48")
+		_, ipnet1, _ := net.ParseCIDR(loCIDRv6)
+		_, ipnet2, _ := net.ParseCIDR(loCIDRv4)
 		return []net.Addr{ipnet1, ipnet2}, nil
 	}
 
@@ -149,8 +157,8 @@ func (n *NetworkInterfaceDetectorMultipleInterfaces) InterfaceByName(name string
 
 func (n *NetworkInterfaceDetectorMultipleInterfaces) Addrs(intf *net.Interface) ([]net.Addr, error) {
 	if intf.Name == "lo" {
-		_, ipnet1, _ := net.ParseCIDR("127.0.0.1/8")
-		_, ipnet2, _ := net.ParseCIDR("2001:DB8::/48")
+		_, ipnet1, _ := net.ParseCIDR(loCIDRv6)
+		_, ipnet2, _ := net.ParseCIDR(loCIDRv4)
 		return []net.Addr{ipnet1, ipnet2}, nil
 	}
 
@@ -269,14 +277,36 @@ func TestNetworkFingerprint_default_device_absent(t *testing.T) {
 
 func TestNetworkFingerPrint_default_device(t *testing.T) {
 	ci.Parallel(t)
-
 	f := &NetworkFingerprint{logger: testlog.HCLogger(t), interfaceDetector: &NetworkInterfaceDetectorOnlyLo{}}
 	node := &structs.Node{
 		Attributes: make(map[string]string),
 	}
-	cfg := &config.Config{NetworkSpeed: 100, NetworkInterface: "lo"}
 
-	request := &FingerprintRequest{Config: cfg, Node: node}
+	testCases := []struct {
+		name       string
+		config     *config.Config
+		expectedIP string
+	}{
+		{
+			name:       "Loopback IPv6",
+			config:     &config.Config{NetworkSpeed: 100, NetworkInterface: "lo", PreferredAddressFamily: "ipv6"},
+			expectedIP: loIPv6,
+		},
+		{
+			name:       "Loopback IPv4",
+			config:     &config.Config{NetworkSpeed: 100, NetworkInterface: "lo", PreferredAddressFamily: "ipv4"},
+			expectedIP: "127.0.0.0", // CIDR 127.0.0.1/8 result in 127.0.0.0 IP ?
+		},
+		{
+			name:       "Loopback No preferred address family",
+			config:     &config.Config{NetworkSpeed: 100, NetworkInterface: "lo"},
+			expectedIP: loIPv6,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			request := &FingerprintRequest{Config: tc.config, Node: node}
 			var response FingerprintResponse
 			err := f.Fingerprint(request, &response)
 			if err != nil {
@@ -306,8 +336,8 @@ func TestNetworkFingerPrint_default_device(t *testing.T) {
 
 			// Test at least the first Network Resource
 			net := response.NodeResources.Networks[0]
-	if net.IP == "" {
-		t.Fatal("Expected Network Resource to not be empty")
+			if !strings.EqualFold(tc.expectedIP, net.IP) {
+				t.Errorf("Expected IP %s; got %s", tc.expectedIP, net.IP)
 			}
 			if net.CIDR == "" {
 				t.Fatal("Expected Network Resource to have a CIDR")
@@ -318,6 +348,8 @@ func TestNetworkFingerPrint_default_device(t *testing.T) {
 			if net.MBits == 0 {
 				t.Fatal("Expected Network Resource to have a non-zero bandwidth")
 			}
+		})
+	}
 }
 
 func TestNetworkFingerPrint_LinkLocal_Allowed(t *testing.T) {

client/taskenv/env.go

@@ -950,7 +950,16 @@ func buildNetworkEnv(envMap map[string]string, nets structs.Networks, driverNet
 func buildPortEnv(envMap map[string]string, p structs.Port, ip string, driverNet *drivers.DriverNetwork) {
 	// Host IP, port, and address
 	portStr := strconv.Itoa(p.Value)
+
+	var ipFamilyPrefix string
+	if strings.Contains(ip, ":") {
+		ipFamilyPrefix = "NOMAD_IPv6_"
+	} else {
+		ipFamilyPrefix = "NOMAD_IPv4_"
+	}
+
 	envMap[IpPrefix+p.Label] = ip
+	envMap[ipFamilyPrefix+p.Label] = ip
 	envMap[HostPortPrefix+p.Label] = portStr
 	envMap[AddrPrefix+p.Label] = net.JoinHostPort(ip, portStr)
 

client/taskenv/env_test.go

@@ -219,9 +219,11 @@ func TestEnvironment_AsList(t *testing.T) {
 		"NOMAD_ADDR_http=127.0.0.1:80",
 		"NOMAD_PORT_http=80",
 		"NOMAD_IP_http=127.0.0.1",
+		"NOMAD_IPv4_http=127.0.0.1",
 		"NOMAD_ADDR_https=127.0.0.1:8080",
 		"NOMAD_PORT_https=443",
 		"NOMAD_IP_https=127.0.0.1",
+		"NOMAD_IPv4_https=127.0.0.1",
 		"NOMAD_HOST_PORT_http=80",
 		"NOMAD_HOST_PORT_https=8080",
 		"NOMAD_TASK_NAME=web",

command/agent/agent.go

@@ -737,6 +737,9 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
 	if agentConfig.Client.NetworkInterface != "" {
 		conf.NetworkInterface = agentConfig.Client.NetworkInterface
 	}
+
+	conf.PreferredAddressFamily = agentConfig.Client.PreferredAddressFamily
+
 	conf.ChrootEnv = agentConfig.Client.ChrootEnv
 	conf.Options = agentConfig.Client.Options
 	if agentConfig.Client.NetworkSpeed != 0 {

command/agent/command.go

@@ -116,6 +116,7 @@ func (c *Command) readConfig() *Config {
 	flags.StringVar(&servers, "servers", "", "")
 	flags.Var((*flaghelper.StringFlag)(&meta), "meta", "")
 	flags.StringVar(&cmdConfig.Client.NetworkInterface, "network-interface", "", "")
+	flags.StringVar((*string)(&cmdConfig.Client.PreferredAddressFamily), "preferred-address-family", "", "ipv4 or ipv6")
 	flags.IntVar(&cmdConfig.Client.NetworkSpeed, "network-speed", 0, "")
 
 	// General options
@@ -485,6 +486,14 @@ func (c *Command) IsValidConfig(config, cmdConfig *Config) bool {
 		return false
 	}
 
+	if err := config.Client.PreferredAddressFamily.Validate(); err != nil {
+		c.Ui.Error(fmt.Sprintf("Invalid preferred-address-family value: %s (valid values: %s, %s)",
+			config.Client.PreferredAddressFamily,
+			structs.NodeNetworkAF_IPv4, structs.NodeNetworkAF_IPv6),
+		)
+		return false
+	}
+
 	if !config.DevMode {
 		// Ensure that we have the directories we need to run.
 		if config.Server.Enabled && config.DataDir == "" {
@@ -1550,6 +1559,11 @@ Client Options:
   -network-interface
     Forces the network fingerprinter to use the specified network interface.
 
+  -preferred-address-family
+    Specify which IP family to prefer when selecting an IP address of the
+    network interface. Valid values are "ipv4" and "ipv6". When not specified,
+    the agent will not sort the addresses and use the first one.
+
   -network-speed
     The default speed for network interfaces in MBits if the link speed can not
     be determined dynamically.

command/agent/config.go

@@ -256,6 +256,11 @@ type ClientConfig struct {
 	// Interface to use for network fingerprinting
 	NetworkInterface string `hcl:"network_interface"`
 
+	// Sort the IP addresses by the preferred IP family. This is useful when
+	// the interface has multiple IP addresses and the client should prefer
+	// one over the other.
+	PreferredAddressFamily structs.NodeNetworkAF `hcl:"preferred_address_family"`
+
 	// NetworkSpeed is used to override any detected or default network link
 	// speed.
 	NetworkSpeed int `hcl:"network_speed"`
@@ -2265,6 +2270,11 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
 	if b.NetworkInterface != "" {
 		result.NetworkInterface = b.NetworkInterface
 	}
+
+	if b.PreferredAddressFamily != "" {
+		result.PreferredAddressFamily = b.PreferredAddressFamily
+	}
+
 	if b.NetworkSpeed != 0 {
 		result.NetworkSpeed = b.NetworkSpeed
 	}

nomad/structs/structs.go

@@ -2729,6 +2729,14 @@ const (
 	NodeNetworkAF_IPv6 NodeNetworkAF = "ipv6"
 )
 
+// Validate validates that NodeNetworkAF has a legal value.
+func (n NodeNetworkAF) Validate() error {
+	if n == "" || n == NodeNetworkAF_IPv4 || n == NodeNetworkAF_IPv6 {
+		return nil
+	}
+	return fmt.Errorf(`network address family must be one of: "", %q, %q`, NodeNetworkAF_IPv4, NodeNetworkAF_IPv6)
+}
+
 type NodeNetworkAddress struct {
 	Family        NodeNetworkAF
 	Alias         string

website/content/docs/configuration/client.mdx

@@ -57,6 +57,12 @@ client {
   [`"fingerprint.network.disallow_link_local"`](#fingerprint-network-disallow_link_local)
   configuration value.
 
+- `preferred_address_family` `(string: "")` - Specifies the preferred address family
+  for the network interface. The value can be `ipv4` or `ipv6`. If the selected network
+  interface has both IPv4 and IPv6 addresses, this option will select an IP address of
+  the preferred family. When the option is not specified, the current behavior is conserved:
+  the first IP address is selected no matter the family.
+
 - `cpu_total_compute` `(int: 0)` - Specifies an override for the total CPU
   compute. This value should be set to `# Cores * Core MHz`. For example, a
   quad-core running at 2 GHz would have a total compute of 8000 (4 \* 2000). Most