diff --git a/.changelog/23389.txt b/.changelog/23389.txt new file mode 100644 index 00000000000..34297e62441 --- /dev/null +++ b/.changelog/23389.txt @@ -0,0 +1,3 @@ +```release-note:improvement +client: add a preferred_address_family config to prefer ipv4 or ipv6 when deducing IP from network interface +``` diff --git a/client/config/config.go b/client/config/config.go index 13db700b5bc..0e2b47dc724 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -116,6 +116,9 @@ type Config struct { // Network interface to be used in network fingerprinting NetworkInterface string + // Preferred address family to be used in network fingerprinting + PreferredAddressFamily structs.NodeNetworkAF + // Network speed is the default speed of network interfaces if they can not // be determined dynamically. NetworkSpeed int diff --git a/client/fingerprint/network.go b/client/fingerprint/network.go index fdcf2f9bc35..b15a3913464 100644 --- a/client/fingerprint/network.go +++ b/client/fingerprint/network.go @@ -6,6 +6,7 @@ package fingerprint import ( "fmt" "net" + "sort" "strings" log "github.com/hashicorp/go-hclog" @@ -100,7 +101,7 @@ func (f *NetworkFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpr // Create the network resources from the interface disallowLinkLocal := cfg.ReadBoolDefault(networkDisallowLinkLocalOption, networkDisallowLinkLocalDefault) - nwResources, err := f.createNetworkResources(mbits, intf, disallowLinkLocal) + nwResources, err := f.createNetworkResources(mbits, intf, disallowLinkLocal, cfg.PreferredAddressFamily) if err != nil { return err } @@ -169,6 +170,7 @@ func (f *NetworkFingerprint) createNodeNetworkResources(ifaces []net.Interface, } else { family = structs.NodeNetworkAF_IPv6 } + for _, alias := range deriveAddressAliases(iface, ip, conf) { newAddr := structs.NodeNetworkAddress{ Address: ip.String(), @@ -190,6 +192,9 @@ func (f *NetworkFingerprint) createNodeNetworkResources(ifaces []net.Interface, } } + sortNodeNetworkAddresses(networkAddrs, conf.PreferredAddressFamily) + sortNodeNetworkAddresses(linkLocalAddrs, conf.PreferredAddressFamily) + if len(networkAddrs) == 0 && len(linkLocalAddrs) > 0 { if disallowLinkLocal { f.logger.Debug("ignoring detected link-local address on interface", "interface", iface.Name) @@ -204,6 +209,9 @@ func (f *NetworkFingerprint) createNodeNetworkResources(ifaces []net.Interface, nets = append(nets, newNetwork) } } + + sortNodeNetworkResources(nets, conf.PreferredAddressFamily) + return nets, nil } @@ -257,7 +265,7 @@ func deriveAddressAliases(iface net.Interface, addr net.IP, config *config.Confi } // createNetworkResources creates network resources for every IP -func (f *NetworkFingerprint) createNetworkResources(throughput int, intf *net.Interface, disallowLinkLocal bool) ([]*structs.NetworkResource, error) { +func (f *NetworkFingerprint) createNetworkResources(throughput int, intf *net.Interface, disallowLinkLocal bool, preferredAF structs.NodeNetworkAF) ([]*structs.NetworkResource, error) { // Find the interface with the name addrs, err := f.interfaceDetector.Addrs(intf) if err != nil { @@ -301,6 +309,9 @@ func (f *NetworkFingerprint) createNetworkResources(throughput int, intf *net.In nwResources = append(nwResources, newNetwork) } + sortNetworkResources(nwResources, preferredAF) + sortNetworkResources(linkLocals, preferredAF) + if len(nwResources) == 0 && len(linkLocals) != 0 { if disallowLinkLocal { f.logger.Debug("ignoring detected link-local address on interface", "interface", intf.Name) @@ -335,3 +346,77 @@ func (f *NetworkFingerprint) findInterface(deviceName string) (*net.Interface, e return f.interfaceDetector.InterfaceByName(deviceName) } + +// Define a type for the comparison function +type LessFunc[T any] func(a, b T) bool + +// Generic sort function +func sortResources[T any](res []T, less LessFunc[T]) { + sort.Slice(res, func(i, j int) bool { + return less(res[i], res[j]) + }) +} + +// Less functions for each resource type and address family +func lessNetworkResourceIPv4(a, b *structs.NetworkResource) bool { + return net.ParseIP(a.IP).To4() != nil && net.ParseIP(b.IP).To4() == nil +} + +func lessNetworkResourceIPv6(a, b *structs.NetworkResource) bool { + return net.ParseIP(a.IP).To4() == nil && net.ParseIP(b.IP).To4() != nil +} + +func lessNodeNetworkResourceIPv4(a, b *structs.NodeNetworkResource) bool { + if len(a.Addresses) == 0 && len(b.Addresses) == 0 { + return false + } else if len(a.Addresses) == 0 { + return false + } else if len(b.Addresses) == 0 { + return true + } else if a.Addresses[0].Family == structs.NodeNetworkAF_IPv4 && b.Addresses[0].Family == structs.NodeNetworkAF_IPv6 { + return true + } + return false +} + +func lessNodeNetworkResourceIPv6(a, b *structs.NodeNetworkResource) bool { + if len(a.Addresses) == 0 { + return false + } else if len(b.Addresses) == 0 { + return true + } + return a.Addresses[0].Family == structs.NodeNetworkAF_IPv6 && b.Addresses[0].Family == structs.NodeNetworkAF_IPv4 +} + +func lessNodeNetworkAddressIPv4(a, b structs.NodeNetworkAddress) bool { + return a.Family == structs.NodeNetworkAF_IPv4 && b.Family == structs.NodeNetworkAF_IPv6 +} + +func lessNodeNetworkAddressIPv6(a, b structs.NodeNetworkAddress) bool { + return a.Family == structs.NodeNetworkAF_IPv6 && b.Family == structs.NodeNetworkAF_IPv4 +} + +// Sorting functions for different resource types and address families +func sortNetworkResources(res []*structs.NetworkResource, preferredAF structs.NodeNetworkAF) { + if preferredAF == structs.NodeNetworkAF_IPv4 { + sortResources(res, lessNetworkResourceIPv4) + } else if preferredAF == structs.NodeNetworkAF_IPv6 { + sortResources(res, lessNetworkResourceIPv6) + } +} + +func sortNodeNetworkResources(res []*structs.NodeNetworkResource, preferredAF structs.NodeNetworkAF) { + if preferredAF == structs.NodeNetworkAF_IPv4 { + sortResources(res, lessNodeNetworkResourceIPv4) + } else if preferredAF == structs.NodeNetworkAF_IPv6 { + sortResources(res, lessNodeNetworkResourceIPv6) + } +} + +func sortNodeNetworkAddresses(res []structs.NodeNetworkAddress, preferredAF structs.NodeNetworkAF) { + if preferredAF == structs.NodeNetworkAF_IPv4 { + sortResources(res, lessNodeNetworkAddressIPv4) + } else if preferredAF == structs.NodeNetworkAF_IPv6 { + sortResources(res, lessNodeNetworkAddressIPv6) + } +} diff --git a/client/fingerprint/network_test.go b/client/fingerprint/network_test.go index cf428d09738..63009e3b928 100644 --- a/client/fingerprint/network_test.go +++ b/client/fingerprint/network_test.go @@ -8,6 +8,7 @@ import ( "net" "os" "sort" + "strings" "testing" "github.com/hashicorp/nomad/ci" @@ -73,6 +74,13 @@ var ( } ) +const ( + loIPv4 = "127.0.0.1" + loCIDRv4 = loIPv4 + "/8" + loIPv6 = "2001:DB8::" + loCIDRv6 = loIPv6 + "/48" +) + // A fake network detector which returns no devices type NetworkInterfaceDetectorNoDevices struct { } @@ -107,8 +115,8 @@ func (n *NetworkInterfaceDetectorOnlyLo) InterfaceByName(name string) (*net.Inte func (n *NetworkInterfaceDetectorOnlyLo) Addrs(intf *net.Interface) ([]net.Addr, error) { if intf.Name == "lo" { - _, ipnet1, _ := net.ParseCIDR("127.0.0.1/8") - _, ipnet2, _ := net.ParseCIDR("2001:DB8::/48") + _, ipnet1, _ := net.ParseCIDR(loCIDRv6) + _, ipnet2, _ := net.ParseCIDR(loCIDRv4) return []net.Addr{ipnet1, ipnet2}, nil } @@ -149,8 +157,8 @@ func (n *NetworkInterfaceDetectorMultipleInterfaces) InterfaceByName(name string func (n *NetworkInterfaceDetectorMultipleInterfaces) Addrs(intf *net.Interface) ([]net.Addr, error) { if intf.Name == "lo" { - _, ipnet1, _ := net.ParseCIDR("127.0.0.1/8") - _, ipnet2, _ := net.ParseCIDR("2001:DB8::/48") + _, ipnet1, _ := net.ParseCIDR(loCIDRv6) + _, ipnet2, _ := net.ParseCIDR(loCIDRv4) return []net.Addr{ipnet1, ipnet2}, nil } @@ -269,54 +277,78 @@ func TestNetworkFingerprint_default_device_absent(t *testing.T) { func TestNetworkFingerPrint_default_device(t *testing.T) { ci.Parallel(t) - f := &NetworkFingerprint{logger: testlog.HCLogger(t), interfaceDetector: &NetworkInterfaceDetectorOnlyLo{}} node := &structs.Node{ Attributes: make(map[string]string), } - cfg := &config.Config{NetworkSpeed: 100, NetworkInterface: "lo"} - request := &FingerprintRequest{Config: cfg, Node: node} - var response FingerprintResponse - err := f.Fingerprint(request, &response) - if err != nil { - t.Fatalf("err: %v", err) + testCases := []struct { + name string + config *config.Config + expectedIP string + }{ + { + name: "Loopback IPv6", + config: &config.Config{NetworkSpeed: 100, NetworkInterface: "lo", PreferredAddressFamily: "ipv6"}, + expectedIP: loIPv6, + }, + { + name: "Loopback IPv4", + config: &config.Config{NetworkSpeed: 100, NetworkInterface: "lo", PreferredAddressFamily: "ipv4"}, + expectedIP: "127.0.0.0", // CIDR 127.0.0.1/8 result in 127.0.0.0 IP ? + }, + { + name: "Loopback No preferred address family", + config: &config.Config{NetworkSpeed: 100, NetworkInterface: "lo"}, + expectedIP: loIPv6, + }, } - if !response.Detected { - t.Fatalf("expected response to be applicable") - } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + request := &FingerprintRequest{Config: tc.config, Node: node} + var response FingerprintResponse + err := f.Fingerprint(request, &response) + if err != nil { + t.Fatalf("err: %v", err) + } - attributes := response.Attributes - if len(attributes) == 0 { - t.Fatalf("should apply") - } + if !response.Detected { + t.Fatalf("expected response to be applicable") + } - assertNodeAttributeContains(t, attributes, "unique.network.ip-address") + attributes := response.Attributes + if len(attributes) == 0 { + t.Fatalf("should apply") + } - ip := attributes["unique.network.ip-address"] - match := net.ParseIP(ip) - if match == nil { - t.Fatalf("Bad IP match: %s", ip) - } + assertNodeAttributeContains(t, attributes, "unique.network.ip-address") - if len(response.NodeResources.Networks) == 0 { - t.Fatal("Expected to find Network Resources") - } + ip := attributes["unique.network.ip-address"] + match := net.ParseIP(ip) + if match == nil { + t.Fatalf("Bad IP match: %s", ip) + } - // Test at least the first Network Resource - net := response.NodeResources.Networks[0] - if net.IP == "" { - t.Fatal("Expected Network Resource to not be empty") - } - if net.CIDR == "" { - t.Fatal("Expected Network Resource to have a CIDR") - } - if net.Device == "" { - t.Fatal("Expected Network Resource to have a Device Name") - } - if net.MBits == 0 { - t.Fatal("Expected Network Resource to have a non-zero bandwidth") + if len(response.NodeResources.Networks) == 0 { + t.Fatal("Expected to find Network Resources") + } + + // Test at least the first Network Resource + net := response.NodeResources.Networks[0] + if !strings.EqualFold(tc.expectedIP, net.IP) { + t.Errorf("Expected IP %s; got %s", tc.expectedIP, net.IP) + } + if net.CIDR == "" { + t.Fatal("Expected Network Resource to have a CIDR") + } + if net.Device == "" { + t.Fatal("Expected Network Resource to have a Device Name") + } + if net.MBits == 0 { + t.Fatal("Expected Network Resource to have a non-zero bandwidth") + } + }) } } diff --git a/client/taskenv/env.go b/client/taskenv/env.go index d3c9eb38bce..e1df689ba73 100644 --- a/client/taskenv/env.go +++ b/client/taskenv/env.go @@ -950,7 +950,16 @@ func buildNetworkEnv(envMap map[string]string, nets structs.Networks, driverNet func buildPortEnv(envMap map[string]string, p structs.Port, ip string, driverNet *drivers.DriverNetwork) { // Host IP, port, and address portStr := strconv.Itoa(p.Value) + + var ipFamilyPrefix string + if strings.Contains(ip, ":") { + ipFamilyPrefix = "NOMAD_IPv6_" + } else { + ipFamilyPrefix = "NOMAD_IPv4_" + } + envMap[IpPrefix+p.Label] = ip + envMap[ipFamilyPrefix+p.Label] = ip envMap[HostPortPrefix+p.Label] = portStr envMap[AddrPrefix+p.Label] = net.JoinHostPort(ip, portStr) diff --git a/client/taskenv/env_test.go b/client/taskenv/env_test.go index a2e6db61a50..fe53484a74d 100644 --- a/client/taskenv/env_test.go +++ b/client/taskenv/env_test.go @@ -219,9 +219,11 @@ func TestEnvironment_AsList(t *testing.T) { "NOMAD_ADDR_http=127.0.0.1:80", "NOMAD_PORT_http=80", "NOMAD_IP_http=127.0.0.1", + "NOMAD_IPv4_http=127.0.0.1", "NOMAD_ADDR_https=127.0.0.1:8080", "NOMAD_PORT_https=443", "NOMAD_IP_https=127.0.0.1", + "NOMAD_IPv4_https=127.0.0.1", "NOMAD_HOST_PORT_http=80", "NOMAD_HOST_PORT_https=8080", "NOMAD_TASK_NAME=web", diff --git a/command/agent/agent.go b/command/agent/agent.go index 1dde7ae2336..c8f3fc31bee 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -737,6 +737,9 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) { if agentConfig.Client.NetworkInterface != "" { conf.NetworkInterface = agentConfig.Client.NetworkInterface } + + conf.PreferredAddressFamily = agentConfig.Client.PreferredAddressFamily + conf.ChrootEnv = agentConfig.Client.ChrootEnv conf.Options = agentConfig.Client.Options if agentConfig.Client.NetworkSpeed != 0 { diff --git a/command/agent/command.go b/command/agent/command.go index ca73d97c223..088bf9a819c 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -116,6 +116,7 @@ func (c *Command) readConfig() *Config { flags.StringVar(&servers, "servers", "", "") flags.Var((*flaghelper.StringFlag)(&meta), "meta", "") flags.StringVar(&cmdConfig.Client.NetworkInterface, "network-interface", "", "") + flags.StringVar((*string)(&cmdConfig.Client.PreferredAddressFamily), "preferred-address-family", "", "ipv4 or ipv6") flags.IntVar(&cmdConfig.Client.NetworkSpeed, "network-speed", 0, "") // General options @@ -485,6 +486,14 @@ func (c *Command) IsValidConfig(config, cmdConfig *Config) bool { return false } + if err := config.Client.PreferredAddressFamily.Validate(); err != nil { + c.Ui.Error(fmt.Sprintf("Invalid preferred-address-family value: %s (valid values: %s, %s)", + config.Client.PreferredAddressFamily, + structs.NodeNetworkAF_IPv4, structs.NodeNetworkAF_IPv6), + ) + return false + } + if !config.DevMode { // Ensure that we have the directories we need to run. if config.Server.Enabled && config.DataDir == "" { @@ -1549,6 +1558,11 @@ Client Options: -network-interface Forces the network fingerprinter to use the specified network interface. + + -preferred-address-family + Specify which IP family to prefer when selecting an IP address of the + network interface. Valid values are "ipv4" and "ipv6". When not specified, + the agent will not sort the addresses and use the first one. -network-speed The default speed for network interfaces in MBits if the link speed can not diff --git a/command/agent/config.go b/command/agent/config.go index cd53fe91d21..ef80e7718cc 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -256,6 +256,11 @@ type ClientConfig struct { // Interface to use for network fingerprinting NetworkInterface string `hcl:"network_interface"` + // Sort the IP addresses by the preferred IP family. This is useful when + // the interface has multiple IP addresses and the client should prefer + // one over the other. + PreferredAddressFamily structs.NodeNetworkAF `hcl:"preferred_address_family"` + // NetworkSpeed is used to override any detected or default network link // speed. NetworkSpeed int `hcl:"network_speed"` @@ -2265,6 +2270,11 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig { if b.NetworkInterface != "" { result.NetworkInterface = b.NetworkInterface } + + if b.PreferredAddressFamily != "" { + result.PreferredAddressFamily = b.PreferredAddressFamily + } + if b.NetworkSpeed != 0 { result.NetworkSpeed = b.NetworkSpeed } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 1d1294ccd37..a2a5636a409 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -2729,6 +2729,14 @@ const ( NodeNetworkAF_IPv6 NodeNetworkAF = "ipv6" ) +// Validate validates that NodeNetworkAF has a legal value. +func (n NodeNetworkAF) Validate() error { + if n == "" || n == NodeNetworkAF_IPv4 || n == NodeNetworkAF_IPv6 { + return nil + } + return fmt.Errorf(`network address family must be one of: "", %q, %q`, NodeNetworkAF_IPv4, NodeNetworkAF_IPv6) +} + type NodeNetworkAddress struct { Family NodeNetworkAF Alias string diff --git a/website/content/docs/configuration/client.mdx b/website/content/docs/configuration/client.mdx index 4ddd968c623..557c3c0ff05 100644 --- a/website/content/docs/configuration/client.mdx +++ b/website/content/docs/configuration/client.mdx @@ -57,6 +57,12 @@ client { [`"fingerprint.network.disallow_link_local"`](#fingerprint-network-disallow_link_local) configuration value. +- `preferred_address_family` `(string: "")` - Specifies the preferred address family + for the network interface. The value can be `ipv4` or `ipv6`. If the selected network + interface has both IPv4 and IPv6 addresses, this option will select an IP address of + the preferred family. When the option is not specified, the current behavior is conserved: + the first IP address is selected no matter the family. + - `cpu_total_compute` `(int: 0)` - Specifies an override for the total CPU compute. This value should be set to `# Cores * Core MHz`. For example, a quad-core running at 2 GHz would have a total compute of 8000 (4 \* 2000). Most