Configurable dynamic port range

The intent of this change is to support shifting the range used for assigning dynamic ports (hashicorp#8186).

This makes the static dynamic port range configurable per new client settings `dynamic_port_range_min` and `dynamic_port_range_max`.

client/config/config.go

@@ -268,6 +268,12 @@ type Config struct {
 	//
 	// This configuration is only considered if no host networks are defined.
 	BindWildcardDefaultHostNetwork bool
+
+	// DynamicPortRangeMin is the lowest port for dynamic ports
+	DynamicPortRangeMin int
+
+	// DynamicPortRangeMin is the highest port for dynamic ports
+	DynamicPortRangeMax int
 }
 
 type ClientTemplateConfig struct {

command/agent/agent.go

@@ -642,6 +642,9 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
 	}
 	conf.BindWildcardDefaultHostNetwork = agentConfig.Client.BindWildcardDefaultHostNetwork
 
+	conf.DynamicPortRangeMin = agentConfig.Client.DynamicPortRangeMin
+	conf.DynamicPortRangeMax = agentConfig.Client.DynamicPortRangeMax
+
 	return conf, nil
 }
 

command/agent/command.go

@@ -30,6 +30,7 @@ import (
 	gatedwriter "github.com/hashicorp/nomad/helper/gated-writer"
 	"github.com/hashicorp/nomad/helper/logging"
 	"github.com/hashicorp/nomad/helper/winsvc"
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/nomad/structs/config"
 	"github.com/hashicorp/nomad/version"
 	"github.com/mitchellh/cli"
@@ -381,6 +382,13 @@ func (c *Command) isValidConfig(config, cmdConfig *Config) bool {
 		}
 	}
 
+	if config.Client.DynamicPortRangeMin > 0 &&
+		config.Client.DynamicPortRangeMax > 0 &&
+		config.Client.DynamicPortRangeMin >= config.Client.DynamicPortRangeMax {
+		c.Ui.Error("dynamic_port_range_min must be less than dynamic_port_range_max.")
+		return false
+	}
+
 	return true
 }
 
@@ -606,6 +614,16 @@ func (c *Command) Run(args []string) int {
 		return 1
 	}
 
+	// Configure dynamic port range
+	if config.Client.DynamicPortRangeMin > 0 && config.Client.DynamicPortRangeMax > 0 {
+		structs.SetDynamicPortRange(
+			structs.PortRange{
+				Min: config.Client.DynamicPortRangeMin,
+				Max: config.Client.DynamicPortRangeMax,
+			},
+		)
+	}
+
 	// Setup the log outputs
 	logFilter, logGate, logOutput := SetupLoggers(c.Ui, config)
 	c.logFilter = logFilter

command/agent/config.go

@@ -301,6 +301,12 @@ type ClientConfig struct {
 
 	// ExtraKeysHCL is used by hcl to surface unexpected keys
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
+
+	// DynamicPortRangeMin is the lowest port for dynamic ports
+	DynamicPortRangeMin int `hcl:"dynamic_port_range_min"`
+
+	// DynamicPortRangeMax is the highest port for dynamic ports
+	DynamicPortRangeMax int `hcl:"dynamic_port_range_max"`
 }
 
 // ClientTemplateConfig is configuration on the client specific to template
@@ -1558,6 +1564,15 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
 	if b.BindWildcardDefaultHostNetwork {
 		result.BindWildcardDefaultHostNetwork = true
 	}
+
+	if b.DynamicPortRangeMin != 0 {
+		result.DynamicPortRangeMin = b.DynamicPortRangeMin
+	}
+
+	if b.DynamicPortRangeMax != 0 {
+		result.DynamicPortRangeMax = b.DynamicPortRangeMax
+	}
+
 	return &result
 }
 

command/agent/config_parse_test.go

@@ -89,6 +89,8 @@ var basicConfig = &Config{
 		CNIPath:             "/tmp/cni_path",
 		BridgeNetworkName:   "custom_bridge_name",
 		BridgeNetworkSubnet: "custom_bridge_subnet",
+		DynamicPortRangeMin: 3000,
+		DynamicPortRangeMax: 4000,
 	},
 	Server: &ServerConfig{
 		Enabled:                   true,
@@ -604,7 +606,9 @@ var sample0 = &Config{
 		RPC:  "host.example.com",
 		Serf: "host.example.com",
 	},
-	Client: &ClientConfig{ServerJoin: &ServerJoin{}},
+	Client: &ClientConfig{
+		ServerJoin: &ServerJoin{},
+	},
 	Server: &ServerConfig{
 		Enabled:         true,
 		BootstrapExpect: 3,

command/agent/testdata/basic.hcl

@@ -75,8 +75,12 @@ client {
     reserved_ports = "1,100,10-12"
   }
 
-  client_min_port  = 1000
-  client_max_port  = 2000
+  client_min_port = 1000
+  client_max_port = 2000
+
+  dynamic_port_range_min = 3000
+  dynamic_port_range_max = 4000
+
   max_kill_timeout = "10s"
 
   stats {

command/agent/testdata/basic.json

@@ -77,6 +77,8 @@
       "cni_path": "/tmp/cni_path",
       "cpu_total_compute": 4444,
       "disable_remote_exec": true,
+      "dynamic_port_range_min": 3000,
+      "dynamic_port_range_max": 4000,
       "enabled": true,
       "gc_disk_usage_threshold": 82,
       "gc_inode_usage_threshold": 91,

nomad/structs/network.go

@@ -7,13 +7,38 @@ import (
 	"sync"
 )
 
-const (
-	// MinDynamicPort is the smallest dynamic port generated
-	MinDynamicPort = 20000
+// PortRange describes the boundaries of a contiguous port range
+type PortRange struct {
+	Min int
+	Max int
+}
+
+var (
+	// default values
+	dynamicPortRangeMin = 20000
+	dynamicPortRangeMax = 32000
+
+	// globally configurable dynamic port range
+	dynamicPortRangeLock sync.Mutex
+	dynamicPortRange     = PortRange{
+		Min: dynamicPortRangeMin,
+		Max: dynamicPortRangeMax,
+	}
+)
 
-	// MaxDynamicPort is the largest dynamic port generated
-	MaxDynamicPort = 32000
+// GetDynamicPortRange returns the globally defined dynamic port range (default: 20000-32000).
+func GetDynamicPortRange() PortRange {
+	return dynamicPortRange
+}
 
+// SetDynamicPortRange reconfigures the dynamic port range.
+func SetDynamicPortRange(p PortRange) {
+	dynamicPortRangeLock.Lock()
+	defer dynamicPortRangeLock.Unlock()
+	dynamicPortRange = p
+}
+
+const (
 	// maxRandPortAttempts is the maximum number of attempt
 	// to assign a random port
 	maxRandPortAttempts = 20
@@ -505,7 +530,8 @@ func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int
 	}
 
 	// Get the indexes of the unset
-	availablePorts := usedSet.IndexesInRange(false, MinDynamicPort, MaxDynamicPort)
+	dynamicPortRange := GetDynamicPortRange()
+	availablePorts := usedSet.IndexesInRange(false, uint(dynamicPortRange.Min), uint(dynamicPortRange.Max))
 
 	// Randomize the amount we need
 	if len(availablePorts) < numDyn {
@@ -540,7 +566,8 @@ func getDynamicPortsStochastic(nodeUsed Bitmap, reservedPorts []Port, count int)
 			return nil, fmt.Errorf("stochastic dynamic port selection failed")
 		}
 
-		randPort := MinDynamicPort + rand.Intn(MaxDynamicPort-MinDynamicPort)
+		dynamicPortRange := GetDynamicPortRange()
+		randPort := dynamicPortRange.Min + rand.Intn(dynamicPortRange.Max-dynamicPortRange.Min)
 		if nodeUsed != nil && nodeUsed.Check(uint(randPort)) {
 			goto PICK
 		}

nomad/structs/network_test.go

@@ -307,7 +307,7 @@ func TestNetworkIndex_AssignNetwork(t *testing.T) {
 // This test ensures that even with a small domain of available ports we are
 // able to make a dynamic port allocation.
 func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {
-
+	dynamicPortRange := GetDynamicPortRange()
 	// Create a node that only has one free port
 	idx := NewNetworkIndex()
 	n := &Node{
@@ -323,7 +323,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {
 		},
 		ReservedResources: &NodeReservedResources{
 			Networks: NodeReservedNetworkResources{
-				ReservedHostPorts: fmt.Sprintf("%d-%d", MinDynamicPort, MaxDynamicPort-1),
+				ReservedHostPorts: fmt.Sprintf("%d-%d", dynamicPortRange.Min, dynamicPortRange.Max-1),
 			},
 		},
 	}
@@ -346,8 +346,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {
 	if len(offer.DynamicPorts) != 1 {
 		t.Fatalf("There should be one dynamic ports")
 	}
-	if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort {
-		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
+	if p := offer.DynamicPorts[0].Value; p != dynamicPortRange.Max {
+		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, dynamicPortRange.Max)
 	}
 }
 
@@ -623,6 +623,7 @@ func TestNetworkIndex_AssignNetwork_Old(t *testing.T) {
 // This test ensures that even with a small domain of available ports we are
 // able to make a dynamic port allocation.
 func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) {
+	dynamicPortRange := GetDynamicPortRange()
 
 	// Create a node that only has one free port
 	idx := NewNetworkIndex()
@@ -646,7 +647,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) {
 			},
 		},
 	}
-	for i := MinDynamicPort; i < MaxDynamicPort; i++ {
+	for i := dynamicPortRange.Min; i < dynamicPortRange.Max; i++ {
 		n.Reserved.Networks[0].ReservedPorts = append(n.Reserved.Networks[0].ReservedPorts, Port{Value: i})
 	}
 
@@ -669,8 +670,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) {
 	if len(offer.DynamicPorts) != 1 {
 		t.Fatalf("There should be three dynamic ports")
 	}
-	if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort {
-		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
+	if p := offer.DynamicPorts[0].Value; p != dynamicPortRange.Max {
+		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, dynamicPortRange.Max)
 	}
 }
 
@@ -686,3 +687,24 @@ func TestIntContains(t *testing.T) {
 		t.Fatalf("bad")
 	}
 }
+
+func TestDynamicPortRange(t *testing.T) {
+	dynamicPortRange := GetDynamicPortRange()
+	require.Equal(t, dynamicPortRangeMin, dynamicPortRange.Min, "min dynamic port range does not match default value")
+	require.Equal(t, dynamicPortRangeMax, dynamicPortRange.Max, "max dynamic port range does not match default value")
+
+	SetDynamicPortRange(PortRange{
+		Min: 1000,
+		Max: 2000,
+	})
+
+	dynamicPortRange = GetDynamicPortRange()
+	require.Equal(t, 1000, dynamicPortRange.Min, "should reconfigure min dynamic port range")
+	require.Equal(t, 2000, dynamicPortRange.Max, "should reconfigure max dynamic port range")
+
+	// restore to defaults
+	defer SetDynamicPortRange(PortRange{
+		Min: dynamicPortRangeMin,
+		Max: dynamicPortRangeMax,
+	})
+}