Enabling --non-masquerade-cidr option (#1361)

* add kubernetes Non Masquerade CIDR in order to be able to install K8s in a network that has already used the 10.0.0.0/8, and therefore activated NAT of a pod to an address in 10.0.0.0/8 * gofmted * rename CIDR to Cidr according to the exciting code * fix kubernetesNonMasqueradeCidr to KUBELET_NON_MASQUERADE_CIDR on master * clean * Review change * change CIDR to Cidr * json lint * added NonMasqueradeCidr validation * documentation
Azure · Oct 9, 2017 · 8281069 · 8281069
1 parent b86448b
commit 8281069
Show file tree

Hide file tree

Showing 15 changed files with 41 additions and 0 deletions.
diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md
@@ -36,6 +36,7 @@ Here are the valid values for the orchestrator types:
 |dnsServiceIP|no|IP address for kube-dns to listen on. If specified must be in the range of `serviceCidr`.|
 |dockerBridgeSubnet|no|The specific IP and subnet used for allocating IP addresses for the docker bridge network created on the kubernetes master and agents. Default value is 172.17.0.1/16. This value is used to configure the docker daemon using the [--bip flag](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0).|
 |serviceCidr|no|IP range for Service IPs, Default is "10.0.0.0/16". This range is never routed outside of a node so does not need to lie within clusterSubnet or the VNet.|
+|nonMasqueradeCidr|no|CIDR block to exclude from default source NAT, Default is "10.0.0.0/8".|
 |enableRbac|no|Enable [Kubernetes RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) (boolean - default == false) |
 |maxPods|no|The maximum number of pods per node. The minimum valid value, necessary for running kube-system pods, is 5. Default value is 30 when networkPolicy equals azure, 110 otherwise.|
 |gcHighThreshold|no|Sets the --image-gc-high-threshold value on the kublet configuration. Default is 85. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) |

diff --git a/parts/kubernetesagentcustomdata.yml b/parts/kubernetesagentcustomdata.yml
@@ -117,6 +117,7 @@ write_files:
     KUBELET_IMAGE_GC_HIGH_THRESHOLD={{WrapAsVariable "gchighthreshold"}}
     KUBELET_IMAGE_GC_LOW_THRESHOLD={{WrapAsVariable "gclowthreshold"}}
 {{if IsKubernetesVersionGe "1.6.0"}}
+    KUBELET_NON_MASQUERADE_CIDR={{WrapAsVariable "kubernetesNonMasqueradeCidr"}}
     KUBELET_FEATURE_GATES=--feature-gates=Accelerators=true
 {{end}}
 

diff --git a/parts/kuberneteskubelet.service b/parts/kuberneteskubelet.service
@@ -50,6 +50,7 @@ ExecStart=/usr/bin/docker run \
         --node-status-update-frequency=${KUBELET_NODE_STATUS_UPDATE_FREQUENCY} \
         --image-gc-high-threshold=${KUBELET_IMAGE_GC_HIGH_THRESHOLD} \
         --image-gc-low-threshold=${KUBELET_IMAGE_GC_LOW_THRESHOLD} \
+        --non-masquerade-cidr=${KUBELET_NON_MASQUERADE_CIDR} \
         --v=2 ${KUBELET_FEATURE_GATES} \
         ${KUBELET_REGISTER_NODE} ${KUBELET_REGISTER_WITH_TAINTS}
 

diff --git a/parts/kubernetesmastercustomdata.yml b/parts/kubernetesmastercustomdata.yml
@@ -209,6 +209,7 @@ write_files:
     KUBELET_IMAGE_GC_LOW_THRESHOLD={{WrapAsVariable "gclowthreshold"}}
 {{if IsKubernetesVersionGe "1.6.0"}}
   {{if HasLinuxAgents}}
+    KUBELET_NON_MASQUERADE_CIDR={{WrapAsVariable "kubernetesNonMasqueradeCidr"}}
     KUBELET_REGISTER_NODE=--register-node=true
     KUBELET_REGISTER_WITH_TAINTS=--register-with-taints={{WrapAsVariable "registerWithTaints"}}
   {{end}}

diff --git a/parts/kubernetesmastervars.t b/parts/kubernetesmastervars.t
@@ -37,6 +37,7 @@
     "cniPluginsURL":"[parameters('cniPluginsURL')]",
     "vnetCniLinuxPluginsURL":"[parameters('vnetCniLinuxPluginsURL')]",
     "vnetCniWindowsPluginsURL":"[parameters('vnetCniWindowsPluginsURL')]",
+    "kubernetesNonMasqueradeCidr": "[parameters('kubernetesNonMasqueradeCidr')]",
     "maxPods": "[parameters('maxPods')]",
     "vnetCidr": "[parameters('vnetCidr')]",
     "gcHighThreshold":"[parameters('gcHighThreshold')]",

diff --git a/parts/kubernetesparams.t b/parts/kubernetesparams.t
@@ -98,6 +98,12 @@
       },
       "type": "string"
     },
+    "kubernetesNonMasqueradeCidr": {
+      "metadata": {
+        "description": "kubernetesNonMasqueradeCidr cluster subnet"
+      },
+      "type": "string"
+    },
     "kubernetesHyperkubeSpec": {
       {{PopulateClassicModeDefaultValue "kubernetesHyperkubeSpec"}}
       "metadata": {

diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go
@@ -21,6 +21,8 @@ const (
 	DefaultKubernetesClusterSubnet = "10.244.0.0/16"
 	// DefaultDockerBridgeSubnet specifies the default subnet for the docker bridge network for masters and agents.
 	DefaultDockerBridgeSubnet = "172.17.0.1/16"
+	// DefaultNonMasqueradeCidr specifies the subnet that should not be masqueraded on host
+	DefaultNonMasqueradeCidr = "10.0.0.0/8"
 	// DefaultFirstConsecutiveKubernetesStaticIP specifies the static IP address on Kubernetes master 0
 	DefaultFirstConsecutiveKubernetesStaticIP = "10.240.255.5"
 	// DefaultAgentSubnetTemplate specifies a default agent subnet

diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go
@@ -205,6 +205,9 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
 		if a.OrchestratorProfile.KubernetesConfig.ServiceCIDR == "" {
 			a.OrchestratorProfile.KubernetesConfig.ServiceCIDR = DefaultKubernetesServiceCIDR
 		}
+		if a.OrchestratorProfile.KubernetesConfig.NonMasqueradeCidr == "" {
+			a.OrchestratorProfile.KubernetesConfig.NonMasqueradeCidr = DefaultNonMasqueradeCidr
+		}
 		if a.OrchestratorProfile.KubernetesConfig.NodeStatusUpdateFrequency == "" {
 			a.OrchestratorProfile.KubernetesConfig.NodeStatusUpdateFrequency = KubeConfigs[k8sVersion]["nodestatusfreq"]
 		}

diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go
@@ -558,6 +558,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool) (paramsMap, err
 		addValue(parametersMap, "cloudProviderRatelimitQPS", strconv.FormatFloat(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitQPS, 'f', -1, 64))
 		addValue(parametersMap, "cloudProviderRatelimitBucket", strconv.Itoa(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitBucket))
 		addValue(parametersMap, "kubeClusterCidr", properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet)
+		addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.OrchestratorProfile.KubernetesConfig.NonMasqueradeCidr)
 		addValue(parametersMap, "dockerBridgeCidr", properties.OrchestratorProfile.KubernetesConfig.DockerBridgeSubnet)
 		addValue(parametersMap, "networkPolicy", properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy)
 		addValue(parametersMap, "cniPluginsURL", cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL)

diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go
@@ -646,6 +646,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
 	vlabs.ClusterSubnet = api.ClusterSubnet
 	vlabs.DNSServiceIP = api.DNSServiceIP
 	vlabs.ServiceCidr = api.ServiceCIDR
+	vlabs.NonMasqueradeCidr = api.NonMasqueradeCidr
 	vlabs.NetworkPolicy = api.NetworkPolicy
 	vlabs.MaxPods = api.MaxPods
 	vlabs.DockerBridgeSubnet = api.DockerBridgeSubnet

diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go
@@ -608,6 +608,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	api.ClusterSubnet = vlabs.ClusterSubnet
 	api.DNSServiceIP = vlabs.DNSServiceIP
 	api.ServiceCIDR = vlabs.ServiceCidr
+	api.NonMasqueradeCidr = vlabs.NonMasqueradeCidr
 	api.NetworkPolicy = vlabs.NetworkPolicy
 	api.MaxPods = vlabs.MaxPods
 	api.DockerBridgeSubnet = vlabs.DockerBridgeSubnet

diff --git a/pkg/api/types.go b/pkg/api/types.go
@@ -160,6 +160,7 @@ type OrchestratorVersionProfile struct {
 type KubernetesConfig struct {
 	KubernetesImageBase              string  `json:"kubernetesImageBase,omitempty"`
 	ClusterSubnet                    string  `json:"clusterSubnet,omitempty"`
+	NonMasqueradeCidr                string  `json:"nonMasqueradeCidr,omitempty"`
 	NetworkPolicy                    string  `json:"networkPolicy,omitempty"`
 	MaxPods                          int     `json:"maxPods,omitempty"`
 	DockerBridgeSubnet               string  `json:"dockerBridgeSubnet,omitempty"`

diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go
@@ -182,6 +182,7 @@ type KubernetesConfig struct {
 	DNSServiceIP                     string  `json:"dnsServiceIP,omitempty"`
 	ServiceCidr                      string  `json:"serviceCidr,omitempty"`
 	NetworkPolicy                    string  `json:"networkPolicy,omitempty"`
+	NonMasqueradeCidr                string  `json:"NonMasqueradeCidr,omitempty"`
 	MaxPods                          int     `json:"maxPods,omitempty"`
 	DockerBridgeSubnet               string  `json:"DockerBridgeSubnet,omitempty"`
 	NodeStatusUpdateFrequency        string  `json:"nodeStatusUpdateFrequency,omitempty"`

diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go
@@ -397,6 +397,12 @@ func (a *KubernetesConfig) Validate(k8sVersion string) error {
 		}
 	}
 
+	if a.NonMasqueradeCidr != "" {
+		if _, _, err := net.ParseCIDR(a.NonMasqueradeCidr); err != nil {
+			return fmt.Errorf("OrchestratorProfile.KubernetesConfig.NonMasqueradeCidr '%s' is an invalid CIDR string", a.NonMasqueradeCidr)
+		}
+	}
+
 	if a.MaxPods != 0 {
 		if a.MaxPods < KubernetesMinMaxPods {
 			return fmt.Errorf("OrchestratorProfile.KubernetesConfig.MaxPods '%v' must be at least %v", a.MaxPods, KubernetesMinMaxPods)

diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go
@@ -94,6 +94,20 @@ func Test_KubernetesConfig_Validate(t *testing.T) {
 			t.Error("should error on invalid DockerBridgeSubnet")
 		}
 
+		c = KubernetesConfig{
+			NonMasqueradeCidr: "10.120.1.0/24",
+		}
+		if err := c.Validate(k8sRelease); err != nil {
+			t.Error("should not error on valid NonMasqueradeCidr")
+		}
+
+		c = KubernetesConfig{
+			NonMasqueradeCidr: "10.120.1.0/invalid",
+		}
+		if err := c.Validate(k8sRelease); err == nil {
+			t.Error("should error on invalid NonMasqueradeCidr")
+		}
+
 		c = KubernetesConfig{
 			MaxPods: KubernetesMinMaxPods - 1,
 		}