From 8281069307e029abbb134222f440646c02100e71 Mon Sep 17 00:00:00 2001 From: David Bourasseau Date: Mon, 9 Oct 2017 20:09:54 +0200 Subject: [PATCH] Enabling --non-masquerade-cidr option (#1361) * add kubernetes Non Masquerade CIDR in order to be able to install K8s in a network that has already used the 10.0.0.0/8, and therefore activated NAT of a pod to an address in 10.0.0.0/8 * gofmted * rename CIDR to Cidr according to the exciting code * fix kubernetesNonMasqueradeCidr to KUBELET_NON_MASQUERADE_CIDR on master * clean * Review change * change CIDR to Cidr * json lint * added NonMasqueradeCidr validation * documentation --- docs/clusterdefinition.md | 1 + parts/kubernetesagentcustomdata.yml | 1 + parts/kuberneteskubelet.service | 1 + parts/kubernetesmastercustomdata.yml | 1 + parts/kubernetesmastervars.t | 1 + parts/kubernetesparams.t | 6 ++++++ pkg/acsengine/const.go | 2 ++ pkg/acsengine/defaults.go | 3 +++ pkg/acsengine/engine.go | 1 + pkg/api/converterfromapi.go | 1 + pkg/api/convertertoapi.go | 1 + pkg/api/types.go | 1 + pkg/api/vlabs/types.go | 1 + pkg/api/vlabs/validate.go | 6 ++++++ pkg/api/vlabs/validate_test.go | 14 ++++++++++++++ 15 files changed, 41 insertions(+) diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 715ddaf9d6..7cc3e880ff 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -36,6 +36,7 @@ Here are the valid values for the orchestrator types: |dnsServiceIP|no|IP address for kube-dns to listen on. If specified must be in the range of `serviceCidr`.| |dockerBridgeSubnet|no|The specific IP and subnet used for allocating IP addresses for the docker bridge network created on the kubernetes master and agents. Default value is 172.17.0.1/16. This value is used to configure the docker daemon using the [--bip flag](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0).| |serviceCidr|no|IP range for Service IPs, Default is "10.0.0.0/16". This range is never routed outside of a node so does not need to lie within clusterSubnet or the VNet.| +|nonMasqueradeCidr|no|CIDR block to exclude from default source NAT, Default is "10.0.0.0/8".| |enableRbac|no|Enable [Kubernetes RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) (boolean - default == false) | |maxPods|no|The maximum number of pods per node. The minimum valid value, necessary for running kube-system pods, is 5. Default value is 30 when networkPolicy equals azure, 110 otherwise.| |gcHighThreshold|no|Sets the --image-gc-high-threshold value on the kublet configuration. Default is 85. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) | diff --git a/parts/kubernetesagentcustomdata.yml b/parts/kubernetesagentcustomdata.yml index 1515d95eaf..d30ff109b0 100644 --- a/parts/kubernetesagentcustomdata.yml +++ b/parts/kubernetesagentcustomdata.yml @@ -117,6 +117,7 @@ write_files: KUBELET_IMAGE_GC_HIGH_THRESHOLD={{WrapAsVariable "gchighthreshold"}} KUBELET_IMAGE_GC_LOW_THRESHOLD={{WrapAsVariable "gclowthreshold"}} {{if IsKubernetesVersionGe "1.6.0"}} + KUBELET_NON_MASQUERADE_CIDR={{WrapAsVariable "kubernetesNonMasqueradeCidr"}} KUBELET_FEATURE_GATES=--feature-gates=Accelerators=true {{end}} diff --git a/parts/kuberneteskubelet.service b/parts/kuberneteskubelet.service index 69e080f9d2..7370379bb3 100644 --- a/parts/kuberneteskubelet.service +++ b/parts/kuberneteskubelet.service @@ -50,6 +50,7 @@ ExecStart=/usr/bin/docker run \ --node-status-update-frequency=${KUBELET_NODE_STATUS_UPDATE_FREQUENCY} \ --image-gc-high-threshold=${KUBELET_IMAGE_GC_HIGH_THRESHOLD} \ --image-gc-low-threshold=${KUBELET_IMAGE_GC_LOW_THRESHOLD} \ + --non-masquerade-cidr=${KUBELET_NON_MASQUERADE_CIDR} \ --v=2 ${KUBELET_FEATURE_GATES} \ ${KUBELET_REGISTER_NODE} ${KUBELET_REGISTER_WITH_TAINTS} diff --git a/parts/kubernetesmastercustomdata.yml b/parts/kubernetesmastercustomdata.yml index 708c974607..7628eeda2d 100644 --- a/parts/kubernetesmastercustomdata.yml +++ b/parts/kubernetesmastercustomdata.yml @@ -209,6 +209,7 @@ write_files: KUBELET_IMAGE_GC_LOW_THRESHOLD={{WrapAsVariable "gclowthreshold"}} {{if IsKubernetesVersionGe "1.6.0"}} {{if HasLinuxAgents}} + KUBELET_NON_MASQUERADE_CIDR={{WrapAsVariable "kubernetesNonMasqueradeCidr"}} KUBELET_REGISTER_NODE=--register-node=true KUBELET_REGISTER_WITH_TAINTS=--register-with-taints={{WrapAsVariable "registerWithTaints"}} {{end}} diff --git a/parts/kubernetesmastervars.t b/parts/kubernetesmastervars.t index fbc6ad148d..e83bfa198b 100644 --- a/parts/kubernetesmastervars.t +++ b/parts/kubernetesmastervars.t @@ -37,6 +37,7 @@ "cniPluginsURL":"[parameters('cniPluginsURL')]", "vnetCniLinuxPluginsURL":"[parameters('vnetCniLinuxPluginsURL')]", "vnetCniWindowsPluginsURL":"[parameters('vnetCniWindowsPluginsURL')]", + "kubernetesNonMasqueradeCidr": "[parameters('kubernetesNonMasqueradeCidr')]", "maxPods": "[parameters('maxPods')]", "vnetCidr": "[parameters('vnetCidr')]", "gcHighThreshold":"[parameters('gcHighThreshold')]", diff --git a/parts/kubernetesparams.t b/parts/kubernetesparams.t index a81294bc01..ecef7fc506 100644 --- a/parts/kubernetesparams.t +++ b/parts/kubernetesparams.t @@ -98,6 +98,12 @@ }, "type": "string" }, + "kubernetesNonMasqueradeCidr": { + "metadata": { + "description": "kubernetesNonMasqueradeCidr cluster subnet" + }, + "type": "string" + }, "kubernetesHyperkubeSpec": { {{PopulateClassicModeDefaultValue "kubernetesHyperkubeSpec"}} "metadata": { diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 722a2b2b0e..d172fd4e9a 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -21,6 +21,8 @@ const ( DefaultKubernetesClusterSubnet = "10.244.0.0/16" // DefaultDockerBridgeSubnet specifies the default subnet for the docker bridge network for masters and agents. DefaultDockerBridgeSubnet = "172.17.0.1/16" + // DefaultNonMasqueradeCidr specifies the subnet that should not be masqueraded on host + DefaultNonMasqueradeCidr = "10.0.0.0/8" // DefaultFirstConsecutiveKubernetesStaticIP specifies the static IP address on Kubernetes master 0 DefaultFirstConsecutiveKubernetesStaticIP = "10.240.255.5" // DefaultAgentSubnetTemplate specifies a default agent subnet diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 7b3416549a..5fe920af63 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -205,6 +205,9 @@ func setOrchestratorDefaults(cs *api.ContainerService) { if a.OrchestratorProfile.KubernetesConfig.ServiceCIDR == "" { a.OrchestratorProfile.KubernetesConfig.ServiceCIDR = DefaultKubernetesServiceCIDR } + if a.OrchestratorProfile.KubernetesConfig.NonMasqueradeCidr == "" { + a.OrchestratorProfile.KubernetesConfig.NonMasqueradeCidr = DefaultNonMasqueradeCidr + } if a.OrchestratorProfile.KubernetesConfig.NodeStatusUpdateFrequency == "" { a.OrchestratorProfile.KubernetesConfig.NodeStatusUpdateFrequency = KubeConfigs[k8sVersion]["nodestatusfreq"] } diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 2e9668ddcd..3bcef452f6 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -558,6 +558,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool) (paramsMap, err addValue(parametersMap, "cloudProviderRatelimitQPS", strconv.FormatFloat(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitQPS, 'f', -1, 64)) addValue(parametersMap, "cloudProviderRatelimitBucket", strconv.Itoa(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitBucket)) addValue(parametersMap, "kubeClusterCidr", properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet) + addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.OrchestratorProfile.KubernetesConfig.NonMasqueradeCidr) addValue(parametersMap, "dockerBridgeCidr", properties.OrchestratorProfile.KubernetesConfig.DockerBridgeSubnet) addValue(parametersMap, "networkPolicy", properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy) addValue(parametersMap, "cniPluginsURL", cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL) diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 93e8eacfb6..6f6b020bc8 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -646,6 +646,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet vlabs.ClusterSubnet = api.ClusterSubnet vlabs.DNSServiceIP = api.DNSServiceIP vlabs.ServiceCidr = api.ServiceCIDR + vlabs.NonMasqueradeCidr = api.NonMasqueradeCidr vlabs.NetworkPolicy = api.NetworkPolicy vlabs.MaxPods = api.MaxPods vlabs.DockerBridgeSubnet = api.DockerBridgeSubnet diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 9a6a8c625a..fd61245bab 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -608,6 +608,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.ClusterSubnet = vlabs.ClusterSubnet api.DNSServiceIP = vlabs.DNSServiceIP api.ServiceCIDR = vlabs.ServiceCidr + api.NonMasqueradeCidr = vlabs.NonMasqueradeCidr api.NetworkPolicy = vlabs.NetworkPolicy api.MaxPods = vlabs.MaxPods api.DockerBridgeSubnet = vlabs.DockerBridgeSubnet diff --git a/pkg/api/types.go b/pkg/api/types.go index 9ec36088fd..1d9b1dc6ac 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -160,6 +160,7 @@ type OrchestratorVersionProfile struct { type KubernetesConfig struct { KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` ClusterSubnet string `json:"clusterSubnet,omitempty"` + NonMasqueradeCidr string `json:"nonMasqueradeCidr,omitempty"` NetworkPolicy string `json:"networkPolicy,omitempty"` MaxPods int `json:"maxPods,omitempty"` DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 61ade59f59..00a069f63c 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -182,6 +182,7 @@ type KubernetesConfig struct { DNSServiceIP string `json:"dnsServiceIP,omitempty"` ServiceCidr string `json:"serviceCidr,omitempty"` NetworkPolicy string `json:"networkPolicy,omitempty"` + NonMasqueradeCidr string `json:"NonMasqueradeCidr,omitempty"` MaxPods int `json:"maxPods,omitempty"` DockerBridgeSubnet string `json:"DockerBridgeSubnet,omitempty"` NodeStatusUpdateFrequency string `json:"nodeStatusUpdateFrequency,omitempty"` diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index acb8264467..1ba72fcbbd 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -397,6 +397,12 @@ func (a *KubernetesConfig) Validate(k8sVersion string) error { } } + if a.NonMasqueradeCidr != "" { + if _, _, err := net.ParseCIDR(a.NonMasqueradeCidr); err != nil { + return fmt.Errorf("OrchestratorProfile.KubernetesConfig.NonMasqueradeCidr '%s' is an invalid CIDR string", a.NonMasqueradeCidr) + } + } + if a.MaxPods != 0 { if a.MaxPods < KubernetesMinMaxPods { return fmt.Errorf("OrchestratorProfile.KubernetesConfig.MaxPods '%v' must be at least %v", a.MaxPods, KubernetesMinMaxPods) diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index 3b21c37d61..73dfbb55e5 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -94,6 +94,20 @@ func Test_KubernetesConfig_Validate(t *testing.T) { t.Error("should error on invalid DockerBridgeSubnet") } + c = KubernetesConfig{ + NonMasqueradeCidr: "10.120.1.0/24", + } + if err := c.Validate(k8sRelease); err != nil { + t.Error("should not error on valid NonMasqueradeCidr") + } + + c = KubernetesConfig{ + NonMasqueradeCidr: "10.120.1.0/invalid", + } + if err := c.Validate(k8sRelease); err == nil { + t.Error("should error on invalid NonMasqueradeCidr") + } + c = KubernetesConfig{ MaxPods: KubernetesMinMaxPods - 1, }