From 32058619b619967149007ef0aec91f0524e7669a Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Fri, 23 Aug 2019 13:49:57 +0200 Subject: [PATCH] WIP kindnet 0.6 --- cmd/kindnetd/cni.go | 28 ++++-- cmd/kindnetd/main.go | 40 +++++++-- cmd/kindnetd/subnets.go | 110 +++++++++++------------ hack/ci/e2e.sh | 5 -- pkg/build/node/cni.go | 7 +- pkg/cluster/config/v1alpha3/types.go | 126 +++++++++++++++++++++++++++ 6 files changed, 238 insertions(+), 78 deletions(-) create mode 100644 pkg/cluster/config/v1alpha3/types.go diff --git a/cmd/kindnetd/cni.go b/cmd/kindnetd/cni.go index b06cc22e66..319fe334f5 100644 --- a/cmd/kindnetd/cni.go +++ b/cmd/kindnetd/cni.go @@ -32,20 +32,32 @@ import ( // CNIConfigInputs is supplied to the CNI config template type CNIConfigInputs struct { - PodCIDR string - DefaultRoute string + PodCIDRs []string + DefaultRoutes []string } // ComputeCNIConfigInputs computes the template inputs for CNIConfigWriter func ComputeCNIConfigInputs(node corev1.Node) CNIConfigInputs { - podCIDR := node.Spec.PodCIDR - defaultRoute := "0.0.0.0/0" - if net.IsIPv6CIDRString(podCIDR) { - defaultRoute = "::/0" + + defaultRoutes := []string{"0.0.0.0/0", "::/0"} + // check if is a dual-stack cluster + if len(node.Spec.PodCIDRs) > 1 { + return CNIConfigInputs{ + PodCIDRs: node.Spec.PodCIDRs, + DefaultRoutes: defaultRoutes, + } + } + // the cluster is single stack + // we use the legacy node.Spec.PodCIDR for backwards compatibility + podCIDRs := []string{"node.Spec.PodCIDR"} + // This is a single stack cluster + defaultRoute := defaultRoutes[:1] + if net.IsIPv6CIDRString(podCIDRs[0]) { + defaultRoute := defaultRoutes[1:] } return CNIConfigInputs{ - PodCIDR: podCIDR, - DefaultRoute: defaultRoute, + PodCIDRs: podCIDRs, + DefaultRoutes: defaultRoute, } } diff --git a/cmd/kindnetd/main.go b/cmd/kindnetd/main.go index 18fd141cb3..8d924cfdd9 100644 --- a/cmd/kindnetd/main.go +++ b/cmd/kindnetd/main.go @@ -36,11 +36,25 @@ import ( // input envs: // - HOST_IP: hould be populated by downward API // - POD_IP: should be populated by downward API +// - POD_IPS: should be populated by downward API (only with dual-stack clusters) // - CNI_CONFIG_TEMPLATE: the cni .conflist template, run with {{ .PodCIDR }} // TODO: improve logging & error handling +// IPFamily defines kindnet networking operating model +type IPFamily string + +const ( + // IPv4Family sets IPFamily to ipv4 + IPv4Family IPFamily = "ipv4" + // IPv6Family sets IPFamily to ipv6 + IPv6Family IPFamily = "ipv6" + // DualStackFamily sets ClusterIPFamily to dual-stack + DualStackFamily IPFamily = "dual-stack" +) + func main() { + // create a Kubernetes client config, err := rest.InClusterConfig() if err != nil { @@ -52,8 +66,8 @@ func main() { } // obtain the host and pod ip addresses - // if both ips are different we are not using the host network hostIP, podIP := os.Getenv("HOST_IP"), os.Getenv("POD_IP") + // if both ips are different we are not using the host network fmt.Printf("hostIP = %s\npodIP = %s\n", hostIP, podIP) if hostIP != podIP { panic(fmt.Sprintf( @@ -61,6 +75,17 @@ func main() { hostIP, podIP, )) } + // obtain all pod ip addresses if they exist + podIPs := strings.Split(os.Getenv("POD_IPS"),",") + // detect the cluster IP family + if len(podIPs) > 1 && net.IsDualStackIPStrings(podIPs) { + ipFamily := DualStackFamily + } else if net.IsIPv6String(podIP) { + ipFamily := IPv6Family + } else { + ipFamily := IPv4Family + } + // used to track if the cni config inputs changed and write the config cniConfigWriter := &CNIConfigWriter{ @@ -68,9 +93,9 @@ func main() { } // enforce ip masquerade rules - // TODO: dual stack...? - masqAgent, _ := NewIPMasqAgent( - net.IsIPv6String(hostIP), + noMasqSubnets := getNoMasqueradeSubnets(clientset) + masqAgentIPv4, _ := NewIPMasqAgent( + false, getNoMasqueradeSubnets(clientset), ) go func() { @@ -155,11 +180,12 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1 } // internalIP returns the internalIP address for node -func internalIP(node corev1.Node) string { +func internalIP(node corev1.Node) []string] { + var ips []string for _, address := range node.Status.Addresses { if address.Type == "InternalIP" { - return address.Address + ips = append(ips, address.Address) } } - return "" + return ips } diff --git a/cmd/kindnetd/subnets.go b/cmd/kindnetd/subnets.go index 236e532649..582aa80612 100644 --- a/cmd/kindnetd/subnets.go +++ b/cmd/kindnetd/subnets.go @@ -18,87 +18,87 @@ package main import ( "os" + "strings" "regexp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" +) + +const ( + kubeadmClusterCIDRRegex = `\s*podSubnet: (.*)\n` + kubeproxyClusterCIDRRegex = `\s*clusterCIDR: (.*)\n` ) -type subnets struct { - pods sets.String - services sets.String +type kubeSubnets struct { + regex string + configmap string + namespace string } -func newSubnets() *subnets { - return &subnets{ - pods: sets.NewString(), - services: sets.NewString(), +func newKubeSubnets(regex, configmap, namespace string) *kubeSubnets { + return &kubeSubnets{ + regex: regex + configmap: configmap + namespace: namespace } -} -func (s *subnets) All() []string { - return s.pods.Union(s.services).List() } +func (ks *kubeSubnets) Get(clientset *kubernetes.Clientset) (string, error) { + cidrRegexp:= regexp.MustCompile(ks.regex) + configMap, err := clientset.CoreV1().ConfigMaps(ks.namespace).Get(ks.configmap, metav1.GetOptions{}) + if err != nil { + println("ERROR: " + err.Error()) + return "", err + } + for _, data := range configMap.Data { + matches = cidrRegexp.FindStringSubmatch(data) + if len(matches) > 0 { + println("MATCH POD: " + matches[1]) + return matches[1], nil + } + } + return "", nil +} +// getNoMasqueradeSubnets tries to get the POD networks subnets to not Masquerade them +// It returns an array of strings with the Cluster CIDR subnets +// It can only obtain the POD subnet parameter from one place for consistency +// The order is: +// 1. POD_SUBNET environment variables +// 2. Pod subnet value in kubeadm configmap +// 3. Cluster CIDR value in kube-proxy configmap func getNoMasqueradeSubnets(clientset *kubernetes.Clientset) []string { - s := newSubnets() - // check for environment variables (legacy) podSubnetEnv := os.Getenv("POD_SUBNET") if podSubnetEnv != "" { - s.pods.Insert(podSubnetEnv) - } - serviceSubnetEnv := os.Getenv("SERVICE_SUBNET") - if serviceSubnetEnv != "" { - s.services.Insert(serviceSubnetEnv) + return strings.Split(podSubnetEnv, ",") } - // try getting from kubeadm config - kubeadmSubnets := getKubeadmSubnets(clientset) - s.pods = s.pods.Union(kubeadmSubnets.pods) - s.services = s.services.Union(kubeadmSubnets.services) + // try getting from kubeadm configmap + kubeadmSubnets := newKubeSubnets(kubeadmClusterCIDRRegex,"kubeadm-config", "kube-system") + podSubnetKubeadm := kubeadmSubnets.Get(clientset) + if podSubnetKubeadm != "" { + return strings.Split(podSubnetKubeadm, ",") + } + // try getting from kubeproxy configmap + kubeproxySubnets := newKubeSubnets(kubeproxyClusterCIDRRegex,"kube-proxy", "kube-system") + podSubnetKubeproxy := kubeproxySubnets.Get(clientset) + if podSubnetKubeproxy != "" { + return strings.Split(podSubnetKubeproxy, ",") + + } + // TODO: we have other fallback options for completeness including: // - kube-apiserver flags // - component config // - defaults - - return s.All() + return nil } -const ( - kubeadmServiceCIDRRegex = `\s*serviceSubnet: (.*)\n` - kubeadmClusterCIDRRegex = `\s*podSubnet: (.*)\n` -) -var ( - kubeadmServiceCIDRRegexp = regexp.MustCompile(kubeadmServiceCIDRRegex) - kubeadmClusterCIDRRegexp = regexp.MustCompile(kubeadmClusterCIDRRegex) -) - -func getKubeadmSubnets(clientset *kubernetes.Clientset) *subnets { - s := newSubnets() - configMap, err := clientset.CoreV1().ConfigMaps("kube-system").Get("kubeadm-config", metav1.GetOptions{}) - if err != nil { - println("ERROR: "+err.Error()) - return s - } - for _, data := range configMap.Data { - matches := kubeadmServiceCIDRRegexp.FindAllStringSubmatch(data, 1) - if len(matches) > 0 { - println("MATCH SERVICE: " + matches[0][1]) - s.services.Insert(matches[0][1]) - } - matches = kubeadmClusterCIDRRegexp.FindAllStringSubmatch(data, 1) - if len(matches) > 0 { - println("MATCH POD: " + matches[0][1]) - s.pods.Insert(matches[0][1]) - } - } - return s -} - diff --git a/hack/ci/e2e.sh b/hack/ci/e2e.sh index b20acf6461..249322fca3 100755 --- a/hack/ci/e2e.sh +++ b/hack/ci/e2e.sh @@ -71,14 +71,9 @@ build() { # up a cluster with kind create_cluster() { -<<<<<<< HEAD - # create the config file - cat < "${ARTIFACTS}/kind-config.yaml" -======= IP_FAMILY="dual-stack" # create the config file cat < "${ARTIFACTS}/kind-config.yaml" ->>>>>>> TEMP: ci testing # config for 1 control plane node and 2 workers # necessary for conformance kind: Cluster diff --git a/pkg/build/node/cni.go b/pkg/build/node/cni.go index 82410ba1db..dab0b52bab 100644 --- a/pkg/build/node/cni.go +++ b/pkg/build/node/cni.go @@ -31,7 +31,6 @@ var defaultCNIImages = []string{"kindest/kindnetd:0.6.0"} const defaultCNIManifest = ` # kindnetd networking manifest -# would you kindly template this file --- apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -160,8 +159,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: POD_SUBNET - value: {{ .PodSubnet }} + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs volumeMounts: - name: cni-cfg mountPath: /etc/cni/net.d diff --git a/pkg/cluster/config/v1alpha3/types.go b/pkg/cluster/config/v1alpha3/types.go new file mode 100644 index 0000000000..27f6546372 --- /dev/null +++ b/pkg/cluster/config/v1alpha3/types.go @@ -0,0 +1,126 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kind/pkg/container/cri" + "sigs.k8s.io/kind/pkg/kustomize" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Cluster contains kind cluster configuration +type Cluster struct { + // TypeMeta representing the type of the object and its API schema version. + metav1.TypeMeta `json:",inline"` + + // Nodes contains the list of nodes defined in the `kind` Cluster + // If unset this will default to a single control-plane node + // Note that if more than one control plane is specified, an external + // control plane load balancer will be provisioned implicitly + Nodes []Node `json:"nodes"` + + /* Advanced fields */ + + // Networking contains cluster wide network settings + Networking Networking `json:"networking"` + + // KubeadmConfigPatches are applied to the generated kubeadm config as + // strategic merge patches to `kustomize build` internally + // https://github.com/kubernetes/community/blob/master/contributors/devel/strategic-merge-patch.md + // This should be an inline yaml blob-string + KubeadmConfigPatches []string `json:"kubeadmConfigPatches,omitempty"` + + // KubeadmConfigPatchesJSON6902 are applied to the generated kubeadm config + // as patchesJson6902 to `kustomize build` + KubeadmConfigPatchesJSON6902 []kustomize.PatchJSON6902 `json:"kubeadmConfigPatchesJson6902,omitempty"` +} + +// Node contains settings for a node in the `kind` Cluster. +// A node in kind config represent a container that will be provisioned with all the components +// required for the assigned role in the Kubernetes cluster +type Node struct { + // Role defines the role of the node in the in the Kubernetes cluster + // created by kind + // + // Defaults to "control-plane" + Role NodeRole `json:"role,omitempty"` + + // Image is the node image to use when creating this node + // If unset a default image will be used, see defaults.Image + Image string `json:"image,omitempty"` + + /* Advanced fields */ + + // ExtraMounts describes additional mount points for the node container + // These may be used to bind a hostPath + ExtraMounts []cri.Mount `json:"extraMounts,omitempty"` + + // ExtraPortMappings describes additional port mappings for the node container + // binded to a host Port + ExtraPortMappings []cri.PortMapping `json:"extraPortMappings,omitempty"` +} + +// NodeRole defines possible role for nodes in a Kubernetes cluster managed by `kind` +type NodeRole string + +const ( + // ControlPlaneRole identifies a node that hosts a Kubernetes control-plane. + // NOTE: in single node clusters, control-plane nodes act also as a worker + // nodes, in which case the taint will be removed. see: + // https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#control-plane-node-isolation + ControlPlaneRole NodeRole = "control-plane" + // WorkerRole identifies a node that hosts a Kubernetes worker + WorkerRole NodeRole = "worker" +) + +// Networking contains cluster wide network settings +type Networking struct { + // IPFamily is the network cluster model, currently it can be ipv4 or ipv6 + IPFamily ClusterIPFamily `json:"ipFamily,omitempty"` + // APIServerPort is the listen port on the host for the Kubernetes API Server + // Defaults to a random port on the host + APIServerPort int32 `json:"apiServerPort,omitempty"` + // APIServerAddress is the listen address on the host for the Kubernetes + // API Server. This should be an IP address. + // + // Defaults to 127.0.0.1 + APIServerAddress string `json:"apiServerAddress,omitempty"` + // PodSubnet is the CIDR used for pod IPs + // kind will select a default if unspecified + PodSubnet string `json:"podSubnet,omitempty"` + // ServiceSubnet is the CIDR used for services VIPs + // kind will select a default if unspecified for IPv6 + ServiceSubnet string `json:"serviceSubnet,omitempty"` + // If DisableDefaultCNI is true, kind will not install the default CNI setup. + // Instead the user should install their own CNI after creating the cluster. + DisableDefaultCNI bool `json:"disableDefaultCNI,omitempty"` +} + +// ClusterIPFamily defines cluster network IP family +type ClusterIPFamily string + +const ( + // IPv4Family sets ClusterIPFamily to ipv4 + IPv4Family ClusterIPFamily = "ipv4" + // IPv6Family sets ClusterIPFamily to ipv6 + IPv6Family ClusterIPFamily = "ipv6" + // DualStackFamily sets ClusterIPFamily to dual-stack + DualStackFamily ClusterIPFamily = "dual-stack" +)