Allow virtual-kubelet to use cluster domain

This allows `--cluster-domain` to be passed to virtual kubelet like a traditional kublet, and use this to generate search-domains for `/etc/resolv.conf` * Set default `cluster-domain` to `cluster-local` to match current kubelet * Added an example usage for the Azure provider * * Only apply to pods with `DNSClusterFirst` to match kubelet * * Merge search-domains with any set in the `dnsConfig` * * Set `ndots` to the default 5 Related: #641 Signed-off-by: Graham Hayes <[email protected]>
grahamhayes · May 23, 2019 · f75d151 · f75d151
1 parent 42061ea
commit f75d151
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 30 deletions.
diff --git a/charts/virtual-kubelet-for-aks/templates/deployment.yaml b/charts/virtual-kubelet-for-aks/templates/deployment.yaml
@@ -50,9 +50,9 @@ spec:
         - name: MASTER_URI
           value: {{ .Values.env.masterUri }}
         - name: CLUSTER_CIDR
-          value: {{ .Values.env.clusterCIDR }}
+          value: {{ .Values.env.clusterCidr }}
         - name: KUBE_DNS_IP
-          value: {{ .Values.env.kubeDnsIP }}
+          value: {{ .Values.env.kubeDnsIp }}
         {{ if .Values.loganalytics.enabled }}
         - name: LOG_ANALYTICS_AUTH_LOCATION
           value: /etc/virtual-kubelet/loganalytics.json

diff --git a/cmd/virtual-kubelet/commands/root/flag.go b/cmd/virtual-kubelet/commands/root/flag.go
@@ -61,6 +61,7 @@ func (mv mapVar) Type() string {
 func installFlags(flags *pflag.FlagSet, c *Opts) {
 	flags.StringVar(&c.KubeConfigPath, "kubeconfig", c.KubeConfigPath, "kube config file to use for connecting to the Kubernetes API server")
 	flags.StringVar(&c.KubeNamespace, "namespace", c.KubeNamespace, "kubernetes namespace (default is 'all')")
+	flags.StringVar(&c.KubeClusterDomain, "cluster-domain", c.KubeClusterDomain, "kubernetes cluster-domain (default is 'cluster.local')")
 	flags.StringVar(&c.NodeName, "nodename", c.NodeName, "kubernetes node name")
 	flags.StringVar(&c.OperatingSystem, "os", c.OperatingSystem, "Operating System (Linux/Windows)")
 	flags.StringVar(&c.Provider, "provider", c.Provider, "cloud provider")

diff --git a/cmd/virtual-kubelet/commands/root/opts.go b/cmd/virtual-kubelet/commands/root/opts.go
@@ -35,7 +35,7 @@ const (
 	DefaultListenPort           = 10250 // TODO(cpuguy83)(VK1.0): Change this to an addr instead of just a port.. we should not be listening on all interfaces.
 	DefaultPodSyncWorkers       = 10
 	DefaultKubeNamespace        = corev1.NamespaceAll
-
+	DefaultKubeClusterDomain    = "cluster.local"
 	DefaultTaintEffect = string(corev1.TaintEffectNoSchedule)
 	DefaultTaintKey    = "virtual-kubelet.io/provider"
 )
@@ -50,6 +50,8 @@ type Opts struct {
 	KubeConfigPath string
 	// Namespace to watch for pods and other resources
 	KubeNamespace string
+	// Domain suffix to append to search domains for the pods created by virtual-kubelet
+	KubeClusterDomain string
 	// Sets the port to listen for requests from the Kubernetes API server
 	ListenPort int32
 
@@ -126,6 +128,10 @@ func SetDefaultOpts(c *Opts) error {
 		c.KubeNamespace = DefaultKubeNamespace
 	}
 
+	if c.KubeClusterDomain == "" {
+		c.KubeClusterDomain = DefaultKubeClusterDomain
+	}
+
 	if c.TaintKey == "" {
 		c.TaintKey = DefaultTaintKey
 	}

diff --git a/cmd/virtual-kubelet/commands/root/root.go b/cmd/virtual-kubelet/commands/root/root.go
@@ -122,7 +122,8 @@ func runRootCommand(ctx context.Context, c Opts) error {
 		ResourceManager: rm,
 		DaemonPort:      int32(c.ListenPort),
 		InternalIP:      os.Getenv("VKUBELET_POD_IP"),
-	}
+		KubeClusterDomain: c.KubeClusterDomain,
+		}
 
 	p, err := register.GetProvider(c.Provider, initConfig)
 	if err != nil {

diff --git a/providers/azure/aci.go b/providers/azure/aci.go
@@ -81,6 +81,7 @@ type ACIProvider struct {
 	vnetName           string
 	vnetResourceGroup  string
 	networkProfile     string
+	clusterDomain      string
 	kubeProxyExtension *aci.Extension
 	kubeDNSIP          string
 	extraUserAgent     string
@@ -139,11 +140,12 @@ func isValidACIRegion(region string) bool {
 }
 
 // NewACIProvider creates a new ACIProvider.
-func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*ACIProvider, error) {
+func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32, clusterDomain string) (*ACIProvider, error) {
 	var p ACIProvider
 	var err error
 
 	p.resourceManager = rm
+	p.clusterDomain = clusterDomain
 
 	if config != "" {
 		f, err := os.Open(config)
@@ -622,30 +624,27 @@ func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup, pod
 	containerGroup.NetworkProfile = &aci.NetworkProfileDefinition{ID: p.networkProfile}
 
 	containerGroup.ContainerGroupProperties.Extensions = []*aci.Extension{p.kubeProxyExtension}
-	containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod.Spec.DNSPolicy, pod.Spec.DNSConfig)
+	//containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod.Spec.DNSPolicy, pod.Spec.DNSConfig)
+	containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod)
 }
 
-func (p *ACIProvider) getDNSConfig(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSConfig) *aci.DNSConfig {
+func (p *ACIProvider) getDNSConfig(pod *v1.Pod) *aci.DNSConfig {
 	nameServers := make([]string, 0)
+	searchDomains := []string{}
+	dnsConfigOptions := []string{}
+
+	defaultDNSOptions := []string{"ndots:5"}
 
-	if dnsPolicy == v1.DNSClusterFirst || dnsPolicy == v1.DNSClusterFirstWithHostNet {
+	if pod.Spec.DNSPolicy == v1.DNSClusterFirst || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet {
 		nameServers = append(nameServers, p.kubeDNSIP)
+		searchDomains = p.generateSearchesForDNSClusterFirst(pod.Spec.DNSConfig, pod)
+		dnsConfigOptions = defaultDNSOptions
 	}
 
-	searchDomains := []string{}
-	options := []string{}
-
-	if dnsConfig != nil {
-		nameServers = omitDuplicates(append(nameServers, dnsConfig.Nameservers...))
-		searchDomains = omitDuplicates(dnsConfig.Searches)
-
-		for _, option := range dnsConfig.Options {
-			op := option.Name
-			if option.Value != nil && *(option.Value) != "" {
-				op = op + ":" + *(option.Value)
-			}
-			options = append(options, op)
-		}
+	if pod.Spec.DNSConfig != nil {
+		nameServers = omitDuplicates(append(nameServers, pod.Spec.DNSConfig.Nameservers...))
+		searchDomains = omitDuplicates(append(searchDomains, pod.Spec.DNSConfig.Searches...))
+		dnsConfigOptions = mergeDNSOptions(dnsConfigOptions, pod.Spec.DNSConfig.Options)
 	}
 
 	if len(nameServers) == 0 {
@@ -655,12 +654,31 @@ func (p *ACIProvider) getDNSConfig(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSC
 	result := aci.DNSConfig{
 		NameServers:   formDNSNameserversFitsLimits(nameServers),
 		SearchDomains: formDNSSearchFitsLimits(searchDomains),
-		Options:       strings.Join(options, " "),
+		Options:       strings.Join(dnsConfigOptions, " "),
 	}
 
 	return &result
 }
 
+// This is taken from the kubelet equivalent -  https://github.com/kubernetes/kubernetes/blob/d24fe8a801748953a5c34fd34faa8005c6ad1770/pkg/kubelet/network/dns/dns.go#L141-L151
+func (p* ACIProvider) generateSearchesForDNSClusterFirst(dnsConfig *v1.PodDNSConfig, pod *v1.Pod) []string {
+
+	hostSearch := []string{}
+
+	if dnsConfig != nil {
+		hostSearch = dnsConfig.Searches
+	}
+	if p.clusterDomain == "" {
+		return hostSearch
+	}
+
+	nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, p.clusterDomain)
+	svcDomain := fmt.Sprintf("svc.%s", p.clusterDomain)
+	clusterSearch := []string{nsSvcDomain, svcDomain, p.clusterDomain}
+
+	return omitDuplicates(append(clusterSearch, hostSearch...))
+}
+
 func omitDuplicates(strs []string) []string {
 	uniqueStrs := make(map[string]bool)
 
@@ -674,6 +692,37 @@ func omitDuplicates(strs []string) []string {
 	return ret
 }
 
+// This is taken from the kubelet equivalent - https://github.com/kubernetes/kubernetes/blob/d24fe8a801748953a5c34fd34faa8005c6ad1770/pkg/kubelet/network/dns/dns.go#L283-L311
+// mergeDNSOptions merges DNS options. If duplicated, entries given by PodDNSConfigOption will
+// overwrite the existing ones.
+func mergeDNSOptions(existingDNSConfigOptions []string, dnsConfigOptions []v1.PodDNSConfigOption) []string {
+	optionsMap := make(map[string]string)
+	for _, op := range existingDNSConfigOptions {
+		if index := strings.Index(op, ":"); index != -1 {
+			optionsMap[op[:index]] = op[index+1:]
+		} else {
+			optionsMap[op] = ""
+		}
+	}
+	for _, op := range dnsConfigOptions {
+		if op.Value != nil {
+			optionsMap[op.Name] = *op.Value
+		} else {
+			optionsMap[op.Name] = ""
+		}
+	}
+	// Reconvert DNS options into a string array.
+	options := []string{}
+	for opName, opValue := range optionsMap {
+		op := opName
+		if opValue != "" {
+			op = op + ":" + opValue
+		}
+		options = append(options, op)
+	}
+	return options
+}
+
 func formDNSNameserversFitsLimits(nameservers []string) []string {
 	if len(nameservers) > maxDNSNameservers {
 		nameservers = nameservers[:maxDNSNameservers]

diff --git a/providers/azure/aci_test.go b/providers/azure/aci_test.go
@@ -838,7 +838,7 @@ func createTestProvider(aadServerMocker *AADMock, aciServerMocker *ACIMock) (*AC
 		return nil, err
 	}
 
-	provider, err := NewACIProvider("example.toml", rm, fakeNodeName, "Linux", "0.0.0.0", 10250)
+	provider, err := NewACIProvider("example.toml", rm, fakeNodeName, "Linux", "0.0.0.0", 10250, "cluster.local")
 	if err != nil {
 		return nil, err
 	}

diff --git a/providers/register/provider_azure.go b/providers/register/provider_azure.go
@@ -19,5 +19,6 @@ func initAzure(cfg InitConfig) (providers.Provider, error) {
 		cfg.OperatingSystem,
 		cfg.InternalIP,
 		cfg.DaemonPort,
+		cfg.KubeClusterDomain,
 	)
 }
diff --git a/providers/register/register.go b/providers/register/register.go
@@ -13,12 +13,13 @@ var providerInits = make(map[string]initFunc)
 
 // InitConfig is the config passed to initialize a registered provider.
 type InitConfig struct {
-	ConfigPath      string
-	NodeName        string
-	OperatingSystem string
-	InternalIP      string
-	DaemonPort      int32
-	ResourceManager *manager.ResourceManager
+	ConfigPath        string
+	NodeName          string
+	OperatingSystem   string
+	InternalIP        string
+	DaemonPort        int32
+	ResourceManager   *manager.ResourceManager
+	KubeClusterDomain string
 }
 
 type initFunc func(InitConfig) (providers.Provider, error)