Allow virtual-kubelet to use cluster domain

This allows `--cluster-domain` to be passed to virtual kubelet like a traditional kublet, and use this to generate search-domains for `/etc/resolv.conf` * Set default `cluster-domain` to `cluster-local` to match current kubelet * Added an example usage for the Azure provider * * Only apply to pods with `DNSClusterFirst` to match kubelet * * Merge search-domains with any set in the `dnsConfig` Related: #641 Signed-off-by: Graham Hayes <[email protected]>
grahamhayes · May 22, 2019 · b11bcb1 · b11bcb1
1 parent 42061ea
commit b11bcb1
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 18 deletions.
diff --git a/cmd/virtual-kubelet/commands/root/flag.go b/cmd/virtual-kubelet/commands/root/flag.go
@@ -61,6 +61,7 @@ func (mv mapVar) Type() string {
 func installFlags(flags *pflag.FlagSet, c *Opts) {
 	flags.StringVar(&c.KubeConfigPath, "kubeconfig", c.KubeConfigPath, "kube config file to use for connecting to the Kubernetes API server")
 	flags.StringVar(&c.KubeNamespace, "namespace", c.KubeNamespace, "kubernetes namespace (default is 'all')")
+	flags.StringVar(&c.KubeClusterDomain, "cluster-domain", c.KubeClusterDomain, "kubernetes cluster-domain (default is 'cluster.local')")
 	flags.StringVar(&c.NodeName, "nodename", c.NodeName, "kubernetes node name")
 	flags.StringVar(&c.OperatingSystem, "os", c.OperatingSystem, "Operating System (Linux/Windows)")
 	flags.StringVar(&c.Provider, "provider", c.Provider, "cloud provider")

diff --git a/cmd/virtual-kubelet/commands/root/opts.go b/cmd/virtual-kubelet/commands/root/opts.go
@@ -35,7 +35,7 @@ const (
 	DefaultListenPort           = 10250 // TODO(cpuguy83)(VK1.0): Change this to an addr instead of just a port.. we should not be listening on all interfaces.
 	DefaultPodSyncWorkers       = 10
 	DefaultKubeNamespace        = corev1.NamespaceAll
-
+	DefaultKubeClusterDomain    = "cluster.local"
 	DefaultTaintEffect = string(corev1.TaintEffectNoSchedule)
 	DefaultTaintKey    = "virtual-kubelet.io/provider"
 )
@@ -50,6 +50,8 @@ type Opts struct {
 	KubeConfigPath string
 	// Namespace to watch for pods and other resources
 	KubeNamespace string
+	// Domain suffix to append to search domains for the pods created by virtual-kubelet
+	KubeClusterDomain string
 	// Sets the port to listen for requests from the Kubernetes API server
 	ListenPort int32
 
@@ -126,6 +128,10 @@ func SetDefaultOpts(c *Opts) error {
 		c.KubeNamespace = DefaultKubeNamespace
 	}
 
+	if c.KubeClusterDomain == "" {
+		c.KubeClusterDomain = DefaultKubeClusterDomain
+	}
+
 	if c.TaintKey == "" {
 		c.TaintKey = DefaultTaintKey
 	}

diff --git a/cmd/virtual-kubelet/commands/root/root.go b/cmd/virtual-kubelet/commands/root/root.go
@@ -122,7 +122,8 @@ func runRootCommand(ctx context.Context, c Opts) error {
 		ResourceManager: rm,
 		DaemonPort:      int32(c.ListenPort),
 		InternalIP:      os.Getenv("VKUBELET_POD_IP"),
-	}
+		KubeClusterDomain: c.KubeClusterDomain,
+		}
 
 	p, err := register.GetProvider(c.Provider, initConfig)
 	if err != nil {

diff --git a/providers/azure/aci.go b/providers/azure/aci.go
@@ -81,6 +81,7 @@ type ACIProvider struct {
 	vnetName           string
 	vnetResourceGroup  string
 	networkProfile     string
+	clusterDomain      string
 	kubeProxyExtension *aci.Extension
 	kubeDNSIP          string
 	extraUserAgent     string
@@ -139,11 +140,12 @@ func isValidACIRegion(region string) bool {
 }
 
 // NewACIProvider creates a new ACIProvider.
-func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*ACIProvider, error) {
+func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32, clusterDomain string) (*ACIProvider, error) {
 	var p ACIProvider
 	var err error
 
 	p.resourceManager = rm
+	p.clusterDomain = clusterDomain
 
 	if config != "" {
 		f, err := os.Open(config)
@@ -622,24 +624,27 @@ func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup, pod
 	containerGroup.NetworkProfile = &aci.NetworkProfileDefinition{ID: p.networkProfile}
 
 	containerGroup.ContainerGroupProperties.Extensions = []*aci.Extension{p.kubeProxyExtension}
-	containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod.Spec.DNSPolicy, pod.Spec.DNSConfig)
+	//containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod.Spec.DNSPolicy, pod.Spec.DNSConfig)
+	containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod)
 }
 
-func (p *ACIProvider) getDNSConfig(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSConfig) *aci.DNSConfig {
+func (p *ACIProvider) getDNSConfig(pod *v1.Pod) *aci.DNSConfig {
 	nameServers := make([]string, 0)
+	searchDomains := []string{}
 
-	if dnsPolicy == v1.DNSClusterFirst || dnsPolicy == v1.DNSClusterFirstWithHostNet {
+	if pod.Spec.DNSPolicy == v1.DNSClusterFirst || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet {
 		nameServers = append(nameServers, p.kubeDNSIP)
+		searchDomains = p.generateSearchesForDNSClusterFirst(pod.Spec.DNSConfig, pod)
 	}
 
-	searchDomains := []string{}
+
 	options := []string{}
 
-	if dnsConfig != nil {
-		nameServers = omitDuplicates(append(nameServers, dnsConfig.Nameservers...))
-		searchDomains = omitDuplicates(dnsConfig.Searches)
+	if pod.Spec.DNSConfig != nil {
+		nameServers = omitDuplicates(append(nameServers, pod.Spec.DNSConfig.Nameservers...))
+		searchDomains = omitDuplicates(append(searchDomains, pod.Spec.DNSConfig.Searches...))
 
-		for _, option := range dnsConfig.Options {
+		for _, option := range pod.Spec.DNSConfig.Options {
 			op := option.Name
 			if option.Value != nil && *(option.Value) != "" {
 				op = op + ":" + *(option.Value)
@@ -661,6 +666,25 @@ func (p *ACIProvider) getDNSConfig(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSC
 	return &result
 }
 
+// This is taken from the kubelet equivalent -  https://github.com/kubernetes/kubernetes/blob/d24fe8a801748953a5c34fd34faa8005c6ad1770/pkg/kubelet/network/dns/dns.go#L141-L151
+func (p* ACIProvider) generateSearchesForDNSClusterFirst(dnsConfig *v1.PodDNSConfig, pod *v1.Pod) []string {
+
+	hostSearch := []string{}
+
+	if dnsConfig != nil {
+		hostSearch = dnsConfig.Searches
+	}
+	if p.clusterDomain == "" {
+		return hostSearch
+	}
+
+	nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, p.clusterDomain)
+	svcDomain := fmt.Sprintf("svc.%s", p.clusterDomain)
+	clusterSearch := []string{nsSvcDomain, svcDomain, p.clusterDomain}
+
+	return omitDuplicates(append(clusterSearch, hostSearch...))
+}
+
 func omitDuplicates(strs []string) []string {
 	uniqueStrs := make(map[string]bool)
 

diff --git a/providers/azure/aci_test.go b/providers/azure/aci_test.go
@@ -838,7 +838,7 @@ func createTestProvider(aadServerMocker *AADMock, aciServerMocker *ACIMock) (*AC
 		return nil, err
 	}
 
-	provider, err := NewACIProvider("example.toml", rm, fakeNodeName, "Linux", "0.0.0.0", 10250)
+	provider, err := NewACIProvider("example.toml", rm, fakeNodeName, "Linux", "0.0.0.0", 10250, "cluster.local")
 	if err != nil {
 		return nil, err
 	}

diff --git a/providers/register/provider_azure.go b/providers/register/provider_azure.go
@@ -19,5 +19,6 @@ func initAzure(cfg InitConfig) (providers.Provider, error) {
 		cfg.OperatingSystem,
 		cfg.InternalIP,
 		cfg.DaemonPort,
+		cfg.KubeClusterDomain,
 	)
 }
diff --git a/providers/register/register.go b/providers/register/register.go
@@ -13,12 +13,13 @@ var providerInits = make(map[string]initFunc)
 
 // InitConfig is the config passed to initialize a registered provider.
 type InitConfig struct {
-	ConfigPath      string
-	NodeName        string
-	OperatingSystem string
-	InternalIP      string
-	DaemonPort      int32
-	ResourceManager *manager.ResourceManager
+	ConfigPath        string
+	NodeName          string
+	OperatingSystem   string
+	InternalIP        string
+	DaemonPort        int32
+	ResourceManager   *manager.ResourceManager
+	KubeClusterDomain string
 }
 
 type initFunc func(InitConfig) (providers.Provider, error)