Allow virtual-kubelet to use cluster domain

This allows `--cluster-domain` to be passed to virtual kubelet like a traditional kublet, and use this to generate search-domains for `/etc/resolv.conf` * Only apply to pods with `DNSClusterFirst` to match kubelet * Merge search-domains with any set in the `dnsConfig` * Set `ndots` to the default 5 Related: #641 Depends-On: virtual-kubelet/virtual-kubelet#642 Signed-off-by: Graham Hayes <[email protected]>
virtual-kubelet · Aug 21, 2019 · 90ddefb · 90ddefb
1 parent 79119b4
commit 90ddefb
Show file tree

Hide file tree

Showing 5 changed files with 158 additions and 60 deletions.
diff --git a/aci.go b/aci.go
@@ -82,6 +82,7 @@ type ACIProvider struct {
 	vnetName           string
 	vnetResourceGroup  string
 	networkProfile     string
+	clusterDomain      string
 	kubeProxyExtension *aci.Extension
 	kubeDNSIP          string
 	extraUserAgent     string
@@ -140,11 +141,12 @@ func isValidACIRegion(region string) bool {
 }
 
 // NewACIProvider creates a new ACIProvider.
-func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*ACIProvider, error) {
+func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32, clusterDomain string) (*ACIProvider, error) {
 	var p ACIProvider
 	var err error
 
 	p.resourceManager = rm
+	p.clusterDomain = clusterDomain
 
 	if config != "" {
 		f, err := os.Open(config)
@@ -626,26 +628,29 @@ func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup, pod
 	}
 
 	containerGroup.NetworkProfile = &aci.NetworkProfileDefinition{ID: p.networkProfile}
+	containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod)
 
 	containerGroup.ContainerGroupProperties.Extensions = []*aci.Extension{p.kubeProxyExtension}
-	containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod.Spec.DNSPolicy, pod.Spec.DNSConfig)
+	containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod)
 }
 
-func (p *ACIProvider) getDNSConfig(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSConfig) *aci.DNSConfig {
+func (p *ACIProvider) getDNSConfig(pod *v1.Pod) *aci.DNSConfig {
 	nameServers := make([]string, 0)
+	searchDomains := []string{}
 
-	if dnsPolicy == v1.DNSClusterFirst || dnsPolicy == v1.DNSClusterFirstWithHostNet {
+	if pod.Spec.DNSPolicy == v1.DNSClusterFirst || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet {
 		nameServers = append(nameServers, p.kubeDNSIP)
+		searchDomains = p.generateSearchesForDNSClusterFirst(pod.Spec.DNSConfig, pod)
 	}
 
-	searchDomains := []string{}
+
 	options := []string{}
 
-	if dnsConfig != nil {
-		nameServers = omitDuplicates(append(nameServers, dnsConfig.Nameservers...))
-		searchDomains = omitDuplicates(dnsConfig.Searches)
+	if pod.Spec.DNSConfig != nil {
+		nameServers = omitDuplicates(append(nameServers, pod.Spec.DNSConfig.Nameservers...))
+		searchDomains = omitDuplicates(append(searchDomains, pod.Spec.DNSConfig.Searches...))
 
-		for _, option := range dnsConfig.Options {
+		for _, option := range pod.Spec.DNSConfig.Options {
 			op := option.Name
 			if option.Value != nil && *(option.Value) != "" {
 				op = op + ":" + *(option.Value)
@@ -667,6 +672,26 @@ func (p *ACIProvider) getDNSConfig(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSC
 	return &result
 }
 
+// This is taken from the kubelet equivalent -  https://github.com/kubernetes/kubernetes/blob/d24fe8a801748953a5c34fd34faa8005c6ad1770/pkg/kubelet/network/dns/dns.go#L141-L151
+func (p* ACIProvider) generateSearchesForDNSClusterFirst(dnsConfig *v1.PodDNSConfig, pod *v1.Pod) []string {
+
+	hostSearch := []string{}
+
+	if dnsConfig != nil {
+		hostSearch = dnsConfig.Searches
+	}
+	if p.clusterDomain == "" {
+		return hostSearch
+	}
+
+	nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, p.clusterDomain)
+	svcDomain := fmt.Sprintf("svc.%s", p.clusterDomain)
+	clusterSearch := []string{nsSvcDomain, svcDomain, p.clusterDomain}
+
+	return omitDuplicates(append(clusterSearch, hostSearch...))
+}
+
+
 func omitDuplicates(strs []string) []string {
 	uniqueStrs := make(map[string]bool)
 

diff --git a/aci_test.go b/aci_test.go
@@ -838,7 +838,7 @@ func createTestProvider(aadServerMocker *AADMock, aciServerMocker *ACIMock) (*AC
 		return nil, err
 	}
 
-	provider, err := NewACIProvider("example.toml", rm, fakeNodeName, "Linux", "0.0.0.0", 10250)
+	provider, err := NewACIProvider("example.toml", rm, fakeNodeName, "Linux", "0.0.0.0", 10250, "cluster.local")
 	if err != nil {
 		return nil, err
 	}

diff --git a/cmd/virtual-kubelet/register.go b/cmd/virtual-kubelet/register.go
@@ -7,6 +7,6 @@ import (
 
 func registerACI(s *providers.Store) error {
 	return s.Register("azure", func(cfg providers.InitConfig) (providers.Provider, error) {
-		return azure.NewACIProvider(cfg.ConfigPath, cfg.ResourceManager, cfg.NodeName, cfg.OperatingSystem, cfg.InternalIP, cfg.DaemonPort)
+		return azure.NewACIProvider(cfg.ConfigPath, cfg.ResourceManager, cfg.NodeName, cfg.OperatingSystem, cfg.InternalIP, cfg.DaemonPort, cfg.KubeClusterDomain)
 	})
 }
diff --git a/go.mod b/go.mod
@@ -1,44 +1,24 @@
 module github.com/virtual-kubelet/azure-aci
 
 require (
-	contrib.go.opencensus.io/exporter/ocagent v0.4.6 // indirect
 	github.com/Azure/azure-sdk-for-go v26.0.0+incompatible
 	github.com/Azure/go-autorest v11.5.0+incompatible
 	github.com/BurntSushi/toml v0.3.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/dimchansky/utfbom v1.1.0
 	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
-	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/gogo/protobuf v1.2.1 // indirect
 	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
-	github.com/golang/protobuf v1.3.1 // indirect
 	github.com/google/uuid v1.1.0
 	github.com/gorilla/mux v1.7.2
 	github.com/gorilla/websocket v1.4.0
-	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
-	github.com/json-iterator/go v1.1.6 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.5
-	github.com/virtual-kubelet/virtual-kubelet v0.9.1
-	go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b
-	golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 // indirect
-	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
+	github.com/virtual-kubelet/virtual-kubelet v0.10.1-0.20190620063316-f6be3ce71309
+	go.opencensus.io v0.20.2
 	golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6
-	golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db // indirect
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	google.golang.org/appengine v1.5.0 // indirect
-	google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107 // indirect
-	google.golang.org/grpc v1.19.1 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
 	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.0.0-20190612125737-db0771252981
 	k8s.io/apimachinery v0.0.0-20190612125636-6a5db36e93ad
@@ -48,8 +28,6 @@ require (
 	k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a // indirect
 )
 
-replace github.com/virtual-kubelet/virtual-kubelet => github.com/virtual-kubelet/virtual-kubelet v0.10.1-0.20190618101111-a00c2f4b8b2c
-
 replace k8s.io/api => k8s.io/api v0.0.0-20190606204050-af9c91bd2759
 
 replace k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d