Skip to content

Commit

Permalink
Merge pull request kubernetes#55651 from MrHohn/kubelet-dns-pkg
Browse files Browse the repository at this point in the history 
Automatic merge from submit-queue (batch tested with PRs 55657, 54758, 47584, 55758, 55651). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Move DNS related kubelet codes into its own package

**What this PR does / why we need it**:
Ref kubernetes/enhancements#504, this PR rearranges DNS related kubelet codes into its own pacakge and adds an OWNERS file.

Again, there is no functional changes, just that codes are moved around and couple fields (`clusterDomain`, `clusterDNS`, `resolverConfig`) are replaced with a `dnsConfigurer` struct.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #NONE 

**Special notes for your reviewer**:
/assign @bowei @thockin 

**Release note**:

```release-note
NONE
```
  • Loading branch information
Kubernetes Submit Queue authored Nov 16, 2017
2 parents 817e8a2 + 0bc2e1f commit 6e950cc
Show file tree
Hide file tree
Showing 10 changed files with 644 additions and 479 deletions.
1 change: 1 addition & 0 deletions pkg/kubelet/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ go_library(
"//pkg/kubelet/metrics:go_default_library",
"//pkg/kubelet/mountpod:go_default_library",
"//pkg/kubelet/network:go_default_library",
"//pkg/kubelet/network/dns:go_default_library",
"//pkg/kubelet/pleg:go_default_library",
"//pkg/kubelet/pod:go_default_library",
"//pkg/kubelet/preemption:go_default_library",
Expand Down
26 changes: 9 additions & 17 deletions pkg/kubelet/kubelet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
"k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/pkg/kubelet/network"
"k8s.io/kubernetes/pkg/kubelet/network/dns"
"k8s.io/kubernetes/pkg/kubelet/pleg"
kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
"k8s.io/kubernetes/pkg/kubelet/preemption"
Expand Down Expand Up @@ -477,6 +478,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
}
httpClient := &http.Client{}
parsedNodeIP := net.ParseIP(nodeIP)

klet := &Kubelet{
hostname: hostname,
Expand All @@ -488,8 +490,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
sourcesReady: config.NewSourcesReady(kubeDeps.PodConfig.SeenAllSources),
registerNode: kubeCfg.RegisterNode,
registerSchedulable: registerSchedulable,
clusterDomain: kubeCfg.ClusterDomain,
clusterDNS: clusterDNS,
dnsConfigurer: dns.NewConfigurer(kubeDeps.Recorder, nodeRef, parsedNodeIP, clusterDNS, kubeCfg.ClusterDomain, kubeCfg.ResolverConfig),
serviceLister: serviceLister,
nodeInfo: nodeInfo,
masterServiceNamespace: masterServiceNamespace,
Expand All @@ -512,11 +513,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
maxPods: int(kubeCfg.MaxPods),
podsPerCore: int(kubeCfg.PodsPerCore),
syncLoopMonitor: atomic.Value{},
resolverConfig: kubeCfg.ResolverConfig,
daemonEndpoints: daemonEndpoints,
containerManager: kubeDeps.ContainerManager,
containerRuntimeName: containerRuntime,
nodeIP: net.ParseIP(nodeIP),
nodeIP: parsedNodeIP,
clock: clock.RealClock{},
enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach,
iptClient: utilipt.New(utilexec.New(), utildbus.New(), utilipt.ProtocolIpv4),
Expand Down Expand Up @@ -806,7 +806,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
experimentalCheckNodeCapabilitiesBeforeMount = false
// Replace the nameserver in containerized-mounter's rootfs/etc/resolve.conf with kubelet.ClusterDNS
// so that service name could be resolved
klet.setupDNSinContainerizedMounter(experimentalMounterPath)
klet.dnsConfigurer.SetupDNSinContainerizedMounter(experimentalMounterPath)
}

// setup volumeManager
Expand Down Expand Up @@ -948,11 +948,8 @@ type Kubelet struct {
// for internal book keeping; access only from within registerWithApiserver
registrationCompleted bool

// If non-empty, use this for container DNS search.
clusterDomain string

// If non-nil, use this for container DNS server.
clusterDNS []net.IP
// dnsConfigurer is used for setting up DNS resolver configuration when launching pods.
dnsConfigurer *dns.Configurer

// masterServiceNamespace is the namespace that the master service is exposed in.
masterServiceNamespace string
Expand Down Expand Up @@ -1097,11 +1094,6 @@ type Kubelet struct {
// Channel for sending pods to kill.
podKillingCh chan *kubecontainer.PodPair

// The configuration file used as the base to generate the container's
// DNS resolver configuration file. This can be used in conjunction with
// clusterDomain and clusterDNS.
resolverConfig string

// Information about the ports which are opened by daemons on Node running this Kubelet server.
daemonEndpoints *v1.NodeDaemonEndpoints

Expand Down Expand Up @@ -1375,8 +1367,8 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
go wait.Until(kl.podKiller, 1*time.Second, wait.NeverStop)

// Start gorouting responsible for checking limits in resolv.conf
if kl.resolverConfig != "" {
go wait.Until(func() { kl.checkLimitsForResolvConf() }, 30*time.Second, wait.NeverStop)
if kl.dnsConfigurer.ResolverConfig != "" {
go wait.Until(func() { kl.dnsConfigurer.CheckLimitsForResolvConf() }, 30*time.Second, wait.NeverStop)
}

// Start component sync loops.
Expand Down
264 changes: 8 additions & 256 deletions pkg/kubelet/kubelet_network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,6 @@ package kubelet

import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"

"github.com/golang/glog"
"k8s.io/api/core/v1"
Expand All @@ -31,7 +26,6 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/network"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/kubelet/util/format"
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
)

Expand Down Expand Up @@ -164,256 +158,6 @@ func (kl *Kubelet) providerRequiresNetworkingConfiguration() bool {
return supported
}

func omitDuplicates(pod *v1.Pod, combinedSearch []string) []string {
uniqueDomains := map[string]bool{}

for _, dnsDomain := range combinedSearch {
if _, exists := uniqueDomains[dnsDomain]; !exists {
combinedSearch[len(uniqueDomains)] = dnsDomain
uniqueDomains[dnsDomain] = true
}
}
return combinedSearch[:len(uniqueDomains)]
}

func (kl *Kubelet) formDNSSearchFitsLimits(pod *v1.Pod, composedSearch []string) []string {
// resolver file Search line current limitations
resolvSearchLineDNSDomainsLimit := 6
resolvSearchLineLenLimit := 255
limitsExceeded := false

if len(composedSearch) > resolvSearchLineDNSDomainsLimit {
composedSearch = composedSearch[:resolvSearchLineDNSDomainsLimit]
limitsExceeded = true
}

if resolvSearchhLineStrLen := len(strings.Join(composedSearch, " ")); resolvSearchhLineStrLen > resolvSearchLineLenLimit {
cutDomainsNum := 0
cutDoaminsLen := 0
for i := len(composedSearch) - 1; i >= 0; i-- {
cutDoaminsLen += len(composedSearch[i]) + 1
cutDomainsNum++

if (resolvSearchhLineStrLen - cutDoaminsLen) <= resolvSearchLineLenLimit {
break
}
}

composedSearch = composedSearch[:(len(composedSearch) - cutDomainsNum)]
limitsExceeded = true
}

if limitsExceeded {
log := fmt.Sprintf("Search Line limits were exceeded, some dns names have been omitted, the applied search line is: %s", strings.Join(composedSearch, " "))
kl.recorder.Event(pod, v1.EventTypeWarning, "DNSSearchForming", log)
glog.Error(log)
}
return composedSearch
}

func (kl *Kubelet) formDNSSearchForDNSDefault(hostSearch []string, pod *v1.Pod) []string {
return kl.formDNSSearchFitsLimits(pod, hostSearch)
}

func (kl *Kubelet) formDNSSearch(hostSearch []string, pod *v1.Pod) []string {
if kl.clusterDomain == "" {
kl.formDNSSearchFitsLimits(pod, hostSearch)
return hostSearch
}

nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, kl.clusterDomain)
svcDomain := fmt.Sprintf("svc.%s", kl.clusterDomain)
dnsSearch := []string{nsSvcDomain, svcDomain, kl.clusterDomain}

combinedSearch := append(dnsSearch, hostSearch...)

combinedSearch = omitDuplicates(pod, combinedSearch)
return kl.formDNSSearchFitsLimits(pod, combinedSearch)
}

func (kl *Kubelet) checkLimitsForResolvConf() {
// resolver file Search line current limitations
resolvSearchLineDNSDomainsLimit := 6
resolvSearchLineLenLimit := 255

f, err := os.Open(kl.resolverConfig)
if err != nil {
kl.recorder.Event(kl.nodeRef, v1.EventTypeWarning, "checkLimitsForResolvConf", err.Error())
glog.Error("checkLimitsForResolvConf: " + err.Error())
return
}
defer f.Close()

_, hostSearch, _, err := kl.parseResolvConf(f)
if err != nil {
kl.recorder.Event(kl.nodeRef, v1.EventTypeWarning, "checkLimitsForResolvConf", err.Error())
glog.Error("checkLimitsForResolvConf: " + err.Error())
return
}

domainCntLimit := resolvSearchLineDNSDomainsLimit

if kl.clusterDomain != "" {
domainCntLimit -= 3
}

if len(hostSearch) > domainCntLimit {
log := fmt.Sprintf("Resolv.conf file '%s' contains search line consisting of more than %d domains!", kl.resolverConfig, domainCntLimit)
kl.recorder.Event(kl.nodeRef, v1.EventTypeWarning, "checkLimitsForResolvConf", log)
glog.Error("checkLimitsForResolvConf: " + log)
return
}

if len(strings.Join(hostSearch, " ")) > resolvSearchLineLenLimit {
log := fmt.Sprintf("Resolv.conf file '%s' contains search line which length is more than allowed %d chars!", kl.resolverConfig, resolvSearchLineLenLimit)
kl.recorder.Event(kl.nodeRef, v1.EventTypeWarning, "checkLimitsForResolvConf", log)
glog.Error("checkLimitsForResolvConf: " + log)
return
}

return
}

// parseResolveConf reads a resolv.conf file from the given reader, and parses
// it into nameservers, searches and options, possibly returning an error.
// TODO: move to utility package
func (kl *Kubelet) parseResolvConf(reader io.Reader) (nameservers []string, searches []string, options []string, err error) {
file, err := ioutil.ReadAll(reader)
if err != nil {
return nil, nil, nil, err
}

// Lines of the form "nameserver 1.2.3.4" accumulate.
nameservers = []string{}

// Lines of the form "search example.com" overrule - last one wins.
searches = []string{}

// Lines of the form "option ndots:5 attempts:2" overrule - last one wins.
// Each option is recorded as an element in the array.
options = []string{}

lines := strings.Split(string(file), "\n")
for l := range lines {
trimmed := strings.TrimSpace(lines[l])
if strings.HasPrefix(trimmed, "#") {
continue
}
fields := strings.Fields(trimmed)
if len(fields) == 0 {
continue
}
if fields[0] == "nameserver" && len(fields) >= 2 {
nameservers = append(nameservers, fields[1])
}
if fields[0] == "search" {
searches = fields[1:]
}
if fields[0] == "options" {
options = fields[1:]
}
}

// There used to be code here to scrub DNS for each cloud, but doesn't
// make sense anymore since cloudproviders are being factored out.
// contact @thockin or @wlan0 for more information

return nameservers, searches, options, nil
}

// GetClusterDNS returns a list of the DNS servers, a list of the DNS search
// domains of the cluster, and a list of resolv.conf options.
func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, []string, bool, error) {
var hostDNS, hostSearch, hostOptions []string
// Get host DNS settings
if kl.resolverConfig != "" {
f, err := os.Open(kl.resolverConfig)
if err != nil {
return nil, nil, nil, false, err
}
defer f.Close()

hostDNS, hostSearch, hostOptions, err = kl.parseResolvConf(f)
if err != nil {
return nil, nil, nil, false, err
}
}
useClusterFirstPolicy := ((pod.Spec.DNSPolicy == v1.DNSClusterFirst && !kubecontainer.IsHostNetworkPod(pod)) || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet)
if useClusterFirstPolicy && len(kl.clusterDNS) == 0 {
// clusterDNS is not known.
// pod with ClusterDNSFirst Policy cannot be created
kl.recorder.Eventf(pod, v1.EventTypeWarning, "MissingClusterDNS", "kubelet does not have ClusterDNS IP configured and cannot create Pod using %q policy. Falling back to DNSDefault policy.", pod.Spec.DNSPolicy)
log := fmt.Sprintf("kubelet does not have ClusterDNS IP configured and cannot create Pod using %q policy. pod: %q. Falling back to DNSDefault policy.", pod.Spec.DNSPolicy, format.Pod(pod))
kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, "MissingClusterDNS", log)

// fallback to DNSDefault
useClusterFirstPolicy = false
}

if !useClusterFirstPolicy {
// When the kubelet --resolv-conf flag is set to the empty string, use
// DNS settings that override the docker default (which is to use
// /etc/resolv.conf) and effectively disable DNS lookups. According to
// the bind documentation, the behavior of the DNS client library when
// "nameservers" are not specified is to "use the nameserver on the
// local machine". A nameserver setting of localhost is equivalent to
// this documented behavior.
if kl.resolverConfig == "" {
hostSearch = []string{"."}
switch {
case kl.nodeIP == nil || kl.nodeIP.To4() != nil:
hostDNS = []string{"127.0.0.1"}
case kl.nodeIP.To16() != nil:
hostDNS = []string{"::1"}
}
} else {
hostSearch = kl.formDNSSearchForDNSDefault(hostSearch, pod)
}
return hostDNS, hostSearch, hostOptions, useClusterFirstPolicy, nil
}

// for a pod with DNSClusterFirst policy, the cluster DNS server is the only nameserver configured for
// the pod. The cluster DNS server itself will forward queries to other nameservers that is configured to use,
// in case the cluster DNS server cannot resolve the DNS query itself
dns := make([]string, len(kl.clusterDNS))
for i, ip := range kl.clusterDNS {
dns[i] = ip.String()
}
dnsSearch := kl.formDNSSearch(hostSearch, pod)

return dns, dnsSearch, hostOptions, useClusterFirstPolicy, nil
}

// Replace the nameserver in containerized-mounter's rootfs/etc/resolve.conf with kubelet.ClusterDNS
func (kl *Kubelet) setupDNSinContainerizedMounter(mounterPath string) {
resolvePath := filepath.Join(strings.TrimSuffix(mounterPath, "/mounter"), "rootfs", "etc", "resolv.conf")
dnsString := ""
for _, dns := range kl.clusterDNS {
dnsString = dnsString + fmt.Sprintf("nameserver %s\n", dns)
}
if kl.resolverConfig != "" {
f, err := os.Open(kl.resolverConfig)
defer f.Close()
if err != nil {
glog.Error("Could not open resolverConf file")
} else {
_, hostSearch, _, err := kl.parseResolvConf(f)
if err != nil {
glog.Errorf("Error for parsing the reslov.conf file: %v", err)
} else {
dnsString = dnsString + "search"
for _, search := range hostSearch {
dnsString = dnsString + fmt.Sprintf(" %s", search)
}
dnsString = dnsString + "\n"
}
}
}
if err := ioutil.WriteFile(resolvePath, []byte(dnsString), 0600); err != nil {
glog.Errorf("Could not write dns nameserver in file %s, with error %v", resolvePath, err)
}
}

// syncNetworkStatus updates the network state
func (kl *Kubelet) syncNetworkStatus() {
// For cri integration, network state will be updated in updateRuntimeUp,
Expand Down Expand Up @@ -536,3 +280,11 @@ func getIPTablesMark(bit int) string {
value := 1 << uint(bit)
return fmt.Sprintf("%#08x/%#08x", value, value)
}

// GetClusterDNS returns a list of the DNS servers, a list of the DNS search
// domains of the cluster, and a list of resolv.conf options.
// This function is defined in kubecontainer.RuntimeHelper interface so we
// have to implement it.
func (kl *Kubelet) GetClusterDNS(pod *v1.Pod) ([]string, []string, []string, bool, error) {
return kl.dnsConfigurer.GetClusterDNS(pod)
}
Loading

0 comments on commit 6e950cc

Please sign in to comment.