dns.go

Documentation: k8s.io/kubernetes/pkg/kubelet/network/dns

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package dns
    18  
    19  import (
    20  	"fmt"
    21  	"io"
    22  	"net"
    23  	"os"
    24  	"path/filepath"
    25  	"strings"
    26  
    27  	v1 "k8s.io/api/core/v1"
    28  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    29  	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
    30  	"k8s.io/client-go/tools/record"
    31  	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
    32  	"k8s.io/kubernetes/pkg/apis/core/validation"
    33  	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
    34  	"k8s.io/kubernetes/pkg/kubelet/util/format"
    35  
    36  	"k8s.io/klog/v2"
    37  	utilio "k8s.io/utils/io"
    38  	utilnet "k8s.io/utils/net"
    39  )
    40  
    41  var (
    42  	// The default dns opt strings.
    43  	defaultDNSOptions = []string{"ndots:5"}
    44  )
    45  
    46  type podDNSType int
    47  
    48  const (
    49  	podDNSCluster podDNSType = iota
    50  	podDNSHost
    51  	podDNSNone
    52  )
    53  
    54  const (
    55  	maxResolvConfLength = 10 * 1 << 20 // 10MB
    56  )
    57  
    58  // Configurer is used for setting up DNS resolver configuration when launching pods.
    59  type Configurer struct {
    60  	recorder         record.EventRecorder
    61  	getHostDNSConfig func(string) (*runtimeapi.DNSConfig, error)
    62  	nodeRef          *v1.ObjectReference
    63  	nodeIPs          []net.IP
    64  
    65  	// If non-nil, use this for container DNS server.
    66  	clusterDNS []net.IP
    67  	// If non-empty, use this for container DNS search.
    68  	ClusterDomain string
    69  	// The path to the DNS resolver configuration file used as the base to generate
    70  	// the container's DNS resolver configuration file. This can be used in
    71  	// conjunction with clusterDomain and clusterDNS.
    72  	ResolverConfig string
    73  }
    74  
    75  // NewConfigurer returns a DNS configurer for launching pods.
    76  func NewConfigurer(recorder record.EventRecorder, nodeRef *v1.ObjectReference, nodeIPs []net.IP, clusterDNS []net.IP, clusterDomain, resolverConfig string) *Configurer {
    77  	return &Configurer{
    78  		recorder:         recorder,
    79  		getHostDNSConfig: getHostDNSConfig,
    80  		nodeRef:          nodeRef,
    81  		nodeIPs:          nodeIPs,
    82  		clusterDNS:       clusterDNS,
    83  		ClusterDomain:    clusterDomain,
    84  		ResolverConfig:   resolverConfig,
    85  	}
    86  }
    87  
    88  func omitDuplicates(strs []string) []string {
    89  	uniqueStrs := make(map[string]bool)
    90  
    91  	var ret []string
    92  	for _, str := range strs {
    93  		if !uniqueStrs[str] {
    94  			ret = append(ret, str)
    95  			uniqueStrs[str] = true
    96  		}
    97  	}
    98  	return ret
    99  }
   100  
   101  func (c *Configurer) formDNSSearchFitsLimits(composedSearch []string, pod *v1.Pod) []string {
   102  	limitsExceeded := false
   103  
   104  	maxDNSSearchPaths, maxDNSSearchListChars := validation.MaxDNSSearchPaths, validation.MaxDNSSearchListChars
   105  
   106  	if len(composedSearch) > maxDNSSearchPaths {
   107  		composedSearch = composedSearch[:maxDNSSearchPaths]
   108  		limitsExceeded = true
   109  	}
   110  
   111  	// In some DNS resolvers(e.g. glibc 2.28), DNS resolving causes abort() if there is a
   112  	// search path exceeding 255 characters. We have to filter them out.
   113  	l := 0
   114  	for _, search := range composedSearch {
   115  		if len(search) > utilvalidation.DNS1123SubdomainMaxLength {
   116  			limitsExceeded = true
   117  			continue
   118  		}
   119  		composedSearch[l] = search
   120  		l++
   121  	}
   122  	composedSearch = composedSearch[:l]
   123  
   124  	if resolvSearchLineStrLen := len(strings.Join(composedSearch, " ")); resolvSearchLineStrLen > maxDNSSearchListChars {
   125  		cutDomainsNum := 0
   126  		cutDomainsLen := 0
   127  		for i := len(composedSearch) - 1; i >= 0; i-- {
   128  			cutDomainsLen += len(composedSearch[i]) + 1
   129  			cutDomainsNum++
   130  
   131  			if (resolvSearchLineStrLen - cutDomainsLen) <= maxDNSSearchListChars {
   132  				break
   133  			}
   134  		}
   135  
   136  		composedSearch = composedSearch[:(len(composedSearch) - cutDomainsNum)]
   137  		limitsExceeded = true
   138  	}
   139  
   140  	if limitsExceeded {
   141  		err := fmt.Errorf("Search Line limits were exceeded, some search paths have been omitted, the applied search line is: %s", strings.Join(composedSearch, " "))
   142  		c.recorder.Event(pod, v1.EventTypeWarning, "DNSConfigForming", err.Error())
   143  		klog.ErrorS(err, "Search Line limits exceeded")
   144  	}
   145  	return composedSearch
   146  }
   147  
   148  func (c *Configurer) formDNSNameserversFitsLimits(nameservers []string, pod *v1.Pod) []string {
   149  	if len(nameservers) > validation.MaxDNSNameservers {
   150  		nameservers = nameservers[0:validation.MaxDNSNameservers]
   151  		err := fmt.Errorf("Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: %s", strings.Join(nameservers, " "))
   152  		c.recorder.Event(pod, v1.EventTypeWarning, "DNSConfigForming", err.Error())
   153  		klog.ErrorS(err, "Nameserver limits exceeded")
   154  	}
   155  	return nameservers
   156  }
   157  
   158  func (c *Configurer) formDNSConfigFitsLimits(dnsConfig *runtimeapi.DNSConfig, pod *v1.Pod) *runtimeapi.DNSConfig {
   159  	dnsConfig.Servers = c.formDNSNameserversFitsLimits(dnsConfig.Servers, pod)
   160  	dnsConfig.Searches = c.formDNSSearchFitsLimits(dnsConfig.Searches, pod)
   161  	return dnsConfig
   162  }
   163  
   164  func (c *Configurer) generateSearchesForDNSClusterFirst(hostSearch []string, pod *v1.Pod) []string {
   165  	if c.ClusterDomain == "" {
   166  		return hostSearch
   167  	}
   168  
   169  	nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, c.ClusterDomain)
   170  	svcDomain := fmt.Sprintf("svc.%s", c.ClusterDomain)
   171  	clusterSearch := []string{nsSvcDomain, svcDomain, c.ClusterDomain}
   172  
   173  	return omitDuplicates(append(clusterSearch, hostSearch...))
   174  }
   175  
   176  // CheckLimitsForResolvConf checks limits in resolv.conf.
   177  func (c *Configurer) CheckLimitsForResolvConf() {
   178  	f, err := os.Open(c.ResolverConfig)
   179  	if err != nil {
   180  		c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", err.Error())
   181  		klog.V(4).InfoS("Check limits for resolv.conf failed at file open", "err", err)
   182  		return
   183  	}
   184  	defer f.Close()
   185  
   186  	_, hostSearch, _, err := parseResolvConf(f)
   187  	if err != nil {
   188  		c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", err.Error())
   189  		klog.V(4).InfoS("Check limits for resolv.conf failed at parse resolv.conf", "err", err)
   190  		return
   191  	}
   192  
   193  	domainCountLimit, maxDNSSearchListChars := validation.MaxDNSSearchPaths, validation.MaxDNSSearchListChars
   194  
   195  	if c.ClusterDomain != "" {
   196  		domainCountLimit -= 3
   197  	}
   198  
   199  	if len(hostSearch) > domainCountLimit {
   200  		log := fmt.Sprintf("Resolv.conf file '%s' contains search line consisting of more than %d domains!", c.ResolverConfig, domainCountLimit)
   201  		c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", log)
   202  		klog.V(4).InfoS("Check limits for resolv.conf failed", "eventlog", log)
   203  		return
   204  	}
   205  
   206  	for _, search := range hostSearch {
   207  		if len(search) > utilvalidation.DNS1123SubdomainMaxLength {
   208  			log := fmt.Sprintf("Resolv.conf file %q contains a search path which length is more than allowed %d chars!", c.ResolverConfig, utilvalidation.DNS1123SubdomainMaxLength)
   209  			c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", log)
   210  			klog.V(4).InfoS("Check limits for resolv.conf failed", "eventlog", log)
   211  			return
   212  		}
   213  	}
   214  
   215  	if len(strings.Join(hostSearch, " ")) > maxDNSSearchListChars {
   216  		log := fmt.Sprintf("Resolv.conf file '%s' contains search line which length is more than allowed %d chars!", c.ResolverConfig, maxDNSSearchListChars)
   217  		c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", log)
   218  		klog.V(4).InfoS("Check limits for resolv.conf failed", "eventlog", log)
   219  		return
   220  	}
   221  }
   222  
   223  // parseResolvConf reads a resolv.conf file from the given reader, and parses
   224  // it into nameservers, searches and options, possibly returning an error.
   225  func parseResolvConf(reader io.Reader) (nameservers []string, searches []string, options []string, err error) {
   226  	file, err := utilio.ReadAtMost(reader, maxResolvConfLength)
   227  	if err != nil {
   228  		return nil, nil, nil, err
   229  	}
   230  
   231  	// Lines of the form "nameserver 1.2.3.4" accumulate.
   232  	nameservers = []string{}
   233  
   234  	// Lines of the form "search example.com" overrule - last one wins.
   235  	searches = []string{}
   236  
   237  	// Lines of the form "option ndots:5 attempts:2" overrule - last one wins.
   238  	// Each option is recorded as an element in the array.
   239  	options = []string{}
   240  
   241  	var allErrors []error
   242  	lines := strings.Split(string(file), "\n")
   243  	for l := range lines {
   244  		trimmed := strings.TrimSpace(lines[l])
   245  		if strings.HasPrefix(trimmed, "#") {
   246  			continue
   247  		}
   248  		fields := strings.Fields(trimmed)
   249  		if len(fields) == 0 {
   250  			continue
   251  		}
   252  		if fields[0] == "nameserver" {
   253  			if len(fields) >= 2 {
   254  				nameservers = append(nameservers, fields[1])
   255  			} else {
   256  				allErrors = append(allErrors, fmt.Errorf("nameserver list is empty "))
   257  			}
   258  		}
   259  		if fields[0] == "search" {
   260  			// Normalise search fields so the same domain with and without trailing dot will only count once, to avoid hitting search validation limits.
   261  			searches = []string{}
   262  			for _, s := range fields[1:] {
   263  				if s != "." {
   264  					searches = append(searches, strings.TrimSuffix(s, "."))
   265  				}
   266  			}
   267  		}
   268  		if fields[0] == "options" {
   269  			options = appendOptions(options, fields[1:]...)
   270  		}
   271  	}
   272  
   273  	return nameservers, searches, options, utilerrors.NewAggregate(allErrors)
   274  }
   275  
   276  // Reads a resolv.conf-like file and returns the DNS config options from it.
   277  // Returns an empty DNSConfig if the given resolverConfigFile is an empty string.
   278  func getDNSConfig(resolverConfigFile string) (*runtimeapi.DNSConfig, error) {
   279  	var hostDNS, hostSearch, hostOptions []string
   280  	// Get host DNS settings
   281  	if resolverConfigFile != "" {
   282  		f, err := os.Open(resolverConfigFile)
   283  		if err != nil {
   284  			klog.ErrorS(err, "Could not open resolv conf file.")
   285  			return nil, err
   286  		}
   287  		defer f.Close()
   288  
   289  		hostDNS, hostSearch, hostOptions, err = parseResolvConf(f)
   290  		if err != nil {
   291  			err := fmt.Errorf("Encountered error while parsing resolv conf file. Error: %w", err)
   292  			klog.ErrorS(err, "Could not parse resolv conf file.")
   293  			return nil, err
   294  		}
   295  	}
   296  	return &runtimeapi.DNSConfig{
   297  		Servers:  hostDNS,
   298  		Searches: hostSearch,
   299  		Options:  hostOptions,
   300  	}, nil
   301  }
   302  
   303  func getPodDNSType(pod *v1.Pod) (podDNSType, error) {
   304  	dnsPolicy := pod.Spec.DNSPolicy
   305  	switch dnsPolicy {
   306  	case v1.DNSNone:
   307  		return podDNSNone, nil
   308  	case v1.DNSClusterFirstWithHostNet:
   309  		return podDNSCluster, nil
   310  	case v1.DNSClusterFirst:
   311  		if !kubecontainer.IsHostNetworkPod(pod) {
   312  			return podDNSCluster, nil
   313  		}
   314  		// Fallback to DNSDefault for pod on hostnetwork.
   315  		fallthrough
   316  	case v1.DNSDefault:
   317  		return podDNSHost, nil
   318  	}
   319  	// This should not happen as kube-apiserver should have rejected
   320  	// invalid dnsPolicy.
   321  	return podDNSCluster, fmt.Errorf("invalid DNSPolicy=%v", dnsPolicy)
   322  }
   323  
   324  // mergeDNSOptions merges DNS options. If duplicated, entries given by PodDNSConfigOption will
   325  // overwrite the existing ones.
   326  func mergeDNSOptions(existingDNSConfigOptions []string, dnsConfigOptions []v1.PodDNSConfigOption) []string {
   327  	optionsMap := make(map[string]string)
   328  	for _, op := range existingDNSConfigOptions {
   329  		if index := strings.Index(op, ":"); index != -1 {
   330  			optionsMap[op[:index]] = op[index+1:]
   331  		} else {
   332  			optionsMap[op] = ""
   333  		}
   334  	}
   335  	for _, op := range dnsConfigOptions {
   336  		if op.Value != nil {
   337  			optionsMap[op.Name] = *op.Value
   338  		} else {
   339  			optionsMap[op.Name] = ""
   340  		}
   341  	}
   342  	// Reconvert DNS options into a string array.
   343  	options := []string{}
   344  	for opName, opValue := range optionsMap {
   345  		op := opName
   346  		if opValue != "" {
   347  			op = op + ":" + opValue
   348  		}
   349  		options = append(options, op)
   350  	}
   351  	return options
   352  }
   353  
   354  // appendOptions appends options to the given list, but does not add duplicates.
   355  // append option will overwrite the previous one either in new line or in the same line.
   356  func appendOptions(options []string, newOption ...string) []string {
   357  	var optionMap = make(map[string]string)
   358  	for _, option := range options {
   359  		optName := strings.Split(option, ":")[0]
   360  		optionMap[optName] = option
   361  	}
   362  	for _, option := range newOption {
   363  		optName := strings.Split(option, ":")[0]
   364  		optionMap[optName] = option
   365  	}
   366  
   367  	options = []string{}
   368  	for _, v := range optionMap {
   369  		options = append(options, v)
   370  	}
   371  	return options
   372  }
   373  
   374  // appendDNSConfig appends DNS servers, search paths and options given by
   375  // PodDNSConfig to the existing DNS config. Duplicated entries will be merged.
   376  // This assumes existingDNSConfig and dnsConfig are not nil.
   377  func appendDNSConfig(existingDNSConfig *runtimeapi.DNSConfig, dnsConfig *v1.PodDNSConfig) *runtimeapi.DNSConfig {
   378  	existingDNSConfig.Servers = omitDuplicates(append(existingDNSConfig.Servers, dnsConfig.Nameservers...))
   379  	existingDNSConfig.Searches = omitDuplicates(append(existingDNSConfig.Searches, dnsConfig.Searches...))
   380  	existingDNSConfig.Options = mergeDNSOptions(existingDNSConfig.Options, dnsConfig.Options)
   381  	return existingDNSConfig
   382  }
   383  
   384  // GetPodDNS returns DNS settings for the pod.
   385  func (c *Configurer) GetPodDNS(pod *v1.Pod) (*runtimeapi.DNSConfig, error) {
   386  	dnsConfig, err := c.getHostDNSConfig(c.ResolverConfig)
   387  	if err != nil {
   388  		return nil, err
   389  	}
   390  
   391  	dnsType, err := getPodDNSType(pod)
   392  	if err != nil {
   393  		klog.ErrorS(err, "Failed to get DNS type for pod. Falling back to DNSClusterFirst policy.", "pod", klog.KObj(pod))
   394  		dnsType = podDNSCluster
   395  	}
   396  	switch dnsType {
   397  	case podDNSNone:
   398  		// DNSNone should use empty DNS settings as the base.
   399  		dnsConfig = &runtimeapi.DNSConfig{}
   400  	case podDNSCluster:
   401  		if len(c.clusterDNS) != 0 {
   402  			// For a pod with DNSClusterFirst policy, the cluster DNS server is
   403  			// the only nameserver configured for the pod. The cluster DNS server
   404  			// itself will forward queries to other nameservers that is configured
   405  			// to use, in case the cluster DNS server cannot resolve the DNS query
   406  			// itself.
   407  			dnsConfig.Servers = []string{}
   408  			for _, ip := range c.clusterDNS {
   409  				dnsConfig.Servers = append(dnsConfig.Servers, ip.String())
   410  			}
   411  			dnsConfig.Searches = c.generateSearchesForDNSClusterFirst(dnsConfig.Searches, pod)
   412  			dnsConfig.Options = defaultDNSOptions
   413  			break
   414  		}
   415  		// clusterDNS is not known. Pod with ClusterDNSFirst Policy cannot be created.
   416  		nodeErrorMsg := fmt.Sprintf("kubelet does not have ClusterDNS IP configured and cannot create Pod using %q policy. Falling back to %q policy.", v1.DNSClusterFirst, v1.DNSDefault)
   417  		c.recorder.Eventf(c.nodeRef, v1.EventTypeWarning, "MissingClusterDNS", nodeErrorMsg)
   418  		c.recorder.Eventf(pod, v1.EventTypeWarning, "MissingClusterDNS", "pod: %q. %s", format.Pod(pod), nodeErrorMsg)
   419  		// Fallback to DNSDefault.
   420  		fallthrough
   421  	case podDNSHost:
   422  		// When the kubelet --resolv-conf flag is set to the empty string, use
   423  		// DNS settings that override the docker default (which is to use
   424  		// /etc/resolv.conf) and effectively disable DNS lookups. According to
   425  		// the bind documentation, the behavior of the DNS client library when
   426  		// "nameservers" are not specified is to "use the nameserver on the
   427  		// local machine". A nameserver setting of localhost is equivalent to
   428  		// this documented behavior.
   429  		if c.ResolverConfig == "" {
   430  			for _, nodeIP := range c.nodeIPs {
   431  				if utilnet.IsIPv6(nodeIP) {
   432  					dnsConfig.Servers = append(dnsConfig.Servers, "::1")
   433  				} else {
   434  					dnsConfig.Servers = append(dnsConfig.Servers, "127.0.0.1")
   435  				}
   436  			}
   437  			if len(dnsConfig.Servers) == 0 {
   438  				dnsConfig.Servers = append(dnsConfig.Servers, "127.0.0.1")
   439  			}
   440  			dnsConfig.Searches = []string{"."}
   441  		}
   442  	}
   443  
   444  	if pod.Spec.DNSConfig != nil {
   445  		dnsConfig = appendDNSConfig(dnsConfig, pod.Spec.DNSConfig)
   446  	}
   447  	return c.formDNSConfigFitsLimits(dnsConfig, pod), nil
   448  }
   449  
   450  // SetupDNSinContainerizedMounter replaces the nameserver in containerized-mounter's rootfs/etc/resolv.conf with kubelet.ClusterDNS
   451  func (c *Configurer) SetupDNSinContainerizedMounter(mounterPath string) {
   452  	resolvePath := filepath.Join(strings.TrimSuffix(mounterPath, "/mounter"), "rootfs", "etc", "resolv.conf")
   453  	dnsString := ""
   454  	for _, dns := range c.clusterDNS {
   455  		dnsString = dnsString + fmt.Sprintf("nameserver %s\n", dns)
   456  	}
   457  	if c.ResolverConfig != "" {
   458  		f, err := os.Open(c.ResolverConfig)
   459  		if err != nil {
   460  			klog.ErrorS(err, "Could not open resolverConf file")
   461  		} else {
   462  			defer f.Close()
   463  			_, hostSearch, _, err := parseResolvConf(f)
   464  			if err != nil {
   465  				klog.ErrorS(err, "Error for parsing the resolv.conf file")
   466  			} else {
   467  				dnsString = dnsString + "search"
   468  				for _, search := range hostSearch {
   469  					dnsString = dnsString + fmt.Sprintf(" %s", search)
   470  				}
   471  				dnsString = dnsString + "\n"
   472  			}
   473  		}
   474  	}
   475  	if err := os.WriteFile(resolvePath, []byte(dnsString), 0600); err != nil {
   476  		klog.ErrorS(err, "Could not write dns nameserver in the file", "path", resolvePath)
   477  	}
   478  }
   479
View as plain text