resource.go

Documentation: k8s.io/kubernetes/test/e2e/framework/node

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package node
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"net"
    24  	"strings"
    25  	"time"
    26  
    27  	"github.com/onsi/ginkgo/v2"
    28  	"github.com/onsi/gomega"
    29  
    30  	v1 "k8s.io/api/core/v1"
    31  	"k8s.io/apimachinery/pkg/api/resource"
    32  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    33  	"k8s.io/apimachinery/pkg/conversion"
    34  	"k8s.io/apimachinery/pkg/fields"
    35  	"k8s.io/apimachinery/pkg/labels"
    36  	"k8s.io/apimachinery/pkg/types"
    37  	"k8s.io/apimachinery/pkg/util/rand"
    38  	"k8s.io/apimachinery/pkg/util/sets"
    39  	"k8s.io/apimachinery/pkg/util/strategicpatch"
    40  	"k8s.io/apimachinery/pkg/util/wait"
    41  	clientset "k8s.io/client-go/kubernetes"
    42  	clientretry "k8s.io/client-go/util/retry"
    43  	"k8s.io/kubernetes/test/e2e/framework"
    44  	netutil "k8s.io/utils/net"
    45  )
    46  
    47  const (
    48  	// poll is how often to Poll pods, nodes and claims.
    49  	poll = 2 * time.Second
    50  
    51  	// singleCallTimeout is how long to try single API calls (like 'get' or 'list'). Used to prevent
    52  	// transient failures from failing tests.
    53  	singleCallTimeout = 5 * time.Minute
    54  
    55  	// ssh port
    56  	sshPort = "22"
    57  )
    58  
    59  var (
    60  	// unreachableTaintTemplate is the taint for when a node becomes unreachable.
    61  	// Copied from pkg/controller/nodelifecycle to avoid pulling extra dependencies
    62  	unreachableTaintTemplate = &v1.Taint{
    63  		Key:    v1.TaintNodeUnreachable,
    64  		Effect: v1.TaintEffectNoExecute,
    65  	}
    66  
    67  	// notReadyTaintTemplate is the taint for when a node is not ready for executing pods.
    68  	// Copied from pkg/controller/nodelifecycle to avoid pulling extra dependencies
    69  	notReadyTaintTemplate = &v1.Taint{
    70  		Key:    v1.TaintNodeNotReady,
    71  		Effect: v1.TaintEffectNoExecute,
    72  	}
    73  
    74  	// updateTaintBackOff contains the maximum retries and the wait interval between two retries.
    75  	updateTaintBackOff = wait.Backoff{
    76  		Steps:    5,
    77  		Duration: 100 * time.Millisecond,
    78  		Jitter:   1.0,
    79  	}
    80  )
    81  
    82  // PodNode is a pod-node pair indicating which node a given pod is running on
    83  type PodNode struct {
    84  	// Pod represents pod name
    85  	Pod string
    86  	// Node represents node name
    87  	Node string
    88  }
    89  
    90  // FirstAddress returns the first address of the given type of each node.
    91  func FirstAddress(nodelist *v1.NodeList, addrType v1.NodeAddressType) string {
    92  	for _, n := range nodelist.Items {
    93  		for _, addr := range n.Status.Addresses {
    94  			if addr.Type == addrType && addr.Address != "" {
    95  				return addr.Address
    96  			}
    97  		}
    98  	}
    99  	return ""
   100  }
   101  
   102  func isNodeConditionSetAsExpected(node *v1.Node, conditionType v1.NodeConditionType, wantTrue, silent bool) bool {
   103  	// Check the node readiness condition (logging all).
   104  	for _, cond := range node.Status.Conditions {
   105  		// Ensure that the condition type and the status matches as desired.
   106  		if cond.Type == conditionType {
   107  			// For NodeReady condition we need to check Taints as well
   108  			if cond.Type == v1.NodeReady {
   109  				hasNodeControllerTaints := false
   110  				// For NodeReady we need to check if Taints are gone as well
   111  				taints := node.Spec.Taints
   112  				for _, taint := range taints {
   113  					if taint.MatchTaint(unreachableTaintTemplate) || taint.MatchTaint(notReadyTaintTemplate) {
   114  						hasNodeControllerTaints = true
   115  						break
   116  					}
   117  				}
   118  				if wantTrue {
   119  					if (cond.Status == v1.ConditionTrue) && !hasNodeControllerTaints {
   120  						return true
   121  					}
   122  					msg := ""
   123  					if !hasNodeControllerTaints {
   124  						msg = fmt.Sprintf("Condition %s of node %s is %v instead of %t. Reason: %v, message: %v",
   125  							conditionType, node.Name, cond.Status == v1.ConditionTrue, wantTrue, cond.Reason, cond.Message)
   126  					} else {
   127  						msg = fmt.Sprintf("Condition %s of node %s is %v, but Node is tainted by NodeController with %v. Failure",
   128  							conditionType, node.Name, cond.Status == v1.ConditionTrue, taints)
   129  					}
   130  					if !silent {
   131  						framework.Logf(msg)
   132  					}
   133  					return false
   134  				}
   135  				// TODO: check if the Node is tainted once we enable NC notReady/unreachable taints by default
   136  				if cond.Status != v1.ConditionTrue {
   137  					return true
   138  				}
   139  				if !silent {
   140  					framework.Logf("Condition %s of node %s is %v instead of %t. Reason: %v, message: %v",
   141  						conditionType, node.Name, cond.Status == v1.ConditionTrue, wantTrue, cond.Reason, cond.Message)
   142  				}
   143  				return false
   144  			}
   145  			if (wantTrue && (cond.Status == v1.ConditionTrue)) || (!wantTrue && (cond.Status != v1.ConditionTrue)) {
   146  				return true
   147  			}
   148  			if !silent {
   149  				framework.Logf("Condition %s of node %s is %v instead of %t. Reason: %v, message: %v",
   150  					conditionType, node.Name, cond.Status == v1.ConditionTrue, wantTrue, cond.Reason, cond.Message)
   151  			}
   152  			return false
   153  		}
   154  
   155  	}
   156  	if !silent {
   157  		framework.Logf("Couldn't find condition %v on node %v", conditionType, node.Name)
   158  	}
   159  	return false
   160  }
   161  
   162  // IsConditionSetAsExpected returns a wantTrue value if the node has a match to the conditionType, otherwise returns an opposite value of the wantTrue with detailed logging.
   163  func IsConditionSetAsExpected(node *v1.Node, conditionType v1.NodeConditionType, wantTrue bool) bool {
   164  	return isNodeConditionSetAsExpected(node, conditionType, wantTrue, false)
   165  }
   166  
   167  // IsConditionSetAsExpectedSilent returns a wantTrue value if the node has a match to the conditionType, otherwise returns an opposite value of the wantTrue.
   168  func IsConditionSetAsExpectedSilent(node *v1.Node, conditionType v1.NodeConditionType, wantTrue bool) bool {
   169  	return isNodeConditionSetAsExpected(node, conditionType, wantTrue, true)
   170  }
   171  
   172  // isConditionUnset returns true if conditions of the given node do not have a match to the given conditionType, otherwise false.
   173  func isConditionUnset(node *v1.Node, conditionType v1.NodeConditionType) bool {
   174  	for _, cond := range node.Status.Conditions {
   175  		if cond.Type == conditionType {
   176  			return false
   177  		}
   178  	}
   179  	return true
   180  }
   181  
   182  // Filter filters nodes in NodeList in place, removing nodes that do not
   183  // satisfy the given condition
   184  func Filter(nodeList *v1.NodeList, fn func(node v1.Node) bool) {
   185  	var l []v1.Node
   186  
   187  	for _, node := range nodeList.Items {
   188  		if fn(node) {
   189  			l = append(l, node)
   190  		}
   191  	}
   192  	nodeList.Items = l
   193  }
   194  
   195  // TotalRegistered returns number of schedulable Nodes.
   196  func TotalRegistered(ctx context.Context, c clientset.Interface) (int, error) {
   197  	nodes, err := waitListSchedulableNodes(ctx, c)
   198  	if err != nil {
   199  		framework.Logf("Failed to list nodes: %v", err)
   200  		return 0, err
   201  	}
   202  	return len(nodes.Items), nil
   203  }
   204  
   205  // TotalReady returns number of ready schedulable Nodes.
   206  func TotalReady(ctx context.Context, c clientset.Interface) (int, error) {
   207  	nodes, err := waitListSchedulableNodes(ctx, c)
   208  	if err != nil {
   209  		framework.Logf("Failed to list nodes: %v", err)
   210  		return 0, err
   211  	}
   212  
   213  	// Filter out not-ready nodes.
   214  	Filter(nodes, func(node v1.Node) bool {
   215  		return IsConditionSetAsExpected(&node, v1.NodeReady, true)
   216  	})
   217  	return len(nodes.Items), nil
   218  }
   219  
   220  // GetSSHExternalIP returns node external IP concatenated with port 22 for ssh
   221  // e.g. 1.2.3.4:22
   222  func GetSSHExternalIP(node *v1.Node) (string, error) {
   223  	framework.Logf("Getting external IP address for %s", node.Name)
   224  
   225  	for _, a := range node.Status.Addresses {
   226  		if a.Type == v1.NodeExternalIP && a.Address != "" {
   227  			return net.JoinHostPort(a.Address, sshPort), nil
   228  		}
   229  	}
   230  	return "", fmt.Errorf("Couldn't get the external IP of host %s with addresses %v", node.Name, node.Status.Addresses)
   231  }
   232  
   233  // GetSSHInternalIP returns node internal IP concatenated with port 22 for ssh
   234  func GetSSHInternalIP(node *v1.Node) (string, error) {
   235  	for _, address := range node.Status.Addresses {
   236  		if address.Type == v1.NodeInternalIP && address.Address != "" {
   237  			return net.JoinHostPort(address.Address, sshPort), nil
   238  		}
   239  	}
   240  
   241  	return "", fmt.Errorf("Couldn't get the internal IP of host %s with addresses %v", node.Name, node.Status.Addresses)
   242  }
   243  
   244  // FirstAddressByTypeAndFamily returns the first address that matches the given type and family of the list of nodes
   245  func FirstAddressByTypeAndFamily(nodelist *v1.NodeList, addrType v1.NodeAddressType, family v1.IPFamily) string {
   246  	for _, n := range nodelist.Items {
   247  		addresses := GetAddressesByTypeAndFamily(&n, addrType, family)
   248  		if len(addresses) > 0 {
   249  			return addresses[0]
   250  		}
   251  	}
   252  	return ""
   253  }
   254  
   255  // GetAddressesByTypeAndFamily returns a list of addresses of the given addressType for the given node
   256  // and filtered by IPFamily
   257  func GetAddressesByTypeAndFamily(node *v1.Node, addressType v1.NodeAddressType, family v1.IPFamily) (ips []string) {
   258  	for _, nodeAddress := range node.Status.Addresses {
   259  		if nodeAddress.Type != addressType {
   260  			continue
   261  		}
   262  		if nodeAddress.Address == "" {
   263  			continue
   264  		}
   265  		if family == v1.IPv6Protocol && netutil.IsIPv6String(nodeAddress.Address) {
   266  			ips = append(ips, nodeAddress.Address)
   267  		}
   268  		if family == v1.IPv4Protocol && !netutil.IsIPv6String(nodeAddress.Address) {
   269  			ips = append(ips, nodeAddress.Address)
   270  		}
   271  	}
   272  	return
   273  }
   274  
   275  // GetAddresses returns a list of addresses of the given addressType for the given node
   276  func GetAddresses(node *v1.Node, addressType v1.NodeAddressType) (ips []string) {
   277  	for j := range node.Status.Addresses {
   278  		nodeAddress := &node.Status.Addresses[j]
   279  		if nodeAddress.Type == addressType && nodeAddress.Address != "" {
   280  			ips = append(ips, nodeAddress.Address)
   281  		}
   282  	}
   283  	return
   284  }
   285  
   286  // CollectAddresses returns a list of addresses of the given addressType for the given list of nodes
   287  func CollectAddresses(nodes *v1.NodeList, addressType v1.NodeAddressType) []string {
   288  	ips := []string{}
   289  	for i := range nodes.Items {
   290  		ips = append(ips, GetAddresses(&nodes.Items[i], addressType)...)
   291  	}
   292  	return ips
   293  }
   294  
   295  // PickIP picks one public node IP
   296  func PickIP(ctx context.Context, c clientset.Interface) (string, error) {
   297  	publicIps, err := GetPublicIps(ctx, c)
   298  	if err != nil {
   299  		return "", fmt.Errorf("get node public IPs error: %w", err)
   300  	}
   301  	if len(publicIps) == 0 {
   302  		return "", fmt.Errorf("got unexpected number (%d) of public IPs", len(publicIps))
   303  	}
   304  	ip := publicIps[0]
   305  	return ip, nil
   306  }
   307  
   308  // GetPublicIps returns a public IP list of nodes.
   309  func GetPublicIps(ctx context.Context, c clientset.Interface) ([]string, error) {
   310  	nodes, err := GetReadySchedulableNodes(ctx, c)
   311  	if err != nil {
   312  		return nil, fmt.Errorf("get schedulable and ready nodes error: %w", err)
   313  	}
   314  	ips := CollectAddresses(nodes, v1.NodeExternalIP)
   315  	if len(ips) == 0 {
   316  		// If ExternalIP isn't set, assume the test programs can reach the InternalIP
   317  		ips = CollectAddresses(nodes, v1.NodeInternalIP)
   318  	}
   319  	return ips, nil
   320  }
   321  
   322  // GetReadySchedulableNodes addresses the common use case of getting nodes you can do work on.
   323  // 1) Needs to be schedulable.
   324  // 2) Needs to be ready.
   325  // If EITHER 1 or 2 is not true, most tests will want to ignore the node entirely.
   326  // If there are no nodes that are both ready and schedulable, this will return an error.
   327  func GetReadySchedulableNodes(ctx context.Context, c clientset.Interface) (nodes *v1.NodeList, err error) {
   328  	nodes, err = checkWaitListSchedulableNodes(ctx, c)
   329  	if err != nil {
   330  		return nil, fmt.Errorf("listing schedulable nodes error: %w", err)
   331  	}
   332  	Filter(nodes, func(node v1.Node) bool {
   333  		return IsNodeSchedulable(&node) && isNodeUntainted(&node)
   334  	})
   335  	if len(nodes.Items) == 0 {
   336  		return nil, fmt.Errorf("there are currently no ready, schedulable nodes in the cluster")
   337  	}
   338  	return nodes, nil
   339  }
   340  
   341  // GetBoundedReadySchedulableNodes is like GetReadySchedulableNodes except that it returns
   342  // at most maxNodes nodes. Use this to keep your test case from blowing up when run on a
   343  // large cluster.
   344  func GetBoundedReadySchedulableNodes(ctx context.Context, c clientset.Interface, maxNodes int) (nodes *v1.NodeList, err error) {
   345  	nodes, err = GetReadySchedulableNodes(ctx, c)
   346  	if err != nil {
   347  		return nil, err
   348  	}
   349  	if len(nodes.Items) > maxNodes {
   350  		shuffled := make([]v1.Node, maxNodes)
   351  		perm := rand.Perm(len(nodes.Items))
   352  		for i, j := range perm {
   353  			if j < len(shuffled) {
   354  				shuffled[j] = nodes.Items[i]
   355  			}
   356  		}
   357  		nodes.Items = shuffled
   358  	}
   359  	return nodes, nil
   360  }
   361  
   362  // GetRandomReadySchedulableNode gets a single randomly-selected node which is available for
   363  // running pods on. If there are no available nodes it will return an error.
   364  func GetRandomReadySchedulableNode(ctx context.Context, c clientset.Interface) (*v1.Node, error) {
   365  	nodes, err := GetReadySchedulableNodes(ctx, c)
   366  	if err != nil {
   367  		return nil, err
   368  	}
   369  	return &nodes.Items[rand.Intn(len(nodes.Items))], nil
   370  }
   371  
   372  // GetReadyNodesIncludingTainted returns all ready nodes, even those which are tainted.
   373  // There are cases when we care about tainted nodes
   374  // E.g. in tests related to nodes with gpu we care about nodes despite
   375  // presence of nvidia.com/gpu=present:NoSchedule taint
   376  func GetReadyNodesIncludingTainted(ctx context.Context, c clientset.Interface) (nodes *v1.NodeList, err error) {
   377  	nodes, err = checkWaitListSchedulableNodes(ctx, c)
   378  	if err != nil {
   379  		return nil, fmt.Errorf("listing schedulable nodes error: %w", err)
   380  	}
   381  	Filter(nodes, func(node v1.Node) bool {
   382  		return IsNodeSchedulable(&node)
   383  	})
   384  	return nodes, nil
   385  }
   386  
   387  // isNodeUntainted tests whether a fake pod can be scheduled on "node", given its current taints.
   388  // TODO: need to discuss wether to return bool and error type
   389  func isNodeUntainted(node *v1.Node) bool {
   390  	return isNodeUntaintedWithNonblocking(node, "")
   391  }
   392  
   393  // isNodeUntaintedWithNonblocking tests whether a fake pod can be scheduled on "node"
   394  // but allows for taints in the list of non-blocking taints.
   395  func isNodeUntaintedWithNonblocking(node *v1.Node, nonblockingTaints string) bool {
   396  	// Simple lookup for nonblocking taints based on comma-delimited list.
   397  	nonblockingTaintsMap := map[string]struct{}{}
   398  	for _, t := range strings.Split(nonblockingTaints, ",") {
   399  		if strings.TrimSpace(t) != "" {
   400  			nonblockingTaintsMap[strings.TrimSpace(t)] = struct{}{}
   401  		}
   402  	}
   403  
   404  	n := node
   405  	if len(nonblockingTaintsMap) > 0 {
   406  		nodeCopy := node.DeepCopy()
   407  		nodeCopy.Spec.Taints = []v1.Taint{}
   408  		for _, v := range node.Spec.Taints {
   409  			if _, isNonblockingTaint := nonblockingTaintsMap[v.Key]; !isNonblockingTaint {
   410  				nodeCopy.Spec.Taints = append(nodeCopy.Spec.Taints, v)
   411  			}
   412  		}
   413  		n = nodeCopy
   414  	}
   415  
   416  	return toleratesTaintsWithNoScheduleNoExecuteEffects(n.Spec.Taints, nil)
   417  }
   418  
   419  func toleratesTaintsWithNoScheduleNoExecuteEffects(taints []v1.Taint, tolerations []v1.Toleration) bool {
   420  	filteredTaints := []v1.Taint{}
   421  	for _, taint := range taints {
   422  		if taint.Effect == v1.TaintEffectNoExecute || taint.Effect == v1.TaintEffectNoSchedule {
   423  			filteredTaints = append(filteredTaints, taint)
   424  		}
   425  	}
   426  
   427  	toleratesTaint := func(taint v1.Taint) bool {
   428  		for _, toleration := range tolerations {
   429  			if toleration.ToleratesTaint(&taint) {
   430  				return true
   431  			}
   432  		}
   433  
   434  		return false
   435  	}
   436  
   437  	for _, taint := range filteredTaints {
   438  		if !toleratesTaint(taint) {
   439  			return false
   440  		}
   441  	}
   442  
   443  	return true
   444  }
   445  
   446  // IsNodeSchedulable returns true if:
   447  // 1) doesn't have "unschedulable" field set
   448  // 2) it also returns true from IsNodeReady
   449  func IsNodeSchedulable(node *v1.Node) bool {
   450  	if node == nil {
   451  		return false
   452  	}
   453  	return !node.Spec.Unschedulable && IsNodeReady(node)
   454  }
   455  
   456  // IsNodeReady returns true if:
   457  // 1) it's Ready condition is set to true
   458  // 2) doesn't have NetworkUnavailable condition set to true
   459  func IsNodeReady(node *v1.Node) bool {
   460  	nodeReady := IsConditionSetAsExpected(node, v1.NodeReady, true)
   461  	networkReady := isConditionUnset(node, v1.NodeNetworkUnavailable) ||
   462  		IsConditionSetAsExpectedSilent(node, v1.NodeNetworkUnavailable, false)
   463  	return nodeReady && networkReady
   464  }
   465  
   466  // isNodeSchedulableWithoutTaints returns true if:
   467  // 1) doesn't have "unschedulable" field set
   468  // 2) it also returns true from IsNodeReady
   469  // 3) it also returns true from isNodeUntainted
   470  func isNodeSchedulableWithoutTaints(node *v1.Node) bool {
   471  	return IsNodeSchedulable(node) && isNodeUntainted(node)
   472  }
   473  
   474  // hasNonblockingTaint returns true if the node contains at least
   475  // one taint with a key matching the regexp.
   476  func hasNonblockingTaint(node *v1.Node, nonblockingTaints string) bool {
   477  	if node == nil {
   478  		return false
   479  	}
   480  
   481  	// Simple lookup for nonblocking taints based on comma-delimited list.
   482  	nonblockingTaintsMap := map[string]struct{}{}
   483  	for _, t := range strings.Split(nonblockingTaints, ",") {
   484  		if strings.TrimSpace(t) != "" {
   485  			nonblockingTaintsMap[strings.TrimSpace(t)] = struct{}{}
   486  		}
   487  	}
   488  
   489  	for _, taint := range node.Spec.Taints {
   490  		if _, hasNonblockingTaint := nonblockingTaintsMap[taint.Key]; hasNonblockingTaint {
   491  			return true
   492  		}
   493  	}
   494  
   495  	return false
   496  }
   497  
   498  // PodNodePairs return podNode pairs for all pods in a namespace
   499  func PodNodePairs(ctx context.Context, c clientset.Interface, ns string) ([]PodNode, error) {
   500  	var result []PodNode
   501  
   502  	podList, err := c.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{})
   503  	if err != nil {
   504  		return result, err
   505  	}
   506  
   507  	for _, pod := range podList.Items {
   508  		result = append(result, PodNode{
   509  			Pod:  pod.Name,
   510  			Node: pod.Spec.NodeName,
   511  		})
   512  	}
   513  
   514  	return result, nil
   515  }
   516  
   517  // GetClusterZones returns the values of zone label collected from all nodes.
   518  func GetClusterZones(ctx context.Context, c clientset.Interface) (sets.String, error) {
   519  	nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
   520  	if err != nil {
   521  		return nil, fmt.Errorf("Error getting nodes while attempting to list cluster zones: %w", err)
   522  	}
   523  
   524  	// collect values of zone label from all nodes
   525  	zones := sets.NewString()
   526  	for _, node := range nodes.Items {
   527  		if zone, found := node.Labels[v1.LabelFailureDomainBetaZone]; found {
   528  			zones.Insert(zone)
   529  		}
   530  
   531  		if zone, found := node.Labels[v1.LabelTopologyZone]; found {
   532  			zones.Insert(zone)
   533  		}
   534  	}
   535  	return zones, nil
   536  }
   537  
   538  // GetSchedulableClusterZones returns the values of zone label collected from all nodes which are schedulable.
   539  func GetSchedulableClusterZones(ctx context.Context, c clientset.Interface) (sets.Set[string], error) {
   540  	// GetReadySchedulableNodes already filters our tainted and unschedulable nodes.
   541  	nodes, err := GetReadySchedulableNodes(ctx, c)
   542  	if err != nil {
   543  		return nil, fmt.Errorf("error getting nodes while attempting to list cluster zones: %w", err)
   544  	}
   545  
   546  	// collect values of zone label from all nodes
   547  	zones := sets.New[string]()
   548  	for _, node := range nodes.Items {
   549  		if zone, found := node.Labels[v1.LabelFailureDomainBetaZone]; found {
   550  			zones.Insert(zone)
   551  		}
   552  
   553  		if zone, found := node.Labels[v1.LabelTopologyZone]; found {
   554  			zones.Insert(zone)
   555  		}
   556  	}
   557  	return zones, nil
   558  }
   559  
   560  // CreatePodsPerNodeForSimpleApp creates pods w/ labels.  Useful for tests which make a bunch of pods w/o any networking.
   561  func CreatePodsPerNodeForSimpleApp(ctx context.Context, c clientset.Interface, namespace, appName string, podSpec func(n v1.Node) v1.PodSpec, maxCount int) map[string]string {
   562  	nodes, err := GetBoundedReadySchedulableNodes(ctx, c, maxCount)
   563  	// TODO use wrapper methods in expect.go after removing core e2e dependency on node
   564  	gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred())
   565  	podLabels := map[string]string{
   566  		"app": appName + "-pod",
   567  	}
   568  	for i, node := range nodes.Items {
   569  		framework.Logf("%v/%v : Creating container with label app=%v-pod", i, maxCount, appName)
   570  		_, err := c.CoreV1().Pods(namespace).Create(ctx, &v1.Pod{
   571  			ObjectMeta: metav1.ObjectMeta{
   572  				Name:   fmt.Sprintf(appName+"-pod-%v", i),
   573  				Labels: podLabels,
   574  			},
   575  			Spec: podSpec(node),
   576  		}, metav1.CreateOptions{})
   577  		// TODO use wrapper methods in expect.go after removing core e2e dependency on node
   578  		gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred())
   579  	}
   580  	return podLabels
   581  }
   582  
   583  // RemoveTaintsOffNode removes a list of taints from the given node
   584  // It is simply a helper wrapper for RemoveTaintOffNode
   585  func RemoveTaintsOffNode(ctx context.Context, c clientset.Interface, nodeName string, taints []v1.Taint) {
   586  	for _, taint := range taints {
   587  		RemoveTaintOffNode(ctx, c, nodeName, taint)
   588  	}
   589  }
   590  
   591  // RemoveTaintOffNode removes the given taint from the given node.
   592  func RemoveTaintOffNode(ctx context.Context, c clientset.Interface, nodeName string, taint v1.Taint) {
   593  	err := removeNodeTaint(ctx, c, nodeName, nil, &taint)
   594  
   595  	// TODO use wrapper methods in expect.go after removing core e2e dependency on node
   596  	gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred())
   597  	verifyThatTaintIsGone(ctx, c, nodeName, &taint)
   598  }
   599  
   600  // AddOrUpdateTaintOnNode adds the given taint to the given node or updates taint.
   601  func AddOrUpdateTaintOnNode(ctx context.Context, c clientset.Interface, nodeName string, taint v1.Taint) {
   602  	// TODO use wrapper methods in expect.go after removing the dependency on this
   603  	// package from the core e2e framework.
   604  	err := addOrUpdateTaintOnNode(ctx, c, nodeName, &taint)
   605  	gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred())
   606  }
   607  
   608  // addOrUpdateTaintOnNode add taints to the node. If taint was added into node, it'll issue API calls
   609  // to update nodes; otherwise, no API calls. Return error if any.
   610  // copied from pkg/controller/controller_utils.go AddOrUpdateTaintOnNode()
   611  func addOrUpdateTaintOnNode(ctx context.Context, c clientset.Interface, nodeName string, taints ...*v1.Taint) error {
   612  	if len(taints) == 0 {
   613  		return nil
   614  	}
   615  	firstTry := true
   616  	return clientretry.RetryOnConflict(updateTaintBackOff, func() error {
   617  		var err error
   618  		var oldNode *v1.Node
   619  		// First we try getting node from the API server cache, as it's cheaper. If it fails
   620  		// we get it from etcd to be sure to have fresh data.
   621  		if firstTry {
   622  			oldNode, err = c.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{ResourceVersion: "0"})
   623  			firstTry = false
   624  		} else {
   625  			oldNode, err = c.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
   626  		}
   627  		if err != nil {
   628  			return err
   629  		}
   630  
   631  		var newNode *v1.Node
   632  		oldNodeCopy := oldNode
   633  		updated := false
   634  		for _, taint := range taints {
   635  			curNewNode, ok, err := addOrUpdateTaint(oldNodeCopy, taint)
   636  			if err != nil {
   637  				return fmt.Errorf("failed to update taint of node")
   638  			}
   639  			updated = updated || ok
   640  			newNode = curNewNode
   641  			oldNodeCopy = curNewNode
   642  		}
   643  		if !updated {
   644  			return nil
   645  		}
   646  		return patchNodeTaints(ctx, c, nodeName, oldNode, newNode)
   647  	})
   648  }
   649  
   650  // addOrUpdateTaint tries to add a taint to annotations list. Returns a new copy of updated Node and true if something was updated
   651  // false otherwise.
   652  // copied from pkg/util/taints/taints.go AddOrUpdateTaint()
   653  func addOrUpdateTaint(node *v1.Node, taint *v1.Taint) (*v1.Node, bool, error) {
   654  	newNode := node.DeepCopy()
   655  	nodeTaints := newNode.Spec.Taints
   656  
   657  	var newTaints []v1.Taint
   658  	updated := false
   659  	for i := range nodeTaints {
   660  		if taint.MatchTaint(&nodeTaints[i]) {
   661  			if semantic.DeepEqual(*taint, nodeTaints[i]) {
   662  				return newNode, false, nil
   663  			}
   664  			newTaints = append(newTaints, *taint)
   665  			updated = true
   666  			continue
   667  		}
   668  
   669  		newTaints = append(newTaints, nodeTaints[i])
   670  	}
   671  
   672  	if !updated {
   673  		newTaints = append(newTaints, *taint)
   674  	}
   675  
   676  	newNode.Spec.Taints = newTaints
   677  	return newNode, true, nil
   678  }
   679  
   680  // semantic can do semantic deep equality checks for core objects.
   681  // Example: apiequality.Semantic.DeepEqual(aPod, aPodWithNonNilButEmptyMaps) == true
   682  // copied from pkg/apis/core/helper/helpers.go Semantic
   683  var semantic = conversion.EqualitiesOrDie(
   684  	func(a, b resource.Quantity) bool {
   685  		// Ignore formatting, only care that numeric value stayed the same.
   686  		// TODO: if we decide it's important, it should be safe to start comparing the format.
   687  		//
   688  		// Uninitialized quantities are equivalent to 0 quantities.
   689  		return a.Cmp(b) == 0
   690  	},
   691  	func(a, b metav1.MicroTime) bool {
   692  		return a.UTC() == b.UTC()
   693  	},
   694  	func(a, b metav1.Time) bool {
   695  		return a.UTC() == b.UTC()
   696  	},
   697  	func(a, b labels.Selector) bool {
   698  		return a.String() == b.String()
   699  	},
   700  	func(a, b fields.Selector) bool {
   701  		return a.String() == b.String()
   702  	},
   703  )
   704  
   705  // removeNodeTaint is for cleaning up taints temporarily added to node,
   706  // won't fail if target taint doesn't exist or has been removed.
   707  // If passed a node it'll check if there's anything to be done, if taint is not present it won't issue
   708  // any API calls.
   709  func removeNodeTaint(ctx context.Context, c clientset.Interface, nodeName string, node *v1.Node, taints ...*v1.Taint) error {
   710  	if len(taints) == 0 {
   711  		return nil
   712  	}
   713  	// Short circuit for limiting amount of API calls.
   714  	if node != nil {
   715  		match := false
   716  		for _, taint := range taints {
   717  			if taintExists(node.Spec.Taints, taint) {
   718  				match = true
   719  				break
   720  			}
   721  		}
   722  		if !match {
   723  			return nil
   724  		}
   725  	}
   726  
   727  	firstTry := true
   728  	return clientretry.RetryOnConflict(updateTaintBackOff, func() error {
   729  		var err error
   730  		var oldNode *v1.Node
   731  		// First we try getting node from the API server cache, as it's cheaper. If it fails
   732  		// we get it from etcd to be sure to have fresh data.
   733  		if firstTry {
   734  			oldNode, err = c.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{ResourceVersion: "0"})
   735  			firstTry = false
   736  		} else {
   737  			oldNode, err = c.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
   738  		}
   739  		if err != nil {
   740  			return err
   741  		}
   742  
   743  		var newNode *v1.Node
   744  		oldNodeCopy := oldNode
   745  		updated := false
   746  		for _, taint := range taints {
   747  			curNewNode, ok, err := removeTaint(oldNodeCopy, taint)
   748  			if err != nil {
   749  				return fmt.Errorf("failed to remove taint of node")
   750  			}
   751  			updated = updated || ok
   752  			newNode = curNewNode
   753  			oldNodeCopy = curNewNode
   754  		}
   755  		if !updated {
   756  			return nil
   757  		}
   758  		return patchNodeTaints(ctx, c, nodeName, oldNode, newNode)
   759  	})
   760  }
   761  
   762  // patchNodeTaints patches node's taints.
   763  func patchNodeTaints(ctx context.Context, c clientset.Interface, nodeName string, oldNode *v1.Node, newNode *v1.Node) error {
   764  	oldData, err := json.Marshal(oldNode)
   765  	if err != nil {
   766  		return fmt.Errorf("failed to marshal old node %#v for node %q: %w", oldNode, nodeName, err)
   767  	}
   768  
   769  	newTaints := newNode.Spec.Taints
   770  	newNodeClone := oldNode.DeepCopy()
   771  	newNodeClone.Spec.Taints = newTaints
   772  	newData, err := json.Marshal(newNodeClone)
   773  	if err != nil {
   774  		return fmt.Errorf("failed to marshal new node %#v for node %q: %w", newNodeClone, nodeName, err)
   775  	}
   776  
   777  	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
   778  	if err != nil {
   779  		return fmt.Errorf("failed to create patch for node %q: %w", nodeName, err)
   780  	}
   781  
   782  	_, err = c.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{})
   783  	return err
   784  }
   785  
   786  // removeTaint tries to remove a taint from annotations list. Returns a new copy of updated Node and true if something was updated
   787  // false otherwise.
   788  func removeTaint(node *v1.Node, taint *v1.Taint) (*v1.Node, bool, error) {
   789  	newNode := node.DeepCopy()
   790  	nodeTaints := newNode.Spec.Taints
   791  	if len(nodeTaints) == 0 {
   792  		return newNode, false, nil
   793  	}
   794  
   795  	if !taintExists(nodeTaints, taint) {
   796  		return newNode, false, nil
   797  	}
   798  
   799  	newTaints, _ := deleteTaint(nodeTaints, taint)
   800  	newNode.Spec.Taints = newTaints
   801  	return newNode, true, nil
   802  }
   803  
   804  // deleteTaint removes all the taints that have the same key and effect to given taintToDelete.
   805  func deleteTaint(taints []v1.Taint, taintToDelete *v1.Taint) ([]v1.Taint, bool) {
   806  	var newTaints []v1.Taint
   807  	deleted := false
   808  	for i := range taints {
   809  		if taintToDelete.MatchTaint(&taints[i]) {
   810  			deleted = true
   811  			continue
   812  		}
   813  		newTaints = append(newTaints, taints[i])
   814  	}
   815  	return newTaints, deleted
   816  }
   817  
   818  func verifyThatTaintIsGone(ctx context.Context, c clientset.Interface, nodeName string, taint *v1.Taint) {
   819  	ginkgo.By("verifying the node doesn't have the taint " + taint.ToString())
   820  	nodeUpdated, err := c.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
   821  
   822  	// TODO use wrapper methods in expect.go after removing core e2e dependency on node
   823  	gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred())
   824  	if taintExists(nodeUpdated.Spec.Taints, taint) {
   825  		framework.Failf("Failed removing taint " + taint.ToString() + " of the node " + nodeName)
   826  	}
   827  }
   828
View as plain text