taint_test.go

Documentation: k8s.io/kubernetes/test/integration/scheduler/taint

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package taint
    18  
    19  // This file tests the Taint feature.
    20  
    21  import (
    22  	"fmt"
    23  	"testing"
    24  	"time"
    25  
    26  	v1 "k8s.io/api/core/v1"
    27  	"k8s.io/apimachinery/pkg/api/resource"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/client-go/informers"
    30  	"k8s.io/client-go/kubernetes"
    31  	restclient "k8s.io/client-go/rest"
    32  	"k8s.io/kubernetes/pkg/controller/nodelifecycle"
    33  	"k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction"
    34  	pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction"
    35  	testutils "k8s.io/kubernetes/test/integration/util"
    36  )
    37  
    38  // imported from testutils
    39  var (
    40  	waitForPodUnschedulable = testutils.WaitForPodUnschedulable
    41  )
    42  
    43  func newPod(nsName, name string, req, limit v1.ResourceList) *v1.Pod {
    44  	return &v1.Pod{
    45  		ObjectMeta: metav1.ObjectMeta{
    46  			Name:      name,
    47  			Namespace: nsName,
    48  		},
    49  		Spec: v1.PodSpec{
    50  			Containers: []v1.Container{
    51  				{
    52  					Name:  "busybox",
    53  					Image: "busybox",
    54  					Resources: v1.ResourceRequirements{
    55  						Requests: req,
    56  						Limits:   limit,
    57  					},
    58  				},
    59  			},
    60  		},
    61  	}
    62  }
    63  
    64  // TestTaintNodeByCondition tests related cases for TaintNodeByCondition feature.
    65  func TestTaintNodeByCondition(t *testing.T) {
    66  	// Build PodToleration Admission.
    67  	admission := podtolerationrestriction.NewPodTolerationsPlugin(&pluginapi.Configuration{})
    68  
    69  	testCtx := testutils.InitTestAPIServer(t, "taint-node-by-condition", admission)
    70  
    71  	// Build clientset and informers for controllers.
    72  	externalClientConfig := restclient.CopyConfig(testCtx.KubeConfig)
    73  	externalClientConfig.QPS = -1
    74  	externalClientset := kubernetes.NewForConfigOrDie(externalClientConfig)
    75  	externalInformers := informers.NewSharedInformerFactory(externalClientset, 0)
    76  
    77  	admission.SetExternalKubeClientSet(externalClientset)
    78  	admission.SetExternalKubeInformerFactory(externalInformers)
    79  
    80  	testCtx = testutils.InitTestScheduler(t, testCtx)
    81  
    82  	cs := testCtx.ClientSet
    83  	nsName := testCtx.NS.Name
    84  
    85  	// Start NodeLifecycleController for taint.
    86  	nc, err := nodelifecycle.NewNodeLifecycleController(
    87  		testCtx.Ctx,
    88  		externalInformers.Coordination().V1().Leases(),
    89  		externalInformers.Core().V1().Pods(),
    90  		externalInformers.Core().V1().Nodes(),
    91  		externalInformers.Apps().V1().DaemonSets(),
    92  		cs,
    93  		time.Hour,   // Node monitor grace period
    94  		time.Second, // Node startup grace period
    95  		time.Second, // Node monitor period
    96  		100,         // Eviction limiter QPS
    97  		100,         // Secondary eviction limiter QPS
    98  		100,         // Large cluster threshold
    99  		100,         // Unhealthy zone threshold
   100  	)
   101  	if err != nil {
   102  		t.Errorf("Failed to create node controller: %v", err)
   103  		return
   104  	}
   105  
   106  	// Waiting for all controllers to sync
   107  	externalInformers.Start(testCtx.Ctx.Done())
   108  	externalInformers.WaitForCacheSync(testCtx.Ctx.Done())
   109  	testutils.SyncSchedulerInformerFactory(testCtx)
   110  
   111  	// Run all controllers
   112  	go nc.Run(testCtx.Ctx)
   113  	go testCtx.Scheduler.Run(testCtx.Ctx)
   114  
   115  	// -------------------------------------------
   116  	// Test TaintNodeByCondition feature.
   117  	// -------------------------------------------
   118  	nodeRes := v1.ResourceList{
   119  		v1.ResourceCPU:    resource.MustParse("4000m"),
   120  		v1.ResourceMemory: resource.MustParse("16Gi"),
   121  		v1.ResourcePods:   resource.MustParse("110"),
   122  	}
   123  
   124  	podRes := v1.ResourceList{
   125  		v1.ResourceCPU:    resource.MustParse("100m"),
   126  		v1.ResourceMemory: resource.MustParse("100Mi"),
   127  	}
   128  
   129  	notReadyToleration := v1.Toleration{
   130  		Key:      v1.TaintNodeNotReady,
   131  		Operator: v1.TolerationOpExists,
   132  		Effect:   v1.TaintEffectNoSchedule,
   133  	}
   134  
   135  	unschedulableToleration := v1.Toleration{
   136  		Key:      v1.TaintNodeUnschedulable,
   137  		Operator: v1.TolerationOpExists,
   138  		Effect:   v1.TaintEffectNoSchedule,
   139  	}
   140  
   141  	memoryPressureToleration := v1.Toleration{
   142  		Key:      v1.TaintNodeMemoryPressure,
   143  		Operator: v1.TolerationOpExists,
   144  		Effect:   v1.TaintEffectNoSchedule,
   145  	}
   146  
   147  	diskPressureToleration := v1.Toleration{
   148  		Key:      v1.TaintNodeDiskPressure,
   149  		Operator: v1.TolerationOpExists,
   150  		Effect:   v1.TaintEffectNoSchedule,
   151  	}
   152  
   153  	networkUnavailableToleration := v1.Toleration{
   154  		Key:      v1.TaintNodeNetworkUnavailable,
   155  		Operator: v1.TolerationOpExists,
   156  		Effect:   v1.TaintEffectNoSchedule,
   157  	}
   158  
   159  	pidPressureToleration := v1.Toleration{
   160  		Key:      v1.TaintNodePIDPressure,
   161  		Operator: v1.TolerationOpExists,
   162  		Effect:   v1.TaintEffectNoSchedule,
   163  	}
   164  
   165  	bestEffortPod := newPod(nsName, "besteffort-pod", nil, nil)
   166  	burstablePod := newPod(nsName, "burstable-pod", podRes, nil)
   167  	guaranteePod := newPod(nsName, "guarantee-pod", podRes, podRes)
   168  
   169  	type podCase struct {
   170  		pod         *v1.Pod
   171  		tolerations []v1.Toleration
   172  		fits        bool
   173  	}
   174  
   175  	// switch to table driven testings
   176  	tests := []struct {
   177  		name           string
   178  		existingTaints []v1.Taint
   179  		nodeConditions []v1.NodeCondition
   180  		unschedulable  bool
   181  		expectedTaints []v1.Taint
   182  		pods           []podCase
   183  	}{
   184  		{
   185  			name: "not-ready node",
   186  			nodeConditions: []v1.NodeCondition{
   187  				{
   188  					Type:   v1.NodeReady,
   189  					Status: v1.ConditionFalse,
   190  				},
   191  			},
   192  			expectedTaints: []v1.Taint{
   193  				{
   194  					Key:    v1.TaintNodeNotReady,
   195  					Effect: v1.TaintEffectNoSchedule,
   196  				},
   197  			},
   198  			pods: []podCase{
   199  				{
   200  					pod:  bestEffortPod,
   201  					fits: false,
   202  				},
   203  				{
   204  					pod:  burstablePod,
   205  					fits: false,
   206  				},
   207  				{
   208  					pod:  guaranteePod,
   209  					fits: false,
   210  				},
   211  				{
   212  					pod:         bestEffortPod,
   213  					tolerations: []v1.Toleration{notReadyToleration},
   214  					fits:        true,
   215  				},
   216  			},
   217  		},
   218  		{
   219  			name:          "unschedulable node",
   220  			unschedulable: true, // node.spec.unschedulable = true
   221  			nodeConditions: []v1.NodeCondition{
   222  				{
   223  					Type:   v1.NodeReady,
   224  					Status: v1.ConditionTrue,
   225  				},
   226  			},
   227  			expectedTaints: []v1.Taint{
   228  				{
   229  					Key:    v1.TaintNodeUnschedulable,
   230  					Effect: v1.TaintEffectNoSchedule,
   231  				},
   232  			},
   233  			pods: []podCase{
   234  				{
   235  					pod:  bestEffortPod,
   236  					fits: false,
   237  				},
   238  				{
   239  					pod:  burstablePod,
   240  					fits: false,
   241  				},
   242  				{
   243  					pod:  guaranteePod,
   244  					fits: false,
   245  				},
   246  				{
   247  					pod:         bestEffortPod,
   248  					tolerations: []v1.Toleration{unschedulableToleration},
   249  					fits:        true,
   250  				},
   251  			},
   252  		},
   253  		{
   254  			name: "memory pressure node",
   255  			nodeConditions: []v1.NodeCondition{
   256  				{
   257  					Type:   v1.NodeMemoryPressure,
   258  					Status: v1.ConditionTrue,
   259  				},
   260  				{
   261  					Type:   v1.NodeReady,
   262  					Status: v1.ConditionTrue,
   263  				},
   264  			},
   265  			expectedTaints: []v1.Taint{
   266  				{
   267  					Key:    v1.TaintNodeMemoryPressure,
   268  					Effect: v1.TaintEffectNoSchedule,
   269  				},
   270  			},
   271  			// In MemoryPressure condition, both Burstable and Guarantee pods are scheduled;
   272  			// BestEffort pod with toleration are also scheduled.
   273  			pods: []podCase{
   274  				{
   275  					pod:  bestEffortPod,
   276  					fits: false,
   277  				},
   278  				{
   279  					pod:         bestEffortPod,
   280  					tolerations: []v1.Toleration{memoryPressureToleration},
   281  					fits:        true,
   282  				},
   283  				{
   284  					pod:         bestEffortPod,
   285  					tolerations: []v1.Toleration{diskPressureToleration},
   286  					fits:        false,
   287  				},
   288  				{
   289  					pod:  burstablePod,
   290  					fits: true,
   291  				},
   292  				{
   293  					pod:  guaranteePod,
   294  					fits: true,
   295  				},
   296  			},
   297  		},
   298  		{
   299  			name: "disk pressure node",
   300  			nodeConditions: []v1.NodeCondition{
   301  				{
   302  					Type:   v1.NodeDiskPressure,
   303  					Status: v1.ConditionTrue,
   304  				},
   305  				{
   306  					Type:   v1.NodeReady,
   307  					Status: v1.ConditionTrue,
   308  				},
   309  			},
   310  			expectedTaints: []v1.Taint{
   311  				{
   312  					Key:    v1.TaintNodeDiskPressure,
   313  					Effect: v1.TaintEffectNoSchedule,
   314  				},
   315  			},
   316  			// In DiskPressure condition, only pods with toleration can be scheduled.
   317  			pods: []podCase{
   318  				{
   319  					pod:  bestEffortPod,
   320  					fits: false,
   321  				},
   322  				{
   323  					pod:  burstablePod,
   324  					fits: false,
   325  				},
   326  				{
   327  					pod:  guaranteePod,
   328  					fits: false,
   329  				},
   330  				{
   331  					pod:         bestEffortPod,
   332  					tolerations: []v1.Toleration{diskPressureToleration},
   333  					fits:        true,
   334  				},
   335  				{
   336  					pod:         bestEffortPod,
   337  					tolerations: []v1.Toleration{memoryPressureToleration},
   338  					fits:        false,
   339  				},
   340  			},
   341  		},
   342  		{
   343  			name: "network unavailable and node is ready",
   344  			nodeConditions: []v1.NodeCondition{
   345  				{
   346  					Type:   v1.NodeNetworkUnavailable,
   347  					Status: v1.ConditionTrue,
   348  				},
   349  				{
   350  					Type:   v1.NodeReady,
   351  					Status: v1.ConditionTrue,
   352  				},
   353  			},
   354  			expectedTaints: []v1.Taint{
   355  				{
   356  					Key:    v1.TaintNodeNetworkUnavailable,
   357  					Effect: v1.TaintEffectNoSchedule,
   358  				},
   359  			},
   360  			pods: []podCase{
   361  				{
   362  					pod:  bestEffortPod,
   363  					fits: false,
   364  				},
   365  				{
   366  					pod:  burstablePod,
   367  					fits: false,
   368  				},
   369  				{
   370  					pod:  guaranteePod,
   371  					fits: false,
   372  				},
   373  				{
   374  					pod: burstablePod,
   375  					tolerations: []v1.Toleration{
   376  						networkUnavailableToleration,
   377  					},
   378  					fits: true,
   379  				},
   380  			},
   381  		},
   382  		{
   383  			name: "network unavailable and node is not ready",
   384  			nodeConditions: []v1.NodeCondition{
   385  				{
   386  					Type:   v1.NodeNetworkUnavailable,
   387  					Status: v1.ConditionTrue,
   388  				},
   389  				{
   390  					Type:   v1.NodeReady,
   391  					Status: v1.ConditionFalse,
   392  				},
   393  			},
   394  			expectedTaints: []v1.Taint{
   395  				{
   396  					Key:    v1.TaintNodeNetworkUnavailable,
   397  					Effect: v1.TaintEffectNoSchedule,
   398  				},
   399  				{
   400  					Key:    v1.TaintNodeNotReady,
   401  					Effect: v1.TaintEffectNoSchedule,
   402  				},
   403  			},
   404  			pods: []podCase{
   405  				{
   406  					pod:  bestEffortPod,
   407  					fits: false,
   408  				},
   409  				{
   410  					pod:  burstablePod,
   411  					fits: false,
   412  				},
   413  				{
   414  					pod:  guaranteePod,
   415  					fits: false,
   416  				},
   417  				{
   418  					pod: burstablePod,
   419  					tolerations: []v1.Toleration{
   420  						networkUnavailableToleration,
   421  					},
   422  					fits: false,
   423  				},
   424  				{
   425  					pod: burstablePod,
   426  					tolerations: []v1.Toleration{
   427  						networkUnavailableToleration,
   428  						notReadyToleration,
   429  					},
   430  					fits: true,
   431  				},
   432  			},
   433  		},
   434  		{
   435  			name: "pid pressure node",
   436  			nodeConditions: []v1.NodeCondition{
   437  				{
   438  					Type:   v1.NodePIDPressure,
   439  					Status: v1.ConditionTrue,
   440  				},
   441  				{
   442  					Type:   v1.NodeReady,
   443  					Status: v1.ConditionTrue,
   444  				},
   445  			},
   446  			expectedTaints: []v1.Taint{
   447  				{
   448  					Key:    v1.TaintNodePIDPressure,
   449  					Effect: v1.TaintEffectNoSchedule,
   450  				},
   451  			},
   452  			pods: []podCase{
   453  				{
   454  					pod:  bestEffortPod,
   455  					fits: false,
   456  				},
   457  				{
   458  					pod:  burstablePod,
   459  					fits: false,
   460  				},
   461  				{
   462  					pod:  guaranteePod,
   463  					fits: false,
   464  				},
   465  				{
   466  					pod:         bestEffortPod,
   467  					tolerations: []v1.Toleration{pidPressureToleration},
   468  					fits:        true,
   469  				},
   470  			},
   471  		},
   472  		{
   473  			name: "multi taints on node",
   474  			nodeConditions: []v1.NodeCondition{
   475  				{
   476  					Type:   v1.NodePIDPressure,
   477  					Status: v1.ConditionTrue,
   478  				},
   479  				{
   480  					Type:   v1.NodeMemoryPressure,
   481  					Status: v1.ConditionTrue,
   482  				},
   483  				{
   484  					Type:   v1.NodeDiskPressure,
   485  					Status: v1.ConditionTrue,
   486  				},
   487  				{
   488  					Type:   v1.NodeReady,
   489  					Status: v1.ConditionTrue,
   490  				},
   491  			},
   492  			expectedTaints: []v1.Taint{
   493  				{
   494  					Key:    v1.TaintNodeDiskPressure,
   495  					Effect: v1.TaintEffectNoSchedule,
   496  				},
   497  				{
   498  					Key:    v1.TaintNodeMemoryPressure,
   499  					Effect: v1.TaintEffectNoSchedule,
   500  				},
   501  				{
   502  					Key:    v1.TaintNodePIDPressure,
   503  					Effect: v1.TaintEffectNoSchedule,
   504  				},
   505  			},
   506  		},
   507  	}
   508  
   509  	for _, test := range tests {
   510  		t.Run(test.name, func(t *testing.T) {
   511  			node := &v1.Node{
   512  				ObjectMeta: metav1.ObjectMeta{
   513  					Name: "node-1",
   514  				},
   515  				Spec: v1.NodeSpec{
   516  					Unschedulable: test.unschedulable,
   517  					Taints:        test.existingTaints,
   518  				},
   519  				Status: v1.NodeStatus{
   520  					Capacity:    nodeRes,
   521  					Allocatable: nodeRes,
   522  					Conditions:  test.nodeConditions,
   523  				},
   524  			}
   525  
   526  			if _, err := cs.CoreV1().Nodes().Create(testCtx.Ctx, node, metav1.CreateOptions{}); err != nil {
   527  				t.Errorf("Failed to create node, err: %v", err)
   528  			}
   529  			if err := testutils.WaitForNodeTaints(cs, node, test.expectedTaints); err != nil {
   530  				node, err = cs.CoreV1().Nodes().Get(testCtx.Ctx, node.Name, metav1.GetOptions{})
   531  				if err != nil {
   532  					t.Errorf("Failed to get node <%s>", node.Name)
   533  				}
   534  
   535  				t.Errorf("Failed to taint node <%s>, expected: %v, got: %v, err: %v", node.Name, test.expectedTaints, node.Spec.Taints, err)
   536  			}
   537  
   538  			var pods []*v1.Pod
   539  			for i, p := range test.pods {
   540  				pod := p.pod.DeepCopy()
   541  				pod.Name = fmt.Sprintf("%s-%d", pod.Name, i)
   542  				pod.Spec.Tolerations = p.tolerations
   543  
   544  				createdPod, err := cs.CoreV1().Pods(pod.Namespace).Create(testCtx.Ctx, pod, metav1.CreateOptions{})
   545  				if err != nil {
   546  					t.Fatalf("Failed to create pod %s/%s, error: %v",
   547  						pod.Namespace, pod.Name, err)
   548  				}
   549  
   550  				pods = append(pods, createdPod)
   551  
   552  				if p.fits {
   553  					if err := testutils.WaitForPodToSchedule(cs, createdPod); err != nil {
   554  						t.Errorf("Failed to schedule pod %s/%s on the node, err: %v",
   555  							pod.Namespace, pod.Name, err)
   556  					}
   557  				} else {
   558  					if err := waitForPodUnschedulable(cs, createdPod); err != nil {
   559  						t.Errorf("Unschedulable pod %s/%s gets scheduled on the node, err: %v",
   560  							pod.Namespace, pod.Name, err)
   561  					}
   562  				}
   563  			}
   564  
   565  			testutils.CleanupPods(testCtx.Ctx, cs, t, pods)
   566  			testutils.CleanupNodes(cs, t)
   567  			testutils.WaitForSchedulerCacheCleanup(testCtx.Scheduler, t)
   568  		})
   569  	}
   570  }
   571
View as plain text