Source file src/k8s.io/kubernetes/test/integration/scheduler/scoring/priorities_test.go

Documentation: k8s.io/kubernetes/test/integration/scheduler/scoring

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package scoring
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"testing"
    24  	"time"
    25  
    26  	v1 "k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/apimachinery/pkg/runtime"
    29  	"k8s.io/apimachinery/pkg/util/intstr"
    30  	"k8s.io/apimachinery/pkg/util/wait"
    31  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    32  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    33  	configv1 "k8s.io/kube-scheduler/config/v1"
    34  	"k8s.io/kubernetes/pkg/features"
    35  	"k8s.io/kubernetes/pkg/scheduler"
    36  	configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
    37  	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality"
    38  	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
    39  	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
    40  	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
    41  	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
    42  	st "k8s.io/kubernetes/pkg/scheduler/testing"
    43  	testutils "k8s.io/kubernetes/test/integration/util"
    44  	imageutils "k8s.io/kubernetes/test/utils/image"
    45  	"k8s.io/utils/pointer"
    46  )
    47  
    48  // imported from testutils
    49  var (
    50  	runPausePod                  = testutils.RunPausePod
    51  	createAndWaitForNodesInCache = testutils.CreateAndWaitForNodesInCache
    52  	createNode                   = testutils.CreateNode
    53  	createNamespacesWithLabels   = testutils.CreateNamespacesWithLabels
    54  	runPodWithContainers         = testutils.RunPodWithContainers
    55  	initPausePod                 = testutils.InitPausePod
    56  	initPodWithContainers        = testutils.InitPodWithContainers
    57  	podScheduledIn               = testutils.PodScheduledIn
    58  	podUnschedulable             = testutils.PodUnschedulable
    59  )
    60  
    61  var (
    62  	hardSpread   = v1.DoNotSchedule
    63  	softSpread   = v1.ScheduleAnyway
    64  	ignorePolicy = v1.NodeInclusionPolicyIgnore
    65  	honorPolicy  = v1.NodeInclusionPolicyHonor
    66  	taints       = []v1.Taint{{Key: v1.TaintNodeUnschedulable, Value: "", Effect: v1.TaintEffectPreferNoSchedule}}
    67  )
    68  
    69  const (
    70  	resourceGPU  = "example.com/gpu"
    71  	pollInterval = 100 * time.Millisecond
    72  )
    73  
    74  // This file tests the scheduler priority functions.
    75  func initTestSchedulerForPriorityTest(t *testing.T, preScorePluginName, scorePluginName string) *testutils.TestContext {
    76  	cc := configv1.KubeSchedulerConfiguration{
    77  		Profiles: []configv1.KubeSchedulerProfile{{
    78  			SchedulerName: pointer.String(v1.DefaultSchedulerName),
    79  			Plugins: &configv1.Plugins{
    80  				PreScore: configv1.PluginSet{
    81  					Disabled: []configv1.Plugin{
    82  						{Name: "*"},
    83  					},
    84  				},
    85  				Score: configv1.PluginSet{
    86  					Enabled: []configv1.Plugin{
    87  						{Name: scorePluginName, Weight: pointer.Int32(1)},
    88  					},
    89  					Disabled: []configv1.Plugin{
    90  						{Name: "*"},
    91  					},
    92  				},
    93  			},
    94  		}},
    95  	}
    96  	if preScorePluginName != "" {
    97  		cc.Profiles[0].Plugins.PreScore.Enabled = append(cc.Profiles[0].Plugins.PreScore.Enabled, configv1.Plugin{Name: preScorePluginName})
    98  	}
    99  	cfg := configtesting.V1ToInternalWithDefaults(t, cc)
   100  	testCtx := testutils.InitTestSchedulerWithOptions(
   101  		t,
   102  		testutils.InitTestAPIServer(t, strings.ToLower(scorePluginName), nil),
   103  		0,
   104  		scheduler.WithProfiles(cfg.Profiles...),
   105  	)
   106  	testutils.SyncSchedulerInformerFactory(testCtx)
   107  	go testCtx.Scheduler.Run(testCtx.Ctx)
   108  	return testCtx
   109  }
   110  
   111  func initTestSchedulerForNodeResourcesTest(t *testing.T) *testutils.TestContext {
   112  	cfg := configtesting.V1ToInternalWithDefaults(t, configv1.KubeSchedulerConfiguration{
   113  		Profiles: []configv1.KubeSchedulerProfile{
   114  			{
   115  				SchedulerName: pointer.String(v1.DefaultSchedulerName),
   116  			},
   117  			{
   118  				SchedulerName: pointer.String("gpu-binpacking-scheduler"),
   119  				PluginConfig: []configv1.PluginConfig{
   120  					{
   121  						Name: noderesources.Name,
   122  						Args: runtime.RawExtension{Object: &configv1.NodeResourcesFitArgs{
   123  							ScoringStrategy: &configv1.ScoringStrategy{
   124  								Type: configv1.MostAllocated,
   125  								Resources: []configv1.ResourceSpec{
   126  									{Name: string(v1.ResourceCPU), Weight: 1},
   127  									{Name: string(v1.ResourceMemory), Weight: 1},
   128  									{Name: resourceGPU, Weight: 2}},
   129  							},
   130  						}},
   131  					},
   132  				},
   133  			},
   134  		},
   135  	})
   136  	testCtx := testutils.InitTestSchedulerWithOptions(
   137  		t,
   138  		testutils.InitTestAPIServer(t, strings.ToLower(noderesources.Name), nil),
   139  		0,
   140  		scheduler.WithProfiles(cfg.Profiles...),
   141  	)
   142  	testutils.SyncSchedulerInformerFactory(testCtx)
   143  	go testCtx.Scheduler.Run(testCtx.Ctx)
   144  	return testCtx
   145  }
   146  
   147  // TestNodeResourcesScoring verifies that scheduler's node resources priority function
   148  // works correctly.
   149  func TestNodeResourcesScoring(t *testing.T) {
   150  	testCtx := initTestSchedulerForNodeResourcesTest(t)
   151  	// Add a few nodes.
   152  	_, err := createAndWaitForNodesInCache(testCtx, "testnode", st.MakeNode().Capacity(
   153  		map[v1.ResourceName]string{
   154  			v1.ResourceCPU:    "8",
   155  			v1.ResourceMemory: "16G",
   156  			resourceGPU:       "4",
   157  		}), 2)
   158  	if err != nil {
   159  		t.Fatal(err)
   160  	}
   161  	cpuBoundPod1, err := runPausePod(testCtx.ClientSet, st.MakePod().Namespace(testCtx.NS.Name).Name("cpubound1").Res(
   162  		map[v1.ResourceName]string{
   163  			v1.ResourceCPU:    "2",
   164  			v1.ResourceMemory: "4G",
   165  			resourceGPU:       "1",
   166  		},
   167  	).Obj())
   168  	if err != nil {
   169  		t.Fatal(err)
   170  	}
   171  	gpuBoundPod1, err := runPausePod(testCtx.ClientSet, st.MakePod().Namespace(testCtx.NS.Name).Name("gpubound1").Res(
   172  		map[v1.ResourceName]string{
   173  			v1.ResourceCPU:    "1",
   174  			v1.ResourceMemory: "2G",
   175  			resourceGPU:       "2",
   176  		},
   177  	).Obj())
   178  	if err != nil {
   179  		t.Fatal(err)
   180  	}
   181  	if cpuBoundPod1.Spec.NodeName == "" || gpuBoundPod1.Spec.NodeName == "" {
   182  		t.Fatalf("pods should have nodeName assigned, got %q and %q",
   183  			cpuBoundPod1.Spec.NodeName, gpuBoundPod1.Spec.NodeName)
   184  	}
   185  
   186  	// Since both pods used the default scheduler, then they should land on two different
   187  	// nodes because the default configuration uses LeastAllocated.
   188  	if cpuBoundPod1.Spec.NodeName == gpuBoundPod1.Spec.NodeName {
   189  		t.Fatalf("pods should have landed on different nodes, both scheduled on %q",
   190  			cpuBoundPod1.Spec.NodeName)
   191  	}
   192  
   193  	// The following pod is using the gpu-binpacking-scheduler profile, which gives a higher weight to
   194  	// GPU-based binpacking, and so it should land on the node with higher GPU utilization.
   195  	cpuBoundPod2, err := runPausePod(testCtx.ClientSet, st.MakePod().Namespace(testCtx.NS.Name).Name("cpubound2").SchedulerName("gpu-binpacking-scheduler").Res(
   196  		map[v1.ResourceName]string{
   197  			v1.ResourceCPU:    "2",
   198  			v1.ResourceMemory: "4G",
   199  			resourceGPU:       "1",
   200  		},
   201  	).Obj())
   202  	if err != nil {
   203  		t.Fatal(err)
   204  	}
   205  	if cpuBoundPod2.Spec.NodeName != gpuBoundPod1.Spec.NodeName {
   206  		t.Errorf("pods should have landed on the same node")
   207  	}
   208  }
   209  
   210  // TestNodeAffinityScoring verifies that scheduler's node affinity priority function
   211  // works correctly.
   212  func TestNodeAffinityScoring(t *testing.T) {
   213  	testCtx := initTestSchedulerForPriorityTest(t, nodeaffinity.Name, nodeaffinity.Name)
   214  	// Add a few nodes.
   215  	_, err := createAndWaitForNodesInCache(testCtx, "testnode", st.MakeNode(), 4)
   216  	if err != nil {
   217  		t.Fatal(err)
   218  	}
   219  	// Add a label to one of the nodes.
   220  	labelKey := "kubernetes.io/node-topologyKey"
   221  	labelValue := "topologyvalue"
   222  	labeledNode, err := createNode(testCtx.ClientSet, st.MakeNode().Name("testnode-4").Label(labelKey, labelValue).Obj())
   223  	if err != nil {
   224  		t.Fatalf("Cannot create labeled node: %v", err)
   225  	}
   226  
   227  	// Create a pod with node affinity.
   228  	podName := "pod-with-node-affinity"
   229  	pod, err := runPausePod(testCtx.ClientSet, initPausePod(&testutils.PausePodConfig{
   230  		Name:      podName,
   231  		Namespace: testCtx.NS.Name,
   232  		Affinity: &v1.Affinity{
   233  			NodeAffinity: &v1.NodeAffinity{
   234  				PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
   235  					{
   236  						Preference: v1.NodeSelectorTerm{
   237  							MatchExpressions: []v1.NodeSelectorRequirement{
   238  								{
   239  									Key:      labelKey,
   240  									Operator: v1.NodeSelectorOpIn,
   241  									Values:   []string{labelValue},
   242  								},
   243  							},
   244  						},
   245  						Weight: 20,
   246  					},
   247  				},
   248  			},
   249  		},
   250  	}))
   251  	if err != nil {
   252  		t.Fatalf("Error running pause pod: %v", err)
   253  	}
   254  	if pod.Spec.NodeName != labeledNode.Name {
   255  		t.Errorf("Pod %v got scheduled on an unexpected node: %v. Expected node: %v.", podName, pod.Spec.NodeName, labeledNode.Name)
   256  	} else {
   257  		t.Logf("Pod %v got successfully scheduled on node %v.", podName, pod.Spec.NodeName)
   258  	}
   259  }
   260  
   261  // TestPodAffinityScoring verifies that scheduler's pod affinity priority function
   262  // works correctly.
   263  func TestPodAffinityScoring(t *testing.T) {
   264  	labelKey := "service"
   265  	labelValue := "S1"
   266  	topologyKey := "node-topologykey"
   267  	topologyValues := []string{}
   268  	for i := 0; i < 5; i++ {
   269  		topologyValues = append(topologyValues, fmt.Sprintf("topologyvalue%d", i))
   270  	}
   271  	tests := []struct {
   272  		name         string
   273  		pod          *testutils.PausePodConfig
   274  		existingPods []*testutils.PausePodConfig
   275  		nodes        []*v1.Node
   276  		// expectedNodeName is the list of node names. The pod should be scheduled on either of them.
   277  		expectedNodeName               []string
   278  		enableMatchLabelKeysInAffinity bool
   279  	}{
   280  		{
   281  			name: "pod affinity",
   282  			pod: &testutils.PausePodConfig{
   283  				Name:      "pod1",
   284  				Namespace: "ns1",
   285  				Affinity: &v1.Affinity{
   286  					PodAffinity: &v1.PodAffinity{
   287  						PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
   288  							{
   289  								PodAffinityTerm: v1.PodAffinityTerm{
   290  									LabelSelector: &metav1.LabelSelector{
   291  										MatchExpressions: []metav1.LabelSelectorRequirement{
   292  											{
   293  												Key:      labelKey,
   294  												Operator: metav1.LabelSelectorOpIn,
   295  												Values:   []string{labelValue, "S3"},
   296  											},
   297  										},
   298  									},
   299  									TopologyKey: topologyKey,
   300  								},
   301  								Weight: 50,
   302  							},
   303  						},
   304  					},
   305  				},
   306  			},
   307  			existingPods: []*testutils.PausePodConfig{
   308  				{
   309  					Name:      "attractor-pod",
   310  					Namespace: "ns1",
   311  					Labels:    map[string]string{labelKey: labelValue},
   312  					NodeName:  "node1",
   313  				},
   314  			},
   315  			nodes: []*v1.Node{
   316  				st.MakeNode().Name("node1").Label(topologyKey, topologyValues[0]).Obj(),
   317  				st.MakeNode().Name("node2").Label(topologyKey, topologyValues[1]).Obj(),
   318  				st.MakeNode().Name("node3").Label(topologyKey, topologyValues[2]).Obj(),
   319  				st.MakeNode().Name("node4").Label(topologyKey, topologyValues[3]).Obj(),
   320  				st.MakeNode().Name("node5").Label(topologyKey, topologyValues[4]).Obj(),
   321  				st.MakeNode().Name("node6").Label(topologyKey, topologyValues[0]).Obj(),
   322  				st.MakeNode().Name("node7").Label(topologyKey, topologyValues[1]).Obj(),
   323  				st.MakeNode().Name("node8").Label(topologyKey, topologyValues[2]).Obj(),
   324  				st.MakeNode().Name("node9").Label(topologyKey, topologyValues[3]).Obj(),
   325  				st.MakeNode().Name("node10").Label(topologyKey, topologyValues[4]).Obj(),
   326  				st.MakeNode().Name("other-node1").Obj(),
   327  				st.MakeNode().Name("other-node2").Obj(),
   328  			},
   329  			expectedNodeName: []string{"node1", "node6"},
   330  		},
   331  		{
   332  			name: "pod affinity with namespace selector",
   333  			pod: &testutils.PausePodConfig{
   334  				Name:      "pod1",
   335  				Namespace: "ns2",
   336  				Affinity: &v1.Affinity{
   337  					PodAffinity: &v1.PodAffinity{
   338  						PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
   339  							{
   340  								PodAffinityTerm: v1.PodAffinityTerm{
   341  									NamespaceSelector: &metav1.LabelSelector{}, // all namespaces
   342  									LabelSelector: &metav1.LabelSelector{
   343  										MatchExpressions: []metav1.LabelSelectorRequirement{
   344  											{
   345  												Key:      labelKey,
   346  												Operator: metav1.LabelSelectorOpIn,
   347  												Values:   []string{labelValue, "S3"},
   348  											},
   349  										},
   350  									},
   351  									TopologyKey: topologyKey,
   352  								},
   353  								Weight: 50,
   354  							},
   355  						},
   356  					},
   357  				},
   358  			},
   359  			existingPods: []*testutils.PausePodConfig{
   360  				{
   361  					Name:      "attractor-pod",
   362  					Namespace: "ns1",
   363  					Labels:    map[string]string{labelKey: labelValue},
   364  					NodeName:  "node1",
   365  				},
   366  			},
   367  			nodes: []*v1.Node{
   368  				st.MakeNode().Name("node1").Label(topologyKey, topologyValues[0]).Obj(),
   369  				st.MakeNode().Name("node2").Label(topologyKey, topologyValues[1]).Obj(),
   370  				st.MakeNode().Name("node3").Label(topologyKey, topologyValues[2]).Obj(),
   371  				st.MakeNode().Name("node4").Label(topologyKey, topologyValues[3]).Obj(),
   372  				st.MakeNode().Name("node5").Label(topologyKey, topologyValues[4]).Obj(),
   373  				st.MakeNode().Name("node6").Label(topologyKey, topologyValues[0]).Obj(),
   374  				st.MakeNode().Name("node7").Label(topologyKey, topologyValues[1]).Obj(),
   375  				st.MakeNode().Name("node8").Label(topologyKey, topologyValues[2]).Obj(),
   376  				st.MakeNode().Name("node9").Label(topologyKey, topologyValues[3]).Obj(),
   377  				st.MakeNode().Name("node10").Label(topologyKey, topologyValues[4]).Obj(),
   378  				st.MakeNode().Name("other-node1").Obj(),
   379  				st.MakeNode().Name("other-node2").Obj(),
   380  			},
   381  			expectedNodeName: []string{"node1", "node6"},
   382  		},
   383  		{
   384  			name: "anti affinity: matchLabelKeys is merged into LabelSelector with In operator (feature flag: enabled)",
   385  			pod: &testutils.PausePodConfig{
   386  				Name:      "incoming",
   387  				Namespace: "ns1",
   388  				Labels:    map[string]string{"foo": "", "bar": "a"},
   389  				Affinity: &v1.Affinity{
   390  					PodAntiAffinity: &v1.PodAntiAffinity{
   391  						PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
   392  							{
   393  								PodAffinityTerm: v1.PodAffinityTerm{
   394  									TopologyKey: topologyKey,
   395  									LabelSelector: &metav1.LabelSelector{
   396  										MatchExpressions: []metav1.LabelSelectorRequirement{
   397  											{
   398  												Key:      "foo",
   399  												Operator: metav1.LabelSelectorOpExists,
   400  											},
   401  										},
   402  									},
   403  									MatchLabelKeys: []string{"bar"},
   404  								},
   405  								Weight: 50,
   406  							},
   407  						},
   408  					},
   409  				},
   410  			},
   411  			existingPods: []*testutils.PausePodConfig{
   412  				// It matches the incoming Pod's anti affinity's labelSelector.
   413  				// BUT, the matchLabelKeys make the existing Pod's anti affinity's labelSelector not match with this label.
   414  				{
   415  					NodeName:  "node1",
   416  					Name:      "pod1",
   417  					Namespace: "ns1",
   418  					Labels:    map[string]string{"foo": "", "bar": "fuga"},
   419  				},
   420  				// It matches the incoming Pod's anti affinity.
   421  				{
   422  					NodeName:  "node2",
   423  					Name:      "pod2",
   424  					Namespace: "ns1",
   425  					Labels:    map[string]string{"foo": "", "bar": "a"},
   426  				},
   427  			},
   428  			nodes: []*v1.Node{
   429  				st.MakeNode().Name("node1").Label(topologyKey, topologyValues[0]).Obj(),
   430  				st.MakeNode().Name("node2").Label(topologyKey, topologyValues[1]).Obj(),
   431  			},
   432  			expectedNodeName:               []string{"node1"},
   433  			enableMatchLabelKeysInAffinity: true,
   434  		},
   435  		{
   436  			name: "anti affinity: mismatchLabelKeys is merged into LabelSelector with NotIn operator  (feature flag: enabled)",
   437  			pod: &testutils.PausePodConfig{
   438  				Name:      "incoming",
   439  				Namespace: "ns1",
   440  				Labels:    map[string]string{"foo": "", "bar": "a"},
   441  				Affinity: &v1.Affinity{
   442  					PodAntiAffinity: &v1.PodAntiAffinity{
   443  						PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
   444  							{
   445  								PodAffinityTerm: v1.PodAffinityTerm{
   446  									TopologyKey: topologyKey,
   447  									LabelSelector: &metav1.LabelSelector{
   448  										MatchExpressions: []metav1.LabelSelectorRequirement{
   449  											{
   450  												Key:      "foo",
   451  												Operator: metav1.LabelSelectorOpExists,
   452  											},
   453  										},
   454  									},
   455  									MismatchLabelKeys: []string{"bar"},
   456  								},
   457  								Weight: 50,
   458  							},
   459  						},
   460  					},
   461  				},
   462  			},
   463  			existingPods: []*testutils.PausePodConfig{
   464  				// It matches the incoming Pod's anti affinity's labelSelector.
   465  				{
   466  					NodeName:  "node1",
   467  					Name:      "pod1",
   468  					Namespace: "ns1",
   469  					Labels:    map[string]string{"foo": "", "bar": "fuga"},
   470  				},
   471  				// It matches the incoming Pod's affinity.
   472  				// But, the mismatchLabelKeys make the existing Pod's anti affinity's labelSelector not match with this label.
   473  				{
   474  					NodeName:  "node2",
   475  					Name:      "pod2",
   476  					Namespace: "ns1",
   477  					Labels:    map[string]string{"foo": "", "bar": "a"},
   478  				},
   479  			},
   480  			nodes: []*v1.Node{
   481  				st.MakeNode().Name("node1").Label(topologyKey, topologyValues[0]).Obj(),
   482  				st.MakeNode().Name("node2").Label(topologyKey, topologyValues[1]).Obj(),
   483  			},
   484  			expectedNodeName:               []string{"node2"},
   485  			enableMatchLabelKeysInAffinity: true,
   486  		},
   487  		{
   488  			name: "affinity: matchLabelKeys is merged into LabelSelector with In operator (feature flag: enabled)",
   489  			pod: &testutils.PausePodConfig{
   490  				Affinity: &v1.Affinity{
   491  					PodAffinity: &v1.PodAffinity{
   492  						PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
   493  							{
   494  								// affinity with pod3.
   495  								PodAffinityTerm: v1.PodAffinityTerm{
   496  									TopologyKey: topologyKey,
   497  									LabelSelector: &metav1.LabelSelector{
   498  										MatchExpressions: []metav1.LabelSelectorRequirement{
   499  											{
   500  												Key:      "foo",
   501  												Operator: metav1.LabelSelectorOpExists,
   502  											},
   503  										},
   504  									},
   505  									MatchLabelKeys: []string{"bar"},
   506  								},
   507  								Weight: 50,
   508  							},
   509  							{
   510  								// affinity with pod1 and pod2.
   511  								// schedule this Pod by this weaker affinity
   512  								// if `matchLabelKeys` above isn't working correctly.
   513  								PodAffinityTerm: v1.PodAffinityTerm{
   514  									TopologyKey: topologyKey,
   515  									LabelSelector: &metav1.LabelSelector{
   516  										MatchExpressions: []metav1.LabelSelectorRequirement{
   517  											{
   518  												Key:      "bar",
   519  												Operator: metav1.LabelSelectorOpIn,
   520  												Values:   []string{"hoge"},
   521  											},
   522  										},
   523  									},
   524  								},
   525  								Weight: 10,
   526  							},
   527  						},
   528  					},
   529  				},
   530  				Name:      "incoming",
   531  				Namespace: "ns1",
   532  				Labels:    map[string]string{"foo": "", "bar": "a"},
   533  			},
   534  			existingPods: []*testutils.PausePodConfig{
   535  				{
   536  					NodeName:  "node1",
   537  					Name:      "pod1",
   538  					Namespace: "ns1",
   539  					Labels:    map[string]string{"foo": "", "bar": "hoge"},
   540  				},
   541  				{
   542  					NodeName:  "node2",
   543  					Name:      "pod2",
   544  					Namespace: "ns1",
   545  					Labels:    map[string]string{"foo": "", "bar": "hoge"},
   546  				},
   547  				{
   548  					NodeName:  "node3",
   549  					Name:      "pod3",
   550  					Namespace: "ns1",
   551  					Labels:    map[string]string{"foo": "", "bar": "a"},
   552  				},
   553  			},
   554  			enableMatchLabelKeysInAffinity: true,
   555  			nodes: []*v1.Node{
   556  				st.MakeNode().Name("node1").Label(topologyKey, topologyValues[0]).Obj(),
   557  				st.MakeNode().Name("node2").Label(topologyKey, topologyValues[1]).Obj(),
   558  				st.MakeNode().Name("node3").Label(topologyKey, topologyValues[2]).Obj(),
   559  				st.MakeNode().Name("node4").Label(topologyKey, topologyValues[0]).Obj(),
   560  				st.MakeNode().Name("node5").Label(topologyKey, topologyValues[1]).Obj(),
   561  				st.MakeNode().Name("node6").Label(topologyKey, topologyValues[2]).Obj(),
   562  			},
   563  			expectedNodeName: []string{"node3", "node6"},
   564  		},
   565  		{
   566  			name: "affinity: mismatchLabelKeys is merged into LabelSelector with NotIn operator (feature flag: enabled)",
   567  			pod: &testutils.PausePodConfig{
   568  				Affinity: &v1.Affinity{
   569  					PodAffinity: &v1.PodAffinity{
   570  						PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
   571  							{
   572  								// affinity with pod3.
   573  								PodAffinityTerm: v1.PodAffinityTerm{
   574  									TopologyKey: topologyKey,
   575  									LabelSelector: &metav1.LabelSelector{
   576  										MatchExpressions: []metav1.LabelSelectorRequirement{
   577  											{
   578  												Key:      "foo",
   579  												Operator: metav1.LabelSelectorOpExists,
   580  											},
   581  										},
   582  									},
   583  									MismatchLabelKeys: []string{"bar"},
   584  								},
   585  								Weight: 50,
   586  							},
   587  							{
   588  								// affinity with pod1 and pod2.
   589  								// schedule this Pod by this weaker affinity
   590  								// if `matchLabelKeys` above isn't working correctly.
   591  								PodAffinityTerm: v1.PodAffinityTerm{
   592  									TopologyKey: topologyKey,
   593  									LabelSelector: &metav1.LabelSelector{
   594  										MatchExpressions: []metav1.LabelSelectorRequirement{
   595  											{
   596  												Key:      "bar",
   597  												Operator: metav1.LabelSelectorOpIn,
   598  												Values:   []string{"hoge"},
   599  											},
   600  										},
   601  									},
   602  								},
   603  								Weight: 10,
   604  							},
   605  						},
   606  					},
   607  				},
   608  				Name:      "incoming",
   609  				Namespace: "ns1",
   610  				Labels:    map[string]string{"foo": "", "bar": "a"},
   611  			},
   612  			existingPods: []*testutils.PausePodConfig{
   613  				{
   614  					NodeName:  "node1",
   615  					Name:      "pod1",
   616  					Namespace: "ns1",
   617  					Labels:    map[string]string{"foo": "", "bar": "a"},
   618  				},
   619  				{
   620  					NodeName:  "node2",
   621  					Name:      "pod2",
   622  					Namespace: "ns1",
   623  					Labels:    map[string]string{"foo": "", "bar": "a"},
   624  				},
   625  				{
   626  					NodeName:  "node3",
   627  					Name:      "pod3",
   628  					Namespace: "ns1",
   629  					Labels:    map[string]string{"foo": "", "bar": "hoge"},
   630  				},
   631  			},
   632  			enableMatchLabelKeysInAffinity: true,
   633  			nodes: []*v1.Node{
   634  				st.MakeNode().Name("node1").Label(topologyKey, topologyValues[0]).Obj(),
   635  				st.MakeNode().Name("node2").Label(topologyKey, topologyValues[1]).Obj(),
   636  				st.MakeNode().Name("node3").Label(topologyKey, topologyValues[2]).Obj(),
   637  				st.MakeNode().Name("node4").Label(topologyKey, topologyValues[0]).Obj(),
   638  				st.MakeNode().Name("node5").Label(topologyKey, topologyValues[1]).Obj(),
   639  				st.MakeNode().Name("node6").Label(topologyKey, topologyValues[2]).Obj(),
   640  			},
   641  			expectedNodeName: []string{"node3", "node6"},
   642  		},
   643  	}
   644  
   645  	for _, tt := range tests {
   646  		t.Run(tt.name, func(t *testing.T) {
   647  			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MatchLabelKeysInPodAffinity, tt.enableMatchLabelKeysInAffinity)()
   648  
   649  			testCtx := initTestSchedulerForPriorityTest(t, interpodaffinity.Name, interpodaffinity.Name)
   650  			if err := createNamespacesWithLabels(testCtx.ClientSet, []string{"ns1", "ns2"}, map[string]string{"team": "team1"}); err != nil {
   651  				t.Fatal(err)
   652  			}
   653  
   654  			for _, n := range tt.nodes {
   655  				if _, err := createNode(testCtx.ClientSet, n); err != nil {
   656  					t.Fatalf("failed to create node: %v", err)
   657  				}
   658  			}
   659  
   660  			for _, p := range tt.existingPods {
   661  				if _, err := runPausePod(testCtx.ClientSet, initPausePod(p)); err != nil {
   662  					t.Fatalf("failed to create existing pod: %v", err)
   663  				}
   664  			}
   665  
   666  			pod, err := runPausePod(testCtx.ClientSet, initPausePod(tt.pod))
   667  			if err != nil {
   668  				t.Fatalf("Error running pause pod: %v", err)
   669  			}
   670  
   671  			err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false, podScheduledIn(testCtx.ClientSet, pod.Namespace, pod.Name, tt.expectedNodeName))
   672  			if err != nil {
   673  				t.Errorf("Error while trying to wait for a pod to be scheduled: %v", err)
   674  			}
   675  		})
   676  	}
   677  }
   678  
   679  // TestImageLocalityScoring verifies that the scheduler's image locality priority function
   680  // works correctly, i.e., the pod gets scheduled to the node where its container images are ready.
   681  func TestImageLocalityScoring(t *testing.T) {
   682  	testCtx := initTestSchedulerForPriorityTest(t, "", imagelocality.Name)
   683  
   684  	// Create a node with the large image.
   685  	// We use a fake large image as the test image used by the pod, which has
   686  	// relatively large image size.
   687  	imageName := "fake-large-image:v1"
   688  	nodeWithLargeImage, err := createNode(
   689  		testCtx.ClientSet,
   690  		st.MakeNode().Name("testnode-large-image").Images(map[string]int64{imageName: 3000 * 1024 * 1024}).Obj(),
   691  	)
   692  	if err != nil {
   693  		t.Fatalf("cannot create node with a large image: %v", err)
   694  	}
   695  
   696  	// Add a few nodes.
   697  	_, err = createAndWaitForNodesInCache(testCtx, "testnode", st.MakeNode(), 10)
   698  	if err != nil {
   699  		t.Fatal(err)
   700  	}
   701  
   702  	// Create a pod with containers each having the specified image.
   703  	podName := "pod-using-large-image"
   704  	pod, err := runPodWithContainers(testCtx.ClientSet, initPodWithContainers(testCtx.ClientSet, &testutils.PodWithContainersConfig{
   705  		Name:       podName,
   706  		Namespace:  testCtx.NS.Name,
   707  		Containers: makeContainersWithImages([]string{imageName}),
   708  	}))
   709  	if err != nil {
   710  		t.Fatalf("error running pod with images: %v", err)
   711  	}
   712  	if pod.Spec.NodeName != nodeWithLargeImage.Name {
   713  		t.Errorf("pod %v got scheduled on an unexpected node: %v. Expected node: %v.", podName, pod.Spec.NodeName, nodeWithLargeImage.Name)
   714  	} else {
   715  		t.Logf("pod %v got successfully scheduled on node %v.", podName, pod.Spec.NodeName)
   716  	}
   717  }
   718  
   719  // makeContainerWithImage returns a list of v1.Container objects for each given image. Duplicates of an image are ignored,
   720  // i.e., each image is used only once.
   721  func makeContainersWithImages(images []string) []v1.Container {
   722  	var containers []v1.Container
   723  	usedImages := make(map[string]struct{})
   724  
   725  	for _, image := range images {
   726  		if _, ok := usedImages[image]; !ok {
   727  			containers = append(containers, v1.Container{
   728  				Name:  strings.Replace(image, ":", "-", -1) + "-container",
   729  				Image: image,
   730  			})
   731  			usedImages[image] = struct{}{}
   732  		}
   733  	}
   734  	return containers
   735  }
   736  
   737  // TestPodTopologySpreadScoring verifies that the PodTopologySpread Score plugin works.
   738  func TestPodTopologySpreadScoring(t *testing.T) {
   739  	pause := imageutils.GetPauseImageName()
   740  	taint := v1.Taint{
   741  		Key:    "k1",
   742  		Value:  "v1",
   743  		Effect: v1.TaintEffectNoSchedule,
   744  	}
   745  
   746  	//  default nodes with labels "zone: zone-{0,1}" and "node: <node name>".
   747  	defaultNodes := []*v1.Node{
   748  		st.MakeNode().Name("node-0").Label("node", "node-0").Label("zone", "zone-0").Taints([]v1.Taint{taint}).Obj(),
   749  		st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-0").Obj(),
   750  		st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Obj(),
   751  		st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-1").Obj(),
   752  	}
   753  
   754  	tests := []struct {
   755  		name                      string
   756  		incomingPod               *v1.Pod
   757  		existingPods              []*v1.Pod
   758  		fits                      bool
   759  		nodes                     []*v1.Node
   760  		want                      []string // nodes expected to schedule onto
   761  		enableNodeInclusionPolicy bool
   762  		enableMatchLabelKeys      bool
   763  	}{
   764  		// note: naming starts at index 0
   765  		// the symbol ~X~ means that node is infeasible
   766  		{
   767  			name: "place pod on a ~0~/1/2/3 cluster with MaxSkew=1, node-1 is the preferred fit",
   768  			incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
   769  				SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil).
   770  				Obj(),
   771  			existingPods: []*v1.Pod{
   772  				st.MakePod().Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(),
   773  				st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
   774  				st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
   775  				st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
   776  				st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(),
   777  				st.MakePod().Name("p3c").Node("node-3").Label("foo", "").Container(pause).Obj(),
   778  			},
   779  			fits:  true,
   780  			nodes: defaultNodes,
   781  			want:  []string{"node-1"},
   782  		},
   783  		{
   784  			name: "combined with hardSpread constraint on a ~4~/0/1/2 cluster",
   785  			incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
   786  				SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil).
   787  				SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil).
   788  				Obj(),
   789  			existingPods: []*v1.Pod{
   790  				st.MakePod().Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(),
   791  				st.MakePod().Name("p0b").Node("node-0").Label("foo", "").Container(pause).Obj(),
   792  				st.MakePod().Name("p0c").Node("node-0").Label("foo", "").Container(pause).Obj(),
   793  				st.MakePod().Name("p0d").Node("node-0").Label("foo", "").Container(pause).Obj(),
   794  				st.MakePod().Name("p2").Node("node-2").Label("foo", "").Container(pause).Obj(),
   795  				st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
   796  				st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(),
   797  			},
   798  			fits:  true,
   799  			nodes: defaultNodes,
   800  			want:  []string{"node-2"},
   801  		},
   802  		{
   803  			// 1. to fulfil "zone" constraint, pods spread across zones as ~3~/0
   804  			// 2. to fulfil "node" constraint, pods spread across zones as 1/~2~/0/~0~
   805  			// node-2 and node 4 are filtered out by plugins
   806  			name: "soft constraint with two node inclusion Constraints, zone: honor/ignore, node: honor/ignore",
   807  			incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
   808  				NodeSelector(map[string]string{"foo": ""}).
   809  				SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil).
   810  				SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, nil).
   811  				Obj(),
   812  			existingPods: []*v1.Pod{
   813  				st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
   814  				st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
   815  				st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
   816  				st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
   817  			},
   818  			fits: true,
   819  			nodes: []*v1.Node{
   820  				st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
   821  				st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(),
   822  				st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Label("foo", "").Obj(),
   823  				st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Obj(),
   824  			},
   825  			want:                      []string{"node-3"},
   826  			enableNodeInclusionPolicy: true,
   827  		},
   828  		{
   829  			// 1. to fulfil "zone" constraint, pods spread across zones as ~3~/~1~
   830  			// 2. to fulfil "node" constraint, pods spread across zones as 1/~0~/0/~0~
   831  			// node-2 and node 4 are filtered out by plugins
   832  			name: "soft constraint with two node inclusion Constraints, zone: ignore/ignore, node: honor/honor",
   833  			incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
   834  				NodeSelector(map[string]string{"foo": ""}).
   835  				SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, &ignorePolicy, nil, nil).
   836  				SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, &honorPolicy, nil).
   837  				Obj(),
   838  			existingPods: []*v1.Pod{
   839  				st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
   840  				st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
   841  				st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
   842  				st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
   843  			},
   844  			fits: true,
   845  			nodes: []*v1.Node{
   846  				st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
   847  				st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(),
   848  				st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Label("foo", "").Obj(),
   849  				st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Obj(),
   850  			},
   851  			want:                      []string{"node-3"},
   852  			enableNodeInclusionPolicy: true,
   853  		},
   854  		{
   855  			name: "matchLabelKeys ignored when feature gate disabled, node-1 is the preferred fit",
   856  			incomingPod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").Container(pause).
   857  				SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, []string{"bar"}).
   858  				Obj(),
   859  			existingPods: []*v1.Pod{
   860  				st.MakePod().Name("p1").Node("node-1").Label("foo", "").Label("bar", "").Container(pause).Obj(),
   861  				st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
   862  				st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
   863  				st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Label("bar", "").Container(pause).Obj(),
   864  				st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Label("bar", "").Container(pause).Obj(),
   865  				st.MakePod().Name("p3c").Node("node-3").Label("foo", "").Container(pause).Obj(),
   866  			},
   867  			fits:                 true,
   868  			nodes:                defaultNodes,
   869  			want:                 []string{"node-1"},
   870  			enableMatchLabelKeys: false,
   871  		},
   872  		{
   873  			name: "matchLabelKeys ANDed with LabelSelector when LabelSelector isn't empty, node-2 is the preferred fit",
   874  			incomingPod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").Container(pause).
   875  				SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil, []string{"bar"}).
   876  				Obj(),
   877  			existingPods: []*v1.Pod{
   878  				st.MakePod().Name("p1").Node("node-1").Label("foo", "").Label("bar", "").Container(pause).Obj(),
   879  				st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
   880  				st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
   881  				st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Label("bar", "").Container(pause).Obj(),
   882  				st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Label("bar", "").Container(pause).Obj(),
   883  				st.MakePod().Name("p3c").Node("node-3").Label("foo", "").Container(pause).Obj(),
   884  			},
   885  			fits:                 true,
   886  			nodes:                defaultNodes,
   887  			want:                 []string{"node-2"},
   888  			enableMatchLabelKeys: true,
   889  		},
   890  		{
   891  			name: "matchLabelKeys ANDed with LabelSelector when LabelSelector is empty, node-1 is the preferred fit",
   892  			incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
   893  				SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Obj(), nil, nil, nil, []string{"foo"}).
   894  				Obj(),
   895  			existingPods: []*v1.Pod{
   896  				st.MakePod().Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(),
   897  				st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
   898  				st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
   899  				st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
   900  				st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(),
   901  				st.MakePod().Name("p3c").Node("node-3").Label("foo", "").Container(pause).Obj(),
   902  			},
   903  			fits:                 true,
   904  			nodes:                defaultNodes,
   905  			want:                 []string{"node-1"},
   906  			enableMatchLabelKeys: true,
   907  		},
   908  	}
   909  	for _, tt := range tests {
   910  		t.Run(tt.name, func(t *testing.T) {
   911  			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeInclusionPolicyInPodTopologySpread, tt.enableNodeInclusionPolicy)()
   912  			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MatchLabelKeysInPodTopologySpread, tt.enableMatchLabelKeys)()
   913  
   914  			testCtx := initTestSchedulerForPriorityTest(t, podtopologyspread.Name, podtopologyspread.Name)
   915  			cs := testCtx.ClientSet
   916  			ns := testCtx.NS.Name
   917  
   918  			for i := range tt.nodes {
   919  				if _, err := createNode(cs, tt.nodes[i]); err != nil {
   920  					t.Fatalf("Cannot create node: %v", err)
   921  				}
   922  			}
   923  
   924  			// set namespace to pods
   925  			for i := range tt.existingPods {
   926  				tt.existingPods[i].SetNamespace(ns)
   927  			}
   928  			tt.incomingPod.SetNamespace(ns)
   929  
   930  			allPods := append(tt.existingPods, tt.incomingPod)
   931  			defer testutils.CleanupPods(testCtx.Ctx, cs, t, allPods)
   932  			for _, pod := range tt.existingPods {
   933  				createdPod, err := cs.CoreV1().Pods(pod.Namespace).Create(testCtx.Ctx, pod, metav1.CreateOptions{})
   934  				if err != nil {
   935  					t.Fatalf("Test Failed: error while creating pod during test: %v", err)
   936  				}
   937  				err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false,
   938  					testutils.PodScheduled(cs, createdPod.Namespace, createdPod.Name))
   939  				if err != nil {
   940  					t.Errorf("Test Failed: error while waiting for pod during test: %v", err)
   941  				}
   942  			}
   943  
   944  			testPod, err := cs.CoreV1().Pods(tt.incomingPod.Namespace).Create(testCtx.Ctx, tt.incomingPod, metav1.CreateOptions{})
   945  			if err != nil {
   946  				t.Fatalf("Test Failed: error while creating pod during test: %v", err)
   947  			}
   948  
   949  			if tt.fits {
   950  				err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false,
   951  					podScheduledIn(cs, testPod.Namespace, testPod.Name, tt.want))
   952  			} else {
   953  				err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false,
   954  					podUnschedulable(cs, testPod.Namespace, testPod.Name))
   955  			}
   956  			if err != nil {
   957  				t.Errorf("Test Failed: %v", err)
   958  			}
   959  		})
   960  	}
   961  }
   962  
   963  // TestDefaultPodTopologySpreadScoring verifies that the PodTopologySpread Score plugin
   964  // with the system default spreading spreads Pods belonging to a Service.
   965  // The setup has 300 nodes over 3 zones.
   966  func TestDefaultPodTopologySpreadScoring(t *testing.T) {
   967  	testCtx := initTestSchedulerForPriorityTest(t, podtopologyspread.Name, podtopologyspread.Name)
   968  	cs := testCtx.ClientSet
   969  	ns := testCtx.NS.Name
   970  
   971  	zoneForNode := make(map[string]string)
   972  	for i := 0; i < 300; i++ {
   973  		nodeName := fmt.Sprintf("node-%d", i)
   974  		zone := fmt.Sprintf("zone-%d", i%3)
   975  		zoneForNode[nodeName] = zone
   976  		_, err := createNode(cs, st.MakeNode().Name(nodeName).Label(v1.LabelHostname, nodeName).Label(v1.LabelTopologyZone, zone).Obj())
   977  		if err != nil {
   978  			t.Fatalf("Cannot create node: %v", err)
   979  		}
   980  	}
   981  
   982  	serviceName := "test-service"
   983  	svc := &v1.Service{
   984  		ObjectMeta: metav1.ObjectMeta{
   985  			Name:      serviceName,
   986  			Namespace: ns,
   987  		},
   988  		Spec: v1.ServiceSpec{
   989  			Selector: map[string]string{
   990  				"service": serviceName,
   991  			},
   992  			Ports: []v1.ServicePort{{
   993  				Port:       80,
   994  				TargetPort: intstr.FromInt32(80),
   995  			}},
   996  		},
   997  	}
   998  	_, err := cs.CoreV1().Services(ns).Create(testCtx.Ctx, svc, metav1.CreateOptions{})
   999  	if err != nil {
  1000  		t.Fatalf("Cannot create Service: %v", err)
  1001  	}
  1002  
  1003  	pause := imageutils.GetPauseImageName()
  1004  	totalPodCnt := 0
  1005  	for _, nPods := range []int{3, 9, 15} {
  1006  		// Append nPods each iteration.
  1007  		t.Run(fmt.Sprintf("%d-pods", totalPodCnt+nPods), func(t *testing.T) {
  1008  			for i := 0; i < nPods; i++ {
  1009  				p := st.MakePod().Name(fmt.Sprintf("p-%d", totalPodCnt)).Label("service", serviceName).Container(pause).Obj()
  1010  				_, err = cs.CoreV1().Pods(ns).Create(testCtx.Ctx, p, metav1.CreateOptions{})
  1011  				if err != nil {
  1012  					t.Fatalf("Cannot create Pod: %v", err)
  1013  				}
  1014  				totalPodCnt++
  1015  			}
  1016  			var pods []v1.Pod
  1017  			// Wait for all Pods scheduled.
  1018  			err = wait.PollUntilContextTimeout(testCtx.Ctx, pollInterval, wait.ForeverTestTimeout, false, func(ctx context.Context) (bool, error) {
  1019  				podList, err := cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{})
  1020  				if err != nil {
  1021  					t.Fatalf("Cannot list pods to verify scheduling: %v", err)
  1022  				}
  1023  				for _, p := range podList.Items {
  1024  					if p.Spec.NodeName == "" {
  1025  						return false, nil
  1026  					}
  1027  				}
  1028  				pods = podList.Items
  1029  				return true, nil
  1030  			})
  1031  			// Verify zone spreading.
  1032  			zoneCnts := make(map[string]int)
  1033  			for _, p := range pods {
  1034  				zoneCnts[zoneForNode[p.Spec.NodeName]]++
  1035  			}
  1036  			maxCnt := 0
  1037  			minCnt := len(pods)
  1038  			for _, c := range zoneCnts {
  1039  				if c > maxCnt {
  1040  					maxCnt = c
  1041  				}
  1042  				if c < minCnt {
  1043  					minCnt = c
  1044  				}
  1045  			}
  1046  			if skew := maxCnt - minCnt; skew != 0 {
  1047  				t.Errorf("Zone skew is %d, should be 0", skew)
  1048  			}
  1049  		})
  1050  	}
  1051  }
  1052  

View as plain text