balanced_allocation_test.go

Documentation: k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package noderesources
    18  
    19  import (
    20  	"context"
    21  	"reflect"
    22  	"testing"
    23  
    24  	v1 "k8s.io/api/core/v1"
    25  	"k8s.io/apimachinery/pkg/api/resource"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	"k8s.io/klog/v2/ktesting"
    28  	"k8s.io/kubernetes/pkg/scheduler/apis/config"
    29  	"k8s.io/kubernetes/pkg/scheduler/framework"
    30  	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
    31  	"k8s.io/kubernetes/pkg/scheduler/framework/runtime"
    32  	"k8s.io/kubernetes/pkg/scheduler/internal/cache"
    33  	st "k8s.io/kubernetes/pkg/scheduler/testing"
    34  	tf "k8s.io/kubernetes/pkg/scheduler/testing/framework"
    35  )
    36  
    37  func TestNodeResourcesBalancedAllocation(t *testing.T) {
    38  	cpuAndMemoryAndGPU := v1.PodSpec{
    39  		Containers: []v1.Container{
    40  			{
    41  				Resources: v1.ResourceRequirements{
    42  					Requests: v1.ResourceList{
    43  						v1.ResourceCPU:    resource.MustParse("1000m"),
    44  						v1.ResourceMemory: resource.MustParse("2000"),
    45  					},
    46  				},
    47  			},
    48  			{
    49  				Resources: v1.ResourceRequirements{
    50  					Requests: v1.ResourceList{
    51  						v1.ResourceCPU:    resource.MustParse("2000m"),
    52  						v1.ResourceMemory: resource.MustParse("3000"),
    53  						"nvidia.com/gpu":  resource.MustParse("3"),
    54  					},
    55  				},
    56  			},
    57  		},
    58  		NodeName: "node1",
    59  	}
    60  	labels1 := map[string]string{
    61  		"foo": "bar",
    62  		"baz": "blah",
    63  	}
    64  	labels2 := map[string]string{
    65  		"bar": "foo",
    66  		"baz": "blah",
    67  	}
    68  	cpuOnly := v1.PodSpec{
    69  		NodeName: "node1",
    70  		Containers: []v1.Container{
    71  			{
    72  				Resources: v1.ResourceRequirements{
    73  					Requests: v1.ResourceList{
    74  						v1.ResourceCPU:    resource.MustParse("1000m"),
    75  						v1.ResourceMemory: resource.MustParse("0"),
    76  					},
    77  				},
    78  			},
    79  			{
    80  				Resources: v1.ResourceRequirements{
    81  					Requests: v1.ResourceList{
    82  						v1.ResourceCPU:    resource.MustParse("2000m"),
    83  						v1.ResourceMemory: resource.MustParse("0"),
    84  					},
    85  				},
    86  			},
    87  		},
    88  	}
    89  	cpuOnly2 := cpuOnly
    90  	cpuOnly2.NodeName = "node2"
    91  	cpuAndMemory := v1.PodSpec{
    92  		NodeName: "node2",
    93  		Containers: []v1.Container{
    94  			{
    95  				Resources: v1.ResourceRequirements{
    96  					Requests: v1.ResourceList{
    97  						v1.ResourceCPU:    resource.MustParse("1000m"),
    98  						v1.ResourceMemory: resource.MustParse("2000"),
    99  					},
   100  				},
   101  			},
   102  			{
   103  				Resources: v1.ResourceRequirements{
   104  					Requests: v1.ResourceList{
   105  						v1.ResourceCPU:    resource.MustParse("2000m"),
   106  						v1.ResourceMemory: resource.MustParse("3000"),
   107  					},
   108  				},
   109  			},
   110  		},
   111  	}
   112  
   113  	defaultResourceBalancedAllocationSet := []config.ResourceSpec{
   114  		{Name: string(v1.ResourceCPU), Weight: 1},
   115  		{Name: string(v1.ResourceMemory), Weight: 1},
   116  	}
   117  	scalarResource := map[string]int64{
   118  		"nvidia.com/gpu": 8,
   119  	}
   120  
   121  	tests := []struct {
   122  		pod          *v1.Pod
   123  		pods         []*v1.Pod
   124  		nodes        []*v1.Node
   125  		expectedList framework.NodeScoreList
   126  		name         string
   127  		args         config.NodeResourcesBalancedAllocationArgs
   128  		runPreScore  bool
   129  	}{
   130  		{
   131  			// Node1 scores (remaining resources) on 0-MaxNodeScore scale
   132  			// CPU Fraction: 0 / 4000 = 0%
   133  			// Memory Fraction: 0 / 10000 = 0%
   134  			// Node1 Score: (1-0) * MaxNodeScore = MaxNodeScore
   135  			// Node2 scores (remaining resources) on 0-MaxNodeScore scale
   136  			// CPU Fraction: 0 / 4000 = 0 %
   137  			// Memory Fraction: 0 / 10000 = 0%
   138  			// Node2 Score: (1-0) * MaxNodeScore = MaxNodeScore
   139  			pod:          st.MakePod().Obj(),
   140  			nodes:        []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 4000, 10000, nil)},
   141  			expectedList: []framework.NodeScore{{Name: "node1", Score: framework.MaxNodeScore}, {Name: "node2", Score: framework.MaxNodeScore}},
   142  			name:         "nothing scheduled, nothing requested",
   143  			args:         config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   144  			runPreScore:  true,
   145  		},
   146  		{
   147  			// Node1 scores on 0-MaxNodeScore scale
   148  			// CPU Fraction: 3000 / 4000= 75%
   149  			// Memory Fraction: 5000 / 10000 = 50%
   150  			// Node1 std: (0.75 - 0.5) / 2 = 0.125
   151  			// Node1 Score: (1 - 0.125)*MaxNodeScore = 87
   152  			// Node2 scores on 0-MaxNodeScore scale
   153  			// CPU Fraction: 3000 / 6000= 50%
   154  			// Memory Fraction: 5000/10000 = 50%
   155  			// Node2 std: 0
   156  			// Node2 Score: (1-0) * MaxNodeScore = MaxNodeScore
   157  			pod:          &v1.Pod{Spec: cpuAndMemory},
   158  			nodes:        []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 6000, 10000, nil)},
   159  			expectedList: []framework.NodeScore{{Name: "node1", Score: 87}, {Name: "node2", Score: framework.MaxNodeScore}},
   160  			name:         "nothing scheduled, resources requested, differently sized nodes",
   161  			args:         config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   162  			runPreScore:  true,
   163  		},
   164  		{
   165  			// Node1 scores on 0-MaxNodeScore scale
   166  			// CPU Fraction: 0 / 4000= 0%
   167  			// Memory Fraction: 0 / 10000 = 0%
   168  			// Node1 std: 0
   169  			// Node1 Score: (1-0) * MaxNodeScore = MaxNodeScore
   170  			// Node2 scores on 0-MaxNodeScore scale
   171  			// CPU Fraction: 0 / 4000= 0%
   172  			// Memory Fraction: 0 / 10000 = 0%
   173  			// Node2 std: 0
   174  			// Node2 Score: (1-0) * MaxNodeScore = MaxNodeScore
   175  			pod:          st.MakePod().Obj(),
   176  			nodes:        []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 4000, 10000, nil)},
   177  			expectedList: []framework.NodeScore{{Name: "node2", Score: framework.MaxNodeScore}, {Name: "node2", Score: framework.MaxNodeScore}},
   178  			name:         "no resources requested, pods without container scheduled",
   179  			pods: []*v1.Pod{
   180  				st.MakePod().Node("node1").Labels(labels2).Obj(),
   181  				st.MakePod().Node("node1").Labels(labels1).Obj(),
   182  				st.MakePod().Node("node2").Labels(labels1).Obj(),
   183  				st.MakePod().Node("node2").Labels(labels1).Obj(),
   184  			},
   185  			args:        config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   186  			runPreScore: true,
   187  		},
   188  		{
   189  			// Node1 scores on 0-MaxNodeScore scale
   190  			// CPU Fraction: 0 / 250 = 0%
   191  			// Memory Fraction: 0 / 1000 = 0%
   192  			// Node1 std: (0 - 0) / 2 = 0
   193  			// Node1 Score: (1 - 0)*MaxNodeScore = 100
   194  			// Node2 scores on 0-MaxNodeScore scale
   195  			// CPU Fraction: 0 / 250 = 0%
   196  			// Memory Fraction: 0 / 1000 = 0%
   197  			// Node2 std: (0 - 0) / 2 = 0
   198  			// Node2 Score: (1 - 0)*MaxNodeScore = 100
   199  			pod:          st.MakePod().Obj(),
   200  			nodes:        []*v1.Node{makeNode("node1", 250, 1000*1024*1024, nil), makeNode("node2", 250, 1000*1024*1024, nil)},
   201  			expectedList: []framework.NodeScore{{Name: "node1", Score: 100}, {Name: "node2", Score: 100}},
   202  			name:         "no resources requested, pods with container scheduled",
   203  			pods: []*v1.Pod{
   204  				st.MakePod().Node("node1").Obj(),
   205  				st.MakePod().Node("node1").Obj(),
   206  			},
   207  			args:        config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   208  			runPreScore: true,
   209  		},
   210  		{
   211  			// Node1 scores on 0-MaxNodeScore scale
   212  			// CPU Fraction: 6000 / 10000 = 60%
   213  			// Memory Fraction: 0 / 20000 = 0%
   214  			// Node1 std: (0.6 - 0) / 2 = 0.3
   215  			// Node1 Score: (1 - 0.3)*MaxNodeScore = 70
   216  			// Node2 scores on 0-MaxNodeScore scale
   217  			// CPU Fraction: 6000 / 10000 = 60%
   218  			// Memory Fraction: 5000 / 20000 = 25%
   219  			// Node2 std: (0.6 - 0.25) / 2 = 0.175
   220  			// Node2 Score: (1 - 0.175)*MaxNodeScore = 82
   221  			pod:          st.MakePod().Obj(),
   222  			nodes:        []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 20000, nil)},
   223  			expectedList: []framework.NodeScore{{Name: "node1", Score: 70}, {Name: "node2", Score: 82}},
   224  			name:         "no resources requested, pods scheduled with resources",
   225  			pods: []*v1.Pod{
   226  				{Spec: cpuOnly, ObjectMeta: metav1.ObjectMeta{Labels: labels2}},
   227  				{Spec: cpuOnly, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
   228  				{Spec: cpuOnly2, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
   229  				{Spec: cpuAndMemory, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
   230  			},
   231  			args:        config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   232  			runPreScore: true,
   233  		},
   234  		{
   235  			// Node1 scores on 0-MaxNodeScore scale
   236  			// CPU Fraction: 6000 / 10000 = 60%
   237  			// Memory Fraction: 5000 / 20000 = 25%
   238  			// Node1 std: (0.6 - 0.25) / 2 = 0.175
   239  			// Node1 Score: (1 - 0.175)*MaxNodeScore = 82
   240  			// Node2 scores on 0-MaxNodeScore scale
   241  			// CPU Fraction: 6000 / 10000 = 60%
   242  			// Memory Fraction: 10000 / 20000 = 50%
   243  			// Node2 std: (0.6 - 0.5) / 2 = 0.05
   244  			// Node2 Score: (1 - 0.05)*MaxNodeScore = 95
   245  			pod:          &v1.Pod{Spec: cpuAndMemory},
   246  			nodes:        []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 20000, nil)},
   247  			expectedList: []framework.NodeScore{{Name: "node1", Score: 82}, {Name: "node2", Score: 95}},
   248  			name:         "resources requested, pods scheduled with resources",
   249  			pods: []*v1.Pod{
   250  				{Spec: cpuOnly},
   251  				{Spec: cpuAndMemory},
   252  			},
   253  			args:        config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   254  			runPreScore: true,
   255  		},
   256  		{
   257  			// Node1 scores on 0-MaxNodeScore scale
   258  			// CPU Fraction: 6000 / 10000 = 60%
   259  			// Memory Fraction: 5000 / 20000 = 25%
   260  			// Node1 std: (0.6 - 0.25) / 2 = 0.175
   261  			// Node1 Score: (1 - 0.175)*MaxNodeScore = 82
   262  			// Node2 scores on 0-MaxNodeScore scale
   263  			// CPU Fraction: 6000 / 10000 = 60%
   264  			// Memory Fraction: 10000 / 50000 = 20%
   265  			// Node2 std: (0.6 - 0.2) / 2 = 0.2
   266  			// Node2 Score: (1 - 0.2)*MaxNodeScore = 80
   267  			pod:          &v1.Pod{Spec: cpuAndMemory},
   268  			nodes:        []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 50000, nil)},
   269  			expectedList: []framework.NodeScore{{Name: "node1", Score: 82}, {Name: "node2", Score: 80}},
   270  			name:         "resources requested, pods scheduled with resources, differently sized nodes",
   271  			pods: []*v1.Pod{
   272  				{Spec: cpuOnly},
   273  				{Spec: cpuAndMemory},
   274  			},
   275  			args:        config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   276  			runPreScore: true,
   277  		},
   278  		{
   279  			// Node1 scores on 0-MaxNodeScore scale
   280  			// CPU Fraction: 6000 / 6000 = 1
   281  			// Memory Fraction: 0 / 10000 = 0
   282  			// Node1 std: (1 - 0) / 2 = 0.5
   283  			// Node1 Score: (1 - 0.5)*MaxNodeScore = 50
   284  			// Node1 Score: MaxNodeScore - (1 - 0) * MaxNodeScore = 0
   285  			// Node2 scores on 0-MaxNodeScore scale
   286  			// CPU Fraction: 6000 / 6000 = 1
   287  			// Memory Fraction 5000 / 10000 = 50%
   288  			// Node2 std: (1 - 0.5) / 2 = 0.25
   289  			// Node2 Score: (1 - 0.25)*MaxNodeScore = 75
   290  			pod:          &v1.Pod{Spec: cpuOnly},
   291  			nodes:        []*v1.Node{makeNode("node1", 6000, 10000, nil), makeNode("node2", 6000, 10000, nil)},
   292  			expectedList: []framework.NodeScore{{Name: "node1", Score: 50}, {Name: "node2", Score: 75}},
   293  			name:         "requested resources at node capacity",
   294  			pods: []*v1.Pod{
   295  				{Spec: cpuOnly},
   296  				{Spec: cpuAndMemory},
   297  			},
   298  			args:        config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   299  			runPreScore: true,
   300  		},
   301  		{
   302  			pod:          st.MakePod().Obj(),
   303  			nodes:        []*v1.Node{makeNode("node1", 0, 0, nil), makeNode("node2", 0, 0, nil)},
   304  			expectedList: []framework.NodeScore{{Name: "node1", Score: 100}, {Name: "node2", Score: 100}},
   305  			name:         "zero node resources, pods scheduled with resources",
   306  			pods: []*v1.Pod{
   307  				{Spec: cpuOnly},
   308  				{Spec: cpuAndMemory},
   309  			},
   310  			args:        config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   311  			runPreScore: true,
   312  		},
   313  		// Node1 scores on 0-MaxNodeScore scale
   314  		// CPU Fraction: 3000 / 3500 = 85.71%
   315  		// Memory Fraction: 5000 / 40000 = 12.5%
   316  		// GPU Fraction: 4 / 8 = 0.5%
   317  		// Node1 std: sqrt(((0.8571 - 0.503) *  (0.8571 - 0.503) + (0.503 - 0.125) * (0.503 - 0.125) + (0.503 - 0.5) * (0.503 - 0.5)) / 3) = 0.3002
   318  		// Node1 Score: (1 - 0.3002)*MaxNodeScore = 70
   319  		// Node2 scores on 0-MaxNodeScore scale
   320  		// CPU Fraction: 3000 / 3500 = 85.71%
   321  		// Memory Fraction: 5000 / 40000 = 12.5%
   322  		// GPU Fraction: 1 / 8 = 12.5%
   323  		// Node2 std: sqrt(((0.8571 - 0.378) *  (0.8571 - 0.378) + (0.378 - 0.125) * (0.378 - 0.125)) + (0.378 - 0.125) * (0.378 - 0.125)) / 3) = 0.345
   324  		// Node2 Score: (1 - 0.358)*MaxNodeScore = 65
   325  		{
   326  			pod: st.MakePod().Req(map[v1.ResourceName]string{
   327  				v1.ResourceMemory: "0",
   328  				"nvidia.com/gpu":  "1",
   329  			}).Obj(),
   330  			nodes:        []*v1.Node{makeNode("node1", 3500, 40000, scalarResource), makeNode("node2", 3500, 40000, scalarResource)},
   331  			expectedList: []framework.NodeScore{{Name: "node1", Score: 70}, {Name: "node2", Score: 65}},
   332  			name:         "include scalar resource on a node for balanced resource allocation",
   333  			pods: []*v1.Pod{
   334  				{Spec: cpuAndMemory},
   335  				{Spec: cpuAndMemoryAndGPU},
   336  			},
   337  			args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{
   338  				{Name: string(v1.ResourceCPU), Weight: 1},
   339  				{Name: string(v1.ResourceMemory), Weight: 1},
   340  				{Name: "nvidia.com/gpu", Weight: 1},
   341  			}},
   342  			runPreScore: true,
   343  		},
   344  		// Only one node (node1) has the scalar resource, pod doesn't request the scalar resource and the scalar resource should be skipped for consideration.
   345  		// Node1: std = 0, score = 100
   346  		// Node2: std = 0, score = 100
   347  		{
   348  			pod:          st.MakePod().Obj(),
   349  			nodes:        []*v1.Node{makeNode("node1", 3500, 40000, scalarResource), makeNode("node2", 3500, 40000, nil)},
   350  			expectedList: []framework.NodeScore{{Name: "node1", Score: 100}, {Name: "node2", Score: 100}},
   351  			name:         "node without the scalar resource results to a higher score",
   352  			pods: []*v1.Pod{
   353  				{Spec: cpuOnly},
   354  				{Spec: cpuOnly2},
   355  			},
   356  			args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{
   357  				{Name: string(v1.ResourceCPU), Weight: 1},
   358  				{Name: "nvidia.com/gpu", Weight: 1},
   359  			}},
   360  			runPreScore: true,
   361  		},
   362  		{
   363  			// Node1 scores on 0-MaxNodeScore scale
   364  			// CPU Fraction: 6000 / 10000 = 60%
   365  			// Memory Fraction: 5000 / 20000 = 25%
   366  			// Node1 std: (0.6 - 0.25) / 2 = 0.175
   367  			// Node1 Score: (1 - 0.175)*MaxNodeScore = 82
   368  			// Node2 scores on 0-MaxNodeScore scale
   369  			// CPU Fraction: 6000 / 10000 = 60%
   370  			// Memory Fraction: 10000 / 20000 = 50%
   371  			// Node2 std: (0.6 - 0.5) / 2 = 0.05
   372  			// Node2 Score: (1 - 0.05)*MaxNodeScore = 95
   373  			pod:          &v1.Pod{Spec: cpuAndMemory},
   374  			nodes:        []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 20000, nil)},
   375  			expectedList: []framework.NodeScore{{Name: "node1", Score: 82}, {Name: "node2", Score: 95}},
   376  			name:         "resources requested, pods scheduled with resources if PreScore not called",
   377  			pods: []*v1.Pod{
   378  				{Spec: cpuOnly},
   379  				{Spec: cpuAndMemory},
   380  			},
   381  			args:        config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
   382  			runPreScore: false,
   383  		},
   384  	}
   385  
   386  	for _, test := range tests {
   387  		t.Run(test.name, func(t *testing.T) {
   388  			snapshot := cache.NewSnapshot(test.pods, test.nodes)
   389  			_, ctx := ktesting.NewTestContext(t)
   390  			ctx, cancel := context.WithCancel(ctx)
   391  			defer cancel()
   392  			fh, _ := runtime.NewFramework(ctx, nil, nil, runtime.WithSnapshotSharedLister(snapshot))
   393  			p, _ := NewBalancedAllocation(ctx, &test.args, fh, feature.Features{})
   394  			state := framework.NewCycleState()
   395  			for i := range test.nodes {
   396  				if test.runPreScore {
   397  					status := p.(framework.PreScorePlugin).PreScore(ctx, state, test.pod, tf.BuildNodeInfos(test.nodes))
   398  					if !status.IsSuccess() {
   399  						t.Errorf("PreScore is expected to return success, but didn't. Got status: %v", status)
   400  					}
   401  				}
   402  				hostResult, status := p.(framework.ScorePlugin).Score(ctx, state, test.pod, test.nodes[i].Name)
   403  				if !status.IsSuccess() {
   404  					t.Errorf("Score is expected to return success, but didn't. Got status: %v", status)
   405  				}
   406  				if !reflect.DeepEqual(test.expectedList[i].Score, hostResult) {
   407  					t.Errorf("got score %v for host %v, expected %v", hostResult, test.nodes[i].Name, test.expectedList[i].Score)
   408  				}
   409  			}
   410  		})
   411  	}
   412  }
   413
View as plain text