server_test.go

Documentation: k8s.io/kubernetes/cmd/kube-scheduler/app

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package app
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"net"
    23  	"net/http"
    24  	"net/http/httptest"
    25  	"os"
    26  	"path/filepath"
    27  	"testing"
    28  	"time"
    29  
    30  	"github.com/google/go-cmp/cmp"
    31  	"github.com/spf13/pflag"
    32  	v1 "k8s.io/api/core/v1"
    33  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    34  	"k8s.io/apimachinery/pkg/runtime"
    35  	"k8s.io/apiserver/pkg/util/feature"
    36  	componentbaseconfig "k8s.io/component-base/config"
    37  	"k8s.io/component-base/featuregate"
    38  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    39  	configv1 "k8s.io/kube-scheduler/config/v1"
    40  	"k8s.io/kubernetes/cmd/kube-scheduler/app/options"
    41  	"k8s.io/kubernetes/pkg/features"
    42  	"k8s.io/kubernetes/pkg/scheduler/apis/config"
    43  	"k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults"
    44  	"k8s.io/kubernetes/pkg/scheduler/framework"
    45  )
    46  
    47  func TestSetup(t *testing.T) {
    48  	// temp dir
    49  	tmpDir, err := os.MkdirTemp("", "scheduler-options")
    50  	if err != nil {
    51  		t.Fatal(err)
    52  	}
    53  	defer os.RemoveAll(tmpDir)
    54  
    55  	// https server
    56  	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
    57  		w.WriteHeader(200)
    58  		w.Write([]byte(`ok`))
    59  	}))
    60  	defer server.Close()
    61  
    62  	configKubeconfig := filepath.Join(tmpDir, "config.kubeconfig")
    63  	if err := os.WriteFile(configKubeconfig, []byte(fmt.Sprintf(`
    64  apiVersion: v1
    65  kind: Config
    66  clusters:
    67  - cluster:
    68      insecure-skip-tls-verify: true
    69      server: %s
    70    name: default
    71  contexts:
    72  - context:
    73      cluster: default
    74      user: default
    75    name: default
    76  current-context: default
    77  users:
    78  - name: default
    79    user:
    80      username: config
    81  `, server.URL)), os.FileMode(0600)); err != nil {
    82  		t.Fatal(err)
    83  	}
    84  
    85  	// plugin config
    86  	pluginConfigFilev1 := filepath.Join(tmpDir, "pluginv1.yaml")
    87  	if err := os.WriteFile(pluginConfigFilev1, []byte(fmt.Sprintf(`
    88  apiVersion: kubescheduler.config.k8s.io/v1
    89  kind: KubeSchedulerConfiguration
    90  clientConnection:
    91    kubeconfig: '%s'
    92  profiles:
    93  - plugins:
    94      multiPoint:
    95        enabled:
    96        - name: SchedulingGates
    97        - name: DefaultBinder
    98        - name: PrioritySort
    99        - name: DefaultPreemption
   100        - name: VolumeBinding
   101        - name: NodeResourcesFit
   102        - name: NodePorts
   103        - name: InterPodAffinity
   104        - name: TaintToleration
   105        disabled:
   106        - name: "*"
   107      preFilter:
   108        disabled:
   109        - name: VolumeBinding
   110        - name: InterPodAffinity
   111      filter:
   112        disabled:
   113        - name: VolumeBinding
   114        - name: InterPodAffinity
   115        - name: TaintToleration
   116      score:
   117        disabled:
   118        - name: VolumeBinding
   119        - name: NodeResourcesFit
   120  `, configKubeconfig)), os.FileMode(0600)); err != nil {
   121  		t.Fatal(err)
   122  	}
   123  
   124  	// plugin config
   125  	simplifiedPluginConfigFilev1 := filepath.Join(tmpDir, "simplifiedPluginv1.yaml")
   126  	if err := os.WriteFile(simplifiedPluginConfigFilev1, []byte(fmt.Sprintf(`
   127  apiVersion: kubescheduler.config.k8s.io/v1
   128  kind: KubeSchedulerConfiguration
   129  clientConnection:
   130    kubeconfig: '%s'
   131  profiles:
   132  - schedulerName: simplified-scheduler
   133  `, configKubeconfig)), os.FileMode(0600)); err != nil {
   134  		t.Fatal(err)
   135  	}
   136  
   137  	// out-of-tree plugin config v1
   138  	outOfTreePluginConfigFilev1 := filepath.Join(tmpDir, "outOfTreePluginv1.yaml")
   139  	if err := os.WriteFile(outOfTreePluginConfigFilev1, []byte(fmt.Sprintf(`
   140  apiVersion: kubescheduler.config.k8s.io/v1
   141  kind: KubeSchedulerConfiguration
   142  clientConnection:
   143    kubeconfig: '%s'
   144  profiles:
   145  - plugins:
   146      preFilter:
   147        enabled:
   148        - name: Foo
   149      filter:
   150        enabled:
   151        - name: Foo
   152  `, configKubeconfig)), os.FileMode(0600)); err != nil {
   153  		t.Fatal(err)
   154  	}
   155  
   156  	// multiple profiles config
   157  	multiProfilesConfig := filepath.Join(tmpDir, "multi-profiles.yaml")
   158  	if err := os.WriteFile(multiProfilesConfig, []byte(fmt.Sprintf(`
   159  apiVersion: kubescheduler.config.k8s.io/v1
   160  kind: KubeSchedulerConfiguration
   161  clientConnection:
   162    kubeconfig: '%s'
   163  profiles:
   164  - schedulerName: "profile-default-plugins"
   165  - schedulerName: "profile-disable-all-filter-and-score-plugins"
   166    plugins:
   167      preFilter:
   168        disabled:
   169        - name: "*"
   170      filter:
   171        disabled:
   172        - name: "*"
   173      postFilter:
   174        disabled:
   175        - name: "*"
   176      preScore:
   177        disabled:
   178        - name: "*"
   179      score:
   180        disabled:
   181        - name: "*"
   182  `, configKubeconfig)), os.FileMode(0600)); err != nil {
   183  		t.Fatal(err)
   184  	}
   185  
   186  	// empty leader-election config
   187  	emptyLeaderElectionConfig := filepath.Join(tmpDir, "empty-leader-election-config.yaml")
   188  	if err := os.WriteFile(emptyLeaderElectionConfig, []byte(fmt.Sprintf(`
   189  apiVersion: kubescheduler.config.k8s.io/v1
   190  kind: KubeSchedulerConfiguration
   191  clientConnection:
   192    kubeconfig: '%s'
   193  `, configKubeconfig)), os.FileMode(0600)); err != nil {
   194  		t.Fatal(err)
   195  	}
   196  
   197  	// leader-election config
   198  	leaderElectionConfig := filepath.Join(tmpDir, "leader-election-config.yaml")
   199  	if err := os.WriteFile(leaderElectionConfig, []byte(fmt.Sprintf(`
   200  apiVersion: kubescheduler.config.k8s.io/v1
   201  kind: KubeSchedulerConfiguration
   202  clientConnection:
   203    kubeconfig: '%s'
   204  leaderElection:
   205    leaseDuration: 1h
   206  `, configKubeconfig)), os.FileMode(0600)); err != nil {
   207  		t.Fatal(err)
   208  	}
   209  
   210  	testcases := []struct {
   211  		name                 string
   212  		flags                []string
   213  		registryOptions      []Option
   214  		restoreFeatures      map[featuregate.Feature]bool
   215  		wantPlugins          map[string]*config.Plugins
   216  		wantLeaderElection   *componentbaseconfig.LeaderElectionConfiguration
   217  		wantClientConnection *componentbaseconfig.ClientConnectionConfiguration
   218  	}{
   219  		{
   220  			name: "default config with an alpha feature enabled",
   221  			flags: []string{
   222  				"--kubeconfig", configKubeconfig,
   223  				"--feature-gates=VolumeCapacityPriority=true",
   224  			},
   225  			wantPlugins: map[string]*config.Plugins{
   226  				"default-scheduler": defaults.ExpandedPluginsV1,
   227  			},
   228  			restoreFeatures: map[featuregate.Feature]bool{
   229  				features.VolumeCapacityPriority: false,
   230  			},
   231  		},
   232  		{
   233  			name: "component configuration v1 with only scheduler name configured",
   234  			flags: []string{
   235  				"--config", simplifiedPluginConfigFilev1,
   236  				"--kubeconfig", configKubeconfig,
   237  			},
   238  			wantPlugins: map[string]*config.Plugins{
   239  				"simplified-scheduler": defaults.ExpandedPluginsV1,
   240  			},
   241  		},
   242  		{
   243  			name: "default config",
   244  			flags: []string{
   245  				"--kubeconfig", configKubeconfig,
   246  			},
   247  			wantPlugins: map[string]*config.Plugins{
   248  				"default-scheduler": defaults.ExpandedPluginsV1,
   249  			},
   250  		},
   251  		{
   252  			name: "component configuration v1",
   253  			flags: []string{
   254  				"--config", pluginConfigFilev1,
   255  				"--kubeconfig", configKubeconfig,
   256  			},
   257  			wantPlugins: map[string]*config.Plugins{
   258  				"default-scheduler": func() *config.Plugins {
   259  					plugins := defaults.ExpandedPluginsV1.DeepCopy()
   260  					plugins.Filter.Enabled = []config.Plugin{
   261  						{Name: "NodeResourcesFit"},
   262  						{Name: "NodePorts"},
   263  					}
   264  					plugins.PreFilter.Enabled = []config.Plugin{
   265  						{Name: "NodeResourcesFit"},
   266  						{Name: "NodePorts"},
   267  					}
   268  					plugins.PreScore.Enabled = []config.Plugin{
   269  						{Name: "VolumeBinding"},
   270  						{Name: "NodeResourcesFit"},
   271  						{Name: "InterPodAffinity"},
   272  						{Name: "TaintToleration"},
   273  					}
   274  					plugins.Score.Enabled = []config.Plugin{
   275  						{Name: "InterPodAffinity", Weight: 1},
   276  						{Name: "TaintToleration", Weight: 1},
   277  					}
   278  					return plugins
   279  				}(),
   280  			},
   281  		},
   282  		{
   283  			name: "out-of-tree component configuration v1",
   284  			flags: []string{
   285  				"--config", outOfTreePluginConfigFilev1,
   286  				"--kubeconfig", configKubeconfig,
   287  			},
   288  			registryOptions: []Option{WithPlugin("Foo", newFoo)},
   289  			wantPlugins: map[string]*config.Plugins{
   290  				"default-scheduler": func() *config.Plugins {
   291  					plugins := defaults.ExpandedPluginsV1.DeepCopy()
   292  					plugins.PreFilter.Enabled = append(plugins.PreFilter.Enabled, config.Plugin{Name: "Foo"})
   293  					plugins.Filter.Enabled = append(plugins.Filter.Enabled, config.Plugin{Name: "Foo"})
   294  					return plugins
   295  				}(),
   296  			},
   297  		},
   298  		{
   299  			name: "leader election CLI args, along with --config arg",
   300  			flags: []string{
   301  				"--leader-elect=false",
   302  				"--leader-elect-lease-duration=2h", // CLI args are favored over the fields in ComponentConfig
   303  				"--kubeconfig=foo",                 // deprecated CLI arg will be ignored if --config is specified
   304  				"--config", emptyLeaderElectionConfig,
   305  			},
   306  			wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{
   307  				LeaderElect:       false,                                    // from CLI args
   308  				LeaseDuration:     metav1.Duration{Duration: 2 * time.Hour}, // from CLI args
   309  				RenewDeadline:     metav1.Duration{Duration: 10 * time.Second},
   310  				RetryPeriod:       metav1.Duration{Duration: 2 * time.Second},
   311  				ResourceLock:      "leases",
   312  				ResourceName:      configv1.SchedulerDefaultLockObjectName,
   313  				ResourceNamespace: configv1.SchedulerDefaultLockObjectNamespace,
   314  			},
   315  			wantClientConnection: &componentbaseconfig.ClientConnectionConfiguration{
   316  				Kubeconfig:  configKubeconfig,
   317  				ContentType: "application/vnd.kubernetes.protobuf",
   318  				QPS:         50,
   319  				Burst:       100,
   320  			},
   321  		},
   322  		{
   323  			name: "leader election CLI args, without --config arg",
   324  			flags: []string{
   325  				"--leader-elect=false",
   326  				"--leader-elect-lease-duration=2h",
   327  				"--leader-elect-resource-namespace=default",
   328  				"--kubeconfig", configKubeconfig, // deprecated CLI arg is honored if --config is not specified
   329  			},
   330  			wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{
   331  				LeaderElect:       false,                                    // from CLI args
   332  				LeaseDuration:     metav1.Duration{Duration: 2 * time.Hour}, // from CLI args
   333  				RenewDeadline:     metav1.Duration{Duration: 10 * time.Second},
   334  				RetryPeriod:       metav1.Duration{Duration: 2 * time.Second},
   335  				ResourceLock:      "leases",
   336  				ResourceName:      configv1.SchedulerDefaultLockObjectName,
   337  				ResourceNamespace: "default", // from CLI args
   338  			},
   339  			wantClientConnection: &componentbaseconfig.ClientConnectionConfiguration{
   340  				Kubeconfig:  configKubeconfig, // from deprecated CLI args
   341  				ContentType: "application/vnd.kubernetes.protobuf",
   342  				QPS:         50,
   343  				Burst:       100,
   344  			},
   345  		},
   346  		{
   347  			name: "leader election settings specified by ComponentConfig only",
   348  			flags: []string{
   349  				"--config", leaderElectionConfig,
   350  			},
   351  			wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{
   352  				LeaderElect:       true,
   353  				LeaseDuration:     metav1.Duration{Duration: 1 * time.Hour}, // from CC
   354  				RenewDeadline:     metav1.Duration{Duration: 10 * time.Second},
   355  				RetryPeriod:       metav1.Duration{Duration: 2 * time.Second},
   356  				ResourceLock:      "leases",
   357  				ResourceName:      configv1.SchedulerDefaultLockObjectName,
   358  				ResourceNamespace: configv1.SchedulerDefaultLockObjectNamespace,
   359  			},
   360  		},
   361  		{
   362  			name: "leader election settings specified by CLI args and ComponentConfig",
   363  			flags: []string{
   364  				"--leader-elect=true",
   365  				"--leader-elect-renew-deadline=5s",
   366  				"--leader-elect-retry-period=1s",
   367  				"--config", leaderElectionConfig,
   368  			},
   369  			wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{
   370  				LeaderElect:       true,
   371  				LeaseDuration:     metav1.Duration{Duration: 1 * time.Hour},   // from CC
   372  				RenewDeadline:     metav1.Duration{Duration: 5 * time.Second}, // from CLI args
   373  				RetryPeriod:       metav1.Duration{Duration: 1 * time.Second}, // from CLI args
   374  				ResourceLock:      "leases",
   375  				ResourceName:      configv1.SchedulerDefaultLockObjectName,
   376  				ResourceNamespace: configv1.SchedulerDefaultLockObjectNamespace,
   377  			},
   378  		},
   379  	}
   380  
   381  	makeListener := func(t *testing.T) net.Listener {
   382  		t.Helper()
   383  		l, err := net.Listen("tcp", ":0")
   384  		if err != nil {
   385  			t.Fatal(err)
   386  		}
   387  		return l
   388  	}
   389  
   390  	for _, tc := range testcases {
   391  		t.Run(tc.name, func(t *testing.T) {
   392  			for k, v := range tc.restoreFeatures {
   393  				defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)()
   394  			}
   395  
   396  			fs := pflag.NewFlagSet("test", pflag.PanicOnError)
   397  			opts := options.NewOptions()
   398  
   399  			// use listeners instead of static ports so parallel test runs don't conflict
   400  			opts.SecureServing.Listener = makeListener(t)
   401  			defer opts.SecureServing.Listener.Close()
   402  
   403  			nfs := opts.Flags
   404  			for _, f := range nfs.FlagSets {
   405  				fs.AddFlagSet(f)
   406  			}
   407  			if err := fs.Parse(tc.flags); err != nil {
   408  				t.Fatal(err)
   409  			}
   410  
   411  			// use listeners instead of static ports so parallel test runs don't conflict
   412  			opts.SecureServing.Listener = makeListener(t)
   413  			defer opts.SecureServing.Listener.Close()
   414  
   415  			ctx, cancel := context.WithCancel(context.Background())
   416  			defer cancel()
   417  			_, sched, err := Setup(ctx, opts, tc.registryOptions...)
   418  			if err != nil {
   419  				t.Fatal(err)
   420  			}
   421  
   422  			if tc.wantPlugins != nil {
   423  				gotPlugins := make(map[string]*config.Plugins)
   424  				for n, p := range sched.Profiles {
   425  					gotPlugins[n] = p.ListPlugins()
   426  				}
   427  
   428  				if diff := cmp.Diff(tc.wantPlugins, gotPlugins); diff != "" {
   429  					t.Errorf("Unexpected plugins diff (-want, +got): %s", diff)
   430  				}
   431  			}
   432  
   433  			if tc.wantLeaderElection != nil {
   434  				gotLeaderElection := opts.ComponentConfig.LeaderElection
   435  				if diff := cmp.Diff(*tc.wantLeaderElection, gotLeaderElection); diff != "" {
   436  					t.Errorf("Unexpected leaderElection diff (-want, +got): %s", diff)
   437  				}
   438  			}
   439  
   440  			if tc.wantClientConnection != nil {
   441  				gotClientConnection := opts.ComponentConfig.ClientConnection
   442  				if diff := cmp.Diff(*tc.wantClientConnection, gotClientConnection); diff != "" {
   443  					t.Errorf("Unexpected clientConnection diff (-want, +got): %s", diff)
   444  				}
   445  			}
   446  		})
   447  	}
   448  }
   449  
   450  // Simulates an out-of-tree plugin.
   451  type foo struct{}
   452  
   453  var _ framework.PreFilterPlugin = &foo{}
   454  var _ framework.FilterPlugin = &foo{}
   455  
   456  func (*foo) Name() string {
   457  	return "Foo"
   458  }
   459  
   460  func newFoo(_ context.Context, _ runtime.Object, _ framework.Handle) (framework.Plugin, error) {
   461  	return &foo{}, nil
   462  }
   463  
   464  func (*foo) PreFilter(_ context.Context, _ *framework.CycleState, _ *v1.Pod) (*framework.PreFilterResult, *framework.Status) {
   465  	return nil, nil
   466  }
   467  
   468  func (*foo) PreFilterExtensions() framework.PreFilterExtensions {
   469  	return nil
   470  }
   471  
   472  func (*foo) Filter(_ context.Context, _ *framework.CycleState, _ *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status {
   473  	return nil
   474  }
   475
View as plain text