options.go

Documentation: k8s.io/kubernetes/cmd/kube-scheduler/app/options

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package options
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"net"
    23  	"os"
    24  	"time"
    25  
    26  	corev1 "k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/apimachinery/pkg/util/uuid"
    29  	apiserveroptions "k8s.io/apiserver/pkg/server/options"
    30  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    31  	"k8s.io/client-go/dynamic"
    32  	"k8s.io/client-go/dynamic/dynamicinformer"
    33  	clientset "k8s.io/client-go/kubernetes"
    34  	restclient "k8s.io/client-go/rest"
    35  	"k8s.io/client-go/tools/clientcmd"
    36  	"k8s.io/client-go/tools/events"
    37  	"k8s.io/client-go/tools/leaderelection"
    38  	"k8s.io/client-go/tools/leaderelection/resourcelock"
    39  	"k8s.io/client-go/tools/record"
    40  	cliflag "k8s.io/component-base/cli/flag"
    41  	componentbaseconfig "k8s.io/component-base/config"
    42  	"k8s.io/component-base/config/options"
    43  	"k8s.io/component-base/logs"
    44  	logsapi "k8s.io/component-base/logs/api/v1"
    45  	"k8s.io/component-base/metrics"
    46  	"k8s.io/klog/v2"
    47  	schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
    48  	"k8s.io/kubernetes/pkg/scheduler"
    49  	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
    50  	"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
    51  	netutils "k8s.io/utils/net"
    52  )
    53  
    54  // Options has all the params needed to run a Scheduler
    55  type Options struct {
    56  	// The default values.
    57  	ComponentConfig *kubeschedulerconfig.KubeSchedulerConfiguration
    58  
    59  	SecureServing  *apiserveroptions.SecureServingOptionsWithLoopback
    60  	Authentication *apiserveroptions.DelegatingAuthenticationOptions
    61  	Authorization  *apiserveroptions.DelegatingAuthorizationOptions
    62  	Metrics        *metrics.Options
    63  	Logs           *logs.Options
    64  	Deprecated     *DeprecatedOptions
    65  	LeaderElection *componentbaseconfig.LeaderElectionConfiguration
    66  
    67  	// ConfigFile is the location of the scheduler server's configuration file.
    68  	ConfigFile string
    69  
    70  	// WriteConfigTo is the path where the default configuration will be written.
    71  	WriteConfigTo string
    72  
    73  	Master string
    74  
    75  	// Flags hold the parsed CLI flags.
    76  	Flags *cliflag.NamedFlagSets
    77  }
    78  
    79  // NewOptions returns default scheduler app options.
    80  func NewOptions() *Options {
    81  	o := &Options{
    82  		SecureServing:  apiserveroptions.NewSecureServingOptions().WithLoopback(),
    83  		Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(),
    84  		Authorization:  apiserveroptions.NewDelegatingAuthorizationOptions(),
    85  		Deprecated: &DeprecatedOptions{
    86  			PodMaxInUnschedulablePodsDuration: 5 * time.Minute,
    87  		},
    88  		LeaderElection: &componentbaseconfig.LeaderElectionConfiguration{
    89  			LeaderElect:       true,
    90  			LeaseDuration:     metav1.Duration{Duration: 15 * time.Second},
    91  			RenewDeadline:     metav1.Duration{Duration: 10 * time.Second},
    92  			RetryPeriod:       metav1.Duration{Duration: 2 * time.Second},
    93  			ResourceLock:      "leases",
    94  			ResourceName:      "kube-scheduler",
    95  			ResourceNamespace: "kube-system",
    96  		},
    97  		Metrics: metrics.NewOptions(),
    98  		Logs:    logs.NewOptions(),
    99  	}
   100  
   101  	o.Authentication.TolerateInClusterLookupFailure = true
   102  	o.Authentication.RemoteKubeConfigFileOptional = true
   103  	o.Authorization.RemoteKubeConfigFileOptional = true
   104  
   105  	// Set the PairName but leave certificate directory blank to generate in-memory by default
   106  	o.SecureServing.ServerCert.CertDirectory = ""
   107  	o.SecureServing.ServerCert.PairName = "kube-scheduler"
   108  	o.SecureServing.BindPort = kubeschedulerconfig.DefaultKubeSchedulerPort
   109  
   110  	o.initFlags()
   111  
   112  	return o
   113  }
   114  
   115  // ApplyDeprecated obtains the deprecated CLI args and set them to `o.ComponentConfig` if specified.
   116  func (o *Options) ApplyDeprecated() {
   117  	if o.Flags == nil {
   118  		return
   119  	}
   120  	// Obtain deprecated CLI args. Set them to cfg if specified in command line.
   121  	deprecated := o.Flags.FlagSet("deprecated")
   122  	if deprecated.Changed("profiling") {
   123  		o.ComponentConfig.EnableProfiling = o.Deprecated.EnableProfiling
   124  	}
   125  	if deprecated.Changed("contention-profiling") {
   126  		o.ComponentConfig.EnableContentionProfiling = o.Deprecated.EnableContentionProfiling
   127  	}
   128  	if deprecated.Changed("kubeconfig") {
   129  		o.ComponentConfig.ClientConnection.Kubeconfig = o.Deprecated.Kubeconfig
   130  	}
   131  	if deprecated.Changed("kube-api-content-type") {
   132  		o.ComponentConfig.ClientConnection.ContentType = o.Deprecated.ContentType
   133  	}
   134  	if deprecated.Changed("kube-api-qps") {
   135  		o.ComponentConfig.ClientConnection.QPS = o.Deprecated.QPS
   136  	}
   137  	if deprecated.Changed("kube-api-burst") {
   138  		o.ComponentConfig.ClientConnection.Burst = o.Deprecated.Burst
   139  	}
   140  }
   141  
   142  // ApplyLeaderElectionTo obtains the CLI args related with leaderelection, and override the values in `cfg`.
   143  // Then the `cfg` object is injected into the `options` object.
   144  func (o *Options) ApplyLeaderElectionTo(cfg *kubeschedulerconfig.KubeSchedulerConfiguration) {
   145  	if o.Flags == nil {
   146  		return
   147  	}
   148  	// Obtain CLI args related with leaderelection. Set them to `cfg` if specified in command line.
   149  	leaderelection := o.Flags.FlagSet("leader election")
   150  	if leaderelection.Changed("leader-elect") {
   151  		cfg.LeaderElection.LeaderElect = o.LeaderElection.LeaderElect
   152  	}
   153  	if leaderelection.Changed("leader-elect-lease-duration") {
   154  		cfg.LeaderElection.LeaseDuration = o.LeaderElection.LeaseDuration
   155  	}
   156  	if leaderelection.Changed("leader-elect-renew-deadline") {
   157  		cfg.LeaderElection.RenewDeadline = o.LeaderElection.RenewDeadline
   158  	}
   159  	if leaderelection.Changed("leader-elect-retry-period") {
   160  		cfg.LeaderElection.RetryPeriod = o.LeaderElection.RetryPeriod
   161  	}
   162  	if leaderelection.Changed("leader-elect-resource-lock") {
   163  		cfg.LeaderElection.ResourceLock = o.LeaderElection.ResourceLock
   164  	}
   165  	if leaderelection.Changed("leader-elect-resource-name") {
   166  		cfg.LeaderElection.ResourceName = o.LeaderElection.ResourceName
   167  	}
   168  	if leaderelection.Changed("leader-elect-resource-namespace") {
   169  		cfg.LeaderElection.ResourceNamespace = o.LeaderElection.ResourceNamespace
   170  	}
   171  
   172  	o.ComponentConfig = cfg
   173  }
   174  
   175  // initFlags initializes flags by section name.
   176  func (o *Options) initFlags() {
   177  	if o.Flags != nil {
   178  		return
   179  	}
   180  
   181  	nfs := cliflag.NamedFlagSets{}
   182  	fs := nfs.FlagSet("misc")
   183  	fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.")
   184  	fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the configuration values to this file and exit.")
   185  	fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
   186  
   187  	o.SecureServing.AddFlags(nfs.FlagSet("secure serving"))
   188  	o.Authentication.AddFlags(nfs.FlagSet("authentication"))
   189  	o.Authorization.AddFlags(nfs.FlagSet("authorization"))
   190  	o.Deprecated.AddFlags(nfs.FlagSet("deprecated"))
   191  	options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election"))
   192  	utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
   193  	o.Metrics.AddFlags(nfs.FlagSet("metrics"))
   194  	logsapi.AddFlags(o.Logs, nfs.FlagSet("logs"))
   195  
   196  	o.Flags = &nfs
   197  }
   198  
   199  // ApplyTo applies the scheduler options to the given scheduler app configuration.
   200  func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) error {
   201  	if len(o.ConfigFile) == 0 {
   202  		// If the --config arg is not specified, honor the deprecated as well as leader election CLI args.
   203  		o.ApplyDeprecated()
   204  		o.ApplyLeaderElectionTo(o.ComponentConfig)
   205  		c.ComponentConfig = *o.ComponentConfig
   206  	} else {
   207  		cfg, err := LoadConfigFromFile(logger, o.ConfigFile)
   208  		if err != nil {
   209  			return err
   210  		}
   211  		// If the --config arg is specified, honor the leader election CLI args only.
   212  		o.ApplyLeaderElectionTo(cfg)
   213  
   214  		if err := validation.ValidateKubeSchedulerConfiguration(cfg); err != nil {
   215  			return err
   216  		}
   217  
   218  		c.ComponentConfig = *cfg
   219  	}
   220  
   221  	// Build kubeconfig first to so that if it fails, it doesn't cause leaking
   222  	// goroutines (started from initializing secure serving - which underneath
   223  	// creates a queue which in its constructor starts a goroutine).
   224  	kubeConfig, err := createKubeConfig(c.ComponentConfig.ClientConnection, o.Master)
   225  	if err != nil {
   226  		return err
   227  	}
   228  	c.KubeConfig = kubeConfig
   229  
   230  	if err := o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil {
   231  		return err
   232  	}
   233  	if o.SecureServing != nil && (o.SecureServing.BindPort != 0 || o.SecureServing.Listener != nil) {
   234  		if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil {
   235  			return err
   236  		}
   237  		if err := o.Authorization.ApplyTo(&c.Authorization); err != nil {
   238  			return err
   239  		}
   240  	}
   241  	o.Metrics.Apply()
   242  
   243  	// Apply value independently instead of using ApplyDeprecated() because it can't be configured via ComponentConfig.
   244  	if o.Deprecated != nil {
   245  		c.PodMaxInUnschedulablePodsDuration = o.Deprecated.PodMaxInUnschedulablePodsDuration
   246  	}
   247  
   248  	return nil
   249  }
   250  
   251  // Validate validates all the required options.
   252  func (o *Options) Validate() []error {
   253  	var errs []error
   254  
   255  	if err := validation.ValidateKubeSchedulerConfiguration(o.ComponentConfig); err != nil {
   256  		errs = append(errs, err.Errors()...)
   257  	}
   258  	errs = append(errs, o.SecureServing.Validate()...)
   259  	errs = append(errs, o.Authentication.Validate()...)
   260  	errs = append(errs, o.Authorization.Validate()...)
   261  	errs = append(errs, o.Metrics.Validate()...)
   262  
   263  	return errs
   264  }
   265  
   266  // Config return a scheduler config object
   267  func (o *Options) Config(ctx context.Context) (*schedulerappconfig.Config, error) {
   268  	logger := klog.FromContext(ctx)
   269  	if o.SecureServing != nil {
   270  		if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{netutils.ParseIPSloppy("127.0.0.1")}); err != nil {
   271  			return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
   272  		}
   273  	}
   274  
   275  	c := &schedulerappconfig.Config{}
   276  	if err := o.ApplyTo(logger, c); err != nil {
   277  		return nil, err
   278  	}
   279  
   280  	// Prepare kube clients.
   281  	client, eventClient, err := createClients(c.KubeConfig)
   282  	if err != nil {
   283  		return nil, err
   284  	}
   285  
   286  	c.EventBroadcaster = events.NewEventBroadcasterAdapterWithContext(ctx, eventClient)
   287  
   288  	// Set up leader election if enabled.
   289  	var leaderElectionConfig *leaderelection.LeaderElectionConfig
   290  	if c.ComponentConfig.LeaderElection.LeaderElect {
   291  		// Use the scheduler name in the first profile to record leader election.
   292  		schedulerName := corev1.DefaultSchedulerName
   293  		if len(c.ComponentConfig.Profiles) != 0 {
   294  			schedulerName = c.ComponentConfig.Profiles[0].SchedulerName
   295  		}
   296  		coreRecorder := c.EventBroadcaster.DeprecatedNewLegacyRecorder(schedulerName)
   297  		leaderElectionConfig, err = makeLeaderElectionConfig(c.ComponentConfig.LeaderElection, c.KubeConfig, coreRecorder)
   298  		if err != nil {
   299  			return nil, err
   300  		}
   301  	}
   302  
   303  	c.Client = client
   304  	c.InformerFactory = scheduler.NewInformerFactory(client, 0)
   305  	dynClient := dynamic.NewForConfigOrDie(c.KubeConfig)
   306  	c.DynInformerFactory = dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynClient, 0, corev1.NamespaceAll, nil)
   307  	c.LeaderElection = leaderElectionConfig
   308  
   309  	return c, nil
   310  }
   311  
   312  // makeLeaderElectionConfig builds a leader election configuration. It will
   313  // create a new resource lock associated with the configuration.
   314  func makeLeaderElectionConfig(config componentbaseconfig.LeaderElectionConfiguration, kubeConfig *restclient.Config, recorder record.EventRecorder) (*leaderelection.LeaderElectionConfig, error) {
   315  	hostname, err := os.Hostname()
   316  	if err != nil {
   317  		return nil, fmt.Errorf("unable to get hostname: %v", err)
   318  	}
   319  	// add a uniquifier so that two processes on the same host don't accidentally both become active
   320  	id := hostname + "_" + string(uuid.NewUUID())
   321  
   322  	rl, err := resourcelock.NewFromKubeconfig(config.ResourceLock,
   323  		config.ResourceNamespace,
   324  		config.ResourceName,
   325  		resourcelock.ResourceLockConfig{
   326  			Identity:      id,
   327  			EventRecorder: recorder,
   328  		},
   329  		kubeConfig,
   330  		config.RenewDeadline.Duration)
   331  	if err != nil {
   332  		return nil, fmt.Errorf("couldn't create resource lock: %v", err)
   333  	}
   334  
   335  	return &leaderelection.LeaderElectionConfig{
   336  		Lock:            rl,
   337  		LeaseDuration:   config.LeaseDuration.Duration,
   338  		RenewDeadline:   config.RenewDeadline.Duration,
   339  		RetryPeriod:     config.RetryPeriod.Duration,
   340  		WatchDog:        leaderelection.NewLeaderHealthzAdaptor(time.Second * 20),
   341  		Name:            "kube-scheduler",
   342  		ReleaseOnCancel: true,
   343  	}, nil
   344  }
   345  
   346  // createKubeConfig creates a kubeConfig from the given config and masterOverride.
   347  // TODO remove masterOverride when CLI flags are removed.
   348  func createKubeConfig(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string) (*restclient.Config, error) {
   349  	kubeConfig, err := clientcmd.BuildConfigFromFlags(masterOverride, config.Kubeconfig)
   350  	if err != nil {
   351  		return nil, err
   352  	}
   353  
   354  	kubeConfig.DisableCompression = true
   355  	kubeConfig.AcceptContentTypes = config.AcceptContentTypes
   356  	kubeConfig.ContentType = config.ContentType
   357  	kubeConfig.QPS = config.QPS
   358  	kubeConfig.Burst = int(config.Burst)
   359  
   360  	return kubeConfig, nil
   361  }
   362  
   363  // createClients creates a kube client and an event client from the given kubeConfig
   364  func createClients(kubeConfig *restclient.Config) (clientset.Interface, clientset.Interface, error) {
   365  	client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeConfig, "scheduler"))
   366  	if err != nil {
   367  		return nil, nil, err
   368  	}
   369  
   370  	eventClient, err := clientset.NewForConfig(kubeConfig)
   371  	if err != nil {
   372  		return nil, nil, err
   373  	}
   374  
   375  	return client, eventClient, nil
   376  }
   377
View as plain text