options.go

Documentation: k8s.io/kubernetes/cmd/kubelet/app/options

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package options contains all of the primary arguments for a kubelet.
    18  package options
    19  
    20  import (
    21  	"fmt"
    22  	_ "net/http/pprof" // Enable pprof HTTP handlers.
    23  	"path/filepath"
    24  	"strings"
    25  
    26  	"github.com/spf13/pflag"
    27  
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/apimachinery/pkg/util/sets"
    30  	"k8s.io/apimachinery/pkg/util/validation"
    31  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    32  	cliflag "k8s.io/component-base/cli/flag"
    33  	logsapi "k8s.io/component-base/logs/api/v1"
    34  	"k8s.io/kubelet/config/v1beta1"
    35  	kubeletapis "k8s.io/kubelet/pkg/apis"
    36  	"k8s.io/kubernetes/pkg/cluster/ports"
    37  	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
    38  	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme"
    39  	kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/config/validation"
    40  	"k8s.io/kubernetes/pkg/kubelet/config"
    41  	utilflag "k8s.io/kubernetes/pkg/util/flag"
    42  )
    43  
    44  const defaultRootDir = "/var/lib/kubelet"
    45  
    46  // KubeletFlags contains configuration flags for the Kubelet.
    47  // A configuration field should go in KubeletFlags instead of KubeletConfiguration if any of these are true:
    48  //   - its value will never, or cannot safely be changed during the lifetime of a node, or
    49  //   - its value cannot be safely shared between nodes at the same time (e.g. a hostname);
    50  //     KubeletConfiguration is intended to be shared between nodes.
    51  //
    52  // In general, please try to avoid adding flags or configuration fields,
    53  // we already have a confusingly large amount of them.
    54  type KubeletFlags struct {
    55  	KubeConfig          string
    56  	BootstrapKubeconfig string
    57  
    58  	// HostnameOverride is the hostname used to identify the kubelet instead
    59  	// of the actual hostname.
    60  	HostnameOverride string
    61  	// NodeIP is IP address of the node.
    62  	// If set, kubelet will use this IP address for the node.
    63  	NodeIP string
    64  
    65  	// Container-runtime-specific options.
    66  	config.ContainerRuntimeOptions
    67  
    68  	// certDirectory is the directory where the TLS certs are located.
    69  	// If tlsCertFile and tlsPrivateKeyFile are provided, this flag will be ignored.
    70  	CertDirectory string
    71  
    72  	// cloudProvider is the provider for cloud services.
    73  	// +optional
    74  	CloudProvider string
    75  
    76  	// cloudConfigFile is the path to the cloud provider configuration file.
    77  	// +optional
    78  	CloudConfigFile string
    79  
    80  	// rootDirectory is the directory path to place kubelet files (volume
    81  	// mounts,etc).
    82  	RootDirectory string
    83  
    84  	// The Kubelet will load its initial configuration from this file.
    85  	// The path may be absolute or relative; relative paths are under the Kubelet's current working directory.
    86  	// Omit this flag to use the combination of built-in default configuration values and flags.
    87  	KubeletConfigFile string
    88  
    89  	// kubeletDropinConfigDirectory is a path to a directory to specify dropins allows the user to optionally specify
    90  	// additional configs to overwrite what is provided by default and in the KubeletConfigFile flag
    91  	KubeletDropinConfigDirectory string
    92  
    93  	// WindowsService should be set to true if kubelet is running as a service on Windows.
    94  	// Its corresponding flag only gets registered in Windows builds.
    95  	WindowsService bool
    96  
    97  	// WindowsPriorityClass sets the priority class associated with the Kubelet process
    98  	// Its corresponding flag only gets registered in Windows builds
    99  	// The default priority class associated with any process in Windows is NORMAL_PRIORITY_CLASS. Keeping it as is
   100  	// to maintain backwards compatibility.
   101  	// Source: https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities
   102  	WindowsPriorityClass string
   103  
   104  	// experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path
   105  	ExperimentalMounterPath string
   106  	// This flag, if set, will avoid including `EvictionHard` limits while computing Node Allocatable.
   107  	// Refer to [Node Allocatable](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable) doc for more information.
   108  	ExperimentalNodeAllocatableIgnoreEvictionThreshold bool
   109  	// Node Labels are the node labels to add when registering the node in the cluster
   110  	NodeLabels map[string]string
   111  	// lockFilePath is the path that kubelet will use to as a lock file.
   112  	// It uses this file as a lock to synchronize with other kubelet processes
   113  	// that may be running.
   114  	LockFilePath string
   115  	// ExitOnLockContention is a flag that signifies to the kubelet that it is running
   116  	// in "bootstrap" mode. This requires that 'LockFilePath' has been set.
   117  	// This will cause the kubelet to listen to inotify events on the lock file,
   118  	// releasing it and exiting when another process tries to open that file.
   119  	ExitOnLockContention bool
   120  	// DEPRECATED FLAGS
   121  	// minimumGCAge is the minimum age for a finished container before it is
   122  	// garbage collected.
   123  	MinimumGCAge metav1.Duration
   124  	// maxPerPodContainerCount is the maximum number of old instances to
   125  	// retain per container. Each container takes up some disk space.
   126  	MaxPerPodContainerCount int32
   127  	// maxContainerCount is the maximum number of old instances of containers
   128  	// to retain globally. Each container takes up some disk space.
   129  	MaxContainerCount int32
   130  	// registerSchedulable tells the kubelet to register the node as
   131  	// schedulable. Won't have any effect if register-node is false.
   132  	// DEPRECATED: use registerWithTaints instead
   133  	RegisterSchedulable bool
   134  	// This flag, if set, instructs the kubelet to keep volumes from terminated pods mounted to the node.
   135  	// This can be useful for debugging volume related issues.
   136  	KeepTerminatedPodVolumes bool
   137  	// SeccompDefault enables the use of `RuntimeDefault` as the default seccomp profile for all workloads on the node.
   138  	SeccompDefault bool
   139  }
   140  
   141  // NewKubeletFlags will create a new KubeletFlags with default values
   142  func NewKubeletFlags() *KubeletFlags {
   143  	return &KubeletFlags{
   144  		ContainerRuntimeOptions: *NewContainerRuntimeOptions(),
   145  		CertDirectory:           "/var/lib/kubelet/pki",
   146  		RootDirectory:           filepath.Clean(defaultRootDir),
   147  		MaxContainerCount:       -1,
   148  		MaxPerPodContainerCount: 1,
   149  		MinimumGCAge:            metav1.Duration{Duration: 0},
   150  		RegisterSchedulable:     true,
   151  		NodeLabels:              make(map[string]string),
   152  	}
   153  }
   154  
   155  // ValidateKubeletFlags validates Kubelet's configuration flags and returns an error if they are invalid.
   156  func ValidateKubeletFlags(f *KubeletFlags) error {
   157  	unknownLabels := sets.NewString()
   158  	invalidLabelErrs := make(map[string][]string)
   159  	for k, v := range f.NodeLabels {
   160  		if isKubernetesLabel(k) && !kubeletapis.IsKubeletLabel(k) {
   161  			unknownLabels.Insert(k)
   162  		}
   163  
   164  		if errs := validation.IsQualifiedName(k); len(errs) > 0 {
   165  			invalidLabelErrs[k] = append(invalidLabelErrs[k], errs...)
   166  		}
   167  		if errs := validation.IsValidLabelValue(v); len(errs) > 0 {
   168  			invalidLabelErrs[v] = append(invalidLabelErrs[v], errs...)
   169  		}
   170  	}
   171  	if len(unknownLabels) > 0 {
   172  		return fmt.Errorf("unknown 'kubernetes.io' or 'k8s.io' labels specified with --node-labels: %v\n--node-labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", unknownLabels.List(), strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", "))
   173  	}
   174  	if len(invalidLabelErrs) > 0 {
   175  		labelErrs := []string{}
   176  		for k, v := range invalidLabelErrs {
   177  			labelErrs = append(labelErrs, fmt.Sprintf("'%s' - %s", k, strings.Join(v, ", ")))
   178  		}
   179  		return fmt.Errorf("invalid node labels: %s", strings.Join(labelErrs, "; "))
   180  	}
   181  	return nil
   182  }
   183  
   184  func isKubernetesLabel(key string) bool {
   185  	namespace := getLabelNamespace(key)
   186  	if namespace == "kubernetes.io" || strings.HasSuffix(namespace, ".kubernetes.io") {
   187  		return true
   188  	}
   189  	if namespace == "k8s.io" || strings.HasSuffix(namespace, ".k8s.io") {
   190  		return true
   191  	}
   192  	return false
   193  }
   194  
   195  func getLabelNamespace(key string) string {
   196  	if parts := strings.SplitN(key, "/", 2); len(parts) == 2 {
   197  		return parts[0]
   198  	}
   199  	return ""
   200  }
   201  
   202  // NewKubeletConfiguration will create a new KubeletConfiguration with default values
   203  func NewKubeletConfiguration() (*kubeletconfig.KubeletConfiguration, error) {
   204  	scheme, _, err := kubeletscheme.NewSchemeAndCodecs()
   205  	if err != nil {
   206  		return nil, err
   207  	}
   208  	versioned := &v1beta1.KubeletConfiguration{}
   209  	scheme.Default(versioned)
   210  	config := &kubeletconfig.KubeletConfiguration{}
   211  	if err := scheme.Convert(versioned, config, nil); err != nil {
   212  		return nil, err
   213  	}
   214  	applyLegacyDefaults(config)
   215  	return config, nil
   216  }
   217  
   218  // applyLegacyDefaults applies legacy default values to the KubeletConfiguration in order to
   219  // preserve the command line API. This is used to construct the baseline default KubeletConfiguration
   220  // before the first round of flag parsing.
   221  func applyLegacyDefaults(kc *kubeletconfig.KubeletConfiguration) {
   222  	// --anonymous-auth
   223  	kc.Authentication.Anonymous.Enabled = true
   224  	// --authentication-token-webhook
   225  	kc.Authentication.Webhook.Enabled = false
   226  	// --authorization-mode
   227  	kc.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow
   228  	// --read-only-port
   229  	kc.ReadOnlyPort = ports.KubeletReadOnlyPort
   230  }
   231  
   232  // KubeletServer encapsulates all of the parameters necessary for starting up
   233  // a kubelet. These can either be set via command line or directly.
   234  type KubeletServer struct {
   235  	KubeletFlags
   236  	kubeletconfig.KubeletConfiguration
   237  }
   238  
   239  // NewKubeletServer will create a new KubeletServer with default values.
   240  func NewKubeletServer() (*KubeletServer, error) {
   241  	config, err := NewKubeletConfiguration()
   242  	if err != nil {
   243  		return nil, err
   244  	}
   245  	return &KubeletServer{
   246  		KubeletFlags:         *NewKubeletFlags(),
   247  		KubeletConfiguration: *config,
   248  	}, nil
   249  }
   250  
   251  // ValidateKubeletServer validates configuration of KubeletServer and returns an error if the input configuration is invalid.
   252  func ValidateKubeletServer(s *KubeletServer) error {
   253  	// please add any KubeletConfiguration validation to the kubeletconfigvalidation.ValidateKubeletConfiguration function
   254  	if err := kubeletconfigvalidation.ValidateKubeletConfiguration(&s.KubeletConfiguration, utilfeature.DefaultFeatureGate); err != nil {
   255  		return err
   256  	}
   257  	if err := ValidateKubeletFlags(&s.KubeletFlags); err != nil {
   258  		return err
   259  	}
   260  	return nil
   261  }
   262  
   263  // AddFlags adds flags for a specific KubeletServer to the specified FlagSet
   264  func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
   265  	s.KubeletFlags.AddFlags(fs)
   266  	AddKubeletConfigFlags(fs, &s.KubeletConfiguration)
   267  }
   268  
   269  // AddFlags adds flags for a specific KubeletFlags to the specified FlagSet
   270  func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
   271  	fs := pflag.NewFlagSet("", pflag.ExitOnError)
   272  	defer func() {
   273  		// Unhide deprecated flags. We want deprecated flags to show in Kubelet help.
   274  		// We have some hidden flags, but we might as well unhide these when they are deprecated,
   275  		// as silently deprecating and removing (even hidden) things is unkind to people who use them.
   276  		fs.VisitAll(func(f *pflag.Flag) {
   277  			if len(f.Deprecated) > 0 {
   278  				f.Hidden = false
   279  			}
   280  		})
   281  		mainfs.AddFlagSet(fs)
   282  	}()
   283  
   284  	f.ContainerRuntimeOptions.AddFlags(fs)
   285  	f.addOSFlags(fs)
   286  
   287  	fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.")
   288  	fs.StringVar(&f.KubeletDropinConfigDirectory, "config-dir", "", "Path to a directory to specify drop-ins, allows the user to optionally specify additional configs to overwrite what is provided by default and in the KubeletConfigFile flag. [default='']")
   289  	fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.")
   290  
   291  	fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+
   292  		"If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used to request a client certificate from the API server. "+
   293  		"On success, a kubeconfig file referencing the generated client certificate and key is written to the path specified by --kubeconfig. "+
   294  		"The client certificate and key file will be stored in the directory pointed by --cert-dir.")
   295  
   296  	fs.StringVar(&f.HostnameOverride, "hostname-override", f.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname. If --cloud-provider is set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).")
   297  
   298  	fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address (or comma-separated dual-stack IP addresses) of the node. If unset, kubelet will use the node's default IPv4 address, if any, or its default IPv6 address if it has no IPv4 addresses. You can pass '::' to make it prefer the default IPv6 address rather than the default IPv4 address.")
   299  
   300  	fs.StringVar(&f.CertDirectory, "cert-dir", f.CertDirectory, "The directory where the TLS certs are located. "+
   301  		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
   302  
   303  	fs.StringVar(&f.CloudProvider, "cloud-provider", f.CloudProvider, "The provider for cloud services. Set to empty string for running with no cloud provider. Set to 'external' for running with an external cloud provider. If set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).")
   304  
   305  	fs.StringVar(&f.RootDirectory, "root-dir", f.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).")
   306  	fs.BoolVar(&f.SeccompDefault, "seccomp-default", f.SeccompDefault, "Enable the use of `RuntimeDefault` as the default seccomp profile for all workloads.")
   307  
   308  	bindableNodeLabels := cliflag.ConfigurationMap(f.NodeLabels)
   309  	fs.Var(&bindableNodeLabels, "node-labels", fmt.Sprintf("Labels to add when registering the node in the cluster.  Labels must be key=value pairs separated by ','. Labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", ")))
   310  
   311  	// EXPERIMENTAL FLAGS
   312  	fs.StringVar(&f.LockFilePath, "lock-file", f.LockFilePath, "<Warning: Alpha feature> The path to file for kubelet to use as a lock file.")
   313  	fs.BoolVar(&f.ExitOnLockContention, "exit-on-lock-contention", f.ExitOnLockContention, "Whether kubelet should exit upon lock-file contention.")
   314  
   315  	// DEPRECATED FLAGS
   316  	fs.DurationVar(&f.MinimumGCAge.Duration, "minimum-container-ttl-duration", f.MinimumGCAge.Duration, "Minimum age for a finished container before it is garbage collected.  Examples: '300ms', '10s' or '2h45m'")
   317  	fs.MarkDeprecated("minimum-container-ttl-duration", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
   318  	fs.Int32Var(&f.MaxPerPodContainerCount, "maximum-dead-containers-per-container", f.MaxPerPodContainerCount, "Maximum number of old instances to retain per container.  Each container takes up some disk space.")
   319  	fs.MarkDeprecated("maximum-dead-containers-per-container", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
   320  	fs.Int32Var(&f.MaxContainerCount, "maximum-dead-containers", f.MaxContainerCount, "Maximum number of old instances of containers to retain globally.  Each container takes up some disk space. To disable, set to a negative number.")
   321  	fs.MarkDeprecated("maximum-dead-containers", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
   322  	fs.BoolVar(&f.RegisterSchedulable, "register-schedulable", f.RegisterSchedulable, "Register the node as schedulable. Won't have any effect if register-node is false.")
   323  	fs.MarkDeprecated("register-schedulable", "will be removed in a future version")
   324  	fs.BoolVar(&f.KeepTerminatedPodVolumes, "keep-terminated-pod-volumes", f.KeepTerminatedPodVolumes, "Keep terminated pod volumes mounted to the node after the pod terminates.  Can be useful for debugging volume related issues.")
   325  	fs.MarkDeprecated("keep-terminated-pod-volumes", "will be removed in a future version")
   326  	fs.StringVar(&f.ExperimentalMounterPath, "experimental-mounter-path", f.ExperimentalMounterPath, "[Experimental] Path of mounter binary. Leave empty to use the default mount.")
   327  	fs.MarkDeprecated("experimental-mounter-path", "will be removed in 1.25 or later. in favor of using CSI.")
   328  	fs.StringVar(&f.CloudConfigFile, "cloud-config", f.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
   329  	fs.MarkDeprecated("cloud-config", "will be removed in 1.25 or later, in favor of removing cloud provider code from Kubelet.")
   330  	fs.BoolVar(&f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "experimental-allocatable-ignore-eviction", f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "When set to 'true', Hard Eviction Thresholds will be ignored while calculating Node Allocatable. See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ for more details. [default=false]")
   331  	fs.MarkDeprecated("experimental-allocatable-ignore-eviction", "will be removed in 1.25 or later.")
   332  }
   333  
   334  // AddKubeletConfigFlags adds flags for a specific kubeletconfig.KubeletConfiguration to the specified FlagSet
   335  func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfiguration) {
   336  	fs := pflag.NewFlagSet("", pflag.ExitOnError)
   337  	defer func() {
   338  		// All KubeletConfiguration flags are now deprecated, and any new flags that point to
   339  		// KubeletConfiguration fields are deprecated-on-creation. When removing flags at the end
   340  		// of their deprecation period, be careful to check that they have *actually* been deprecated
   341  		// members of the KubeletConfiguration for the entire deprecation period:
   342  		// e.g. if a flag was added after this deprecation function, it may not be at the end
   343  		// of its lifetime yet, even if the rest are.
   344  		deprecated := "This parameter should be set via the config file specified by the Kubelet's --config flag. See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ for more information."
   345  		// Some flags are permanently (?) meant to be available. In
   346  		// Kubernetes 1.23, the following options were added to
   347  		// LoggingConfiguration (long-term goal, more complete
   348  		// configuration file) but deprecating the flags seemed
   349  		// premature.
   350  		notDeprecated := map[string]bool{
   351  			"v":                   true,
   352  			"vmodule":             true,
   353  			"log-flush-frequency": true,
   354  			"provider-id":         true,
   355  		}
   356  		fs.VisitAll(func(f *pflag.Flag) {
   357  			if notDeprecated[f.Name] {
   358  				return
   359  			}
   360  			f.Deprecated = deprecated
   361  		})
   362  		mainfs.AddFlagSet(fs)
   363  	}()
   364  
   365  	fs.BoolVar(&c.EnableServer, "enable-server", c.EnableServer, "Enable the Kubelet's server")
   366  
   367  	fs.BoolVar(&c.FailSwapOn, "fail-swap-on", c.FailSwapOn, "Makes the Kubelet fail to start if swap is enabled on the node. ")
   368  	fs.StringVar(&c.StaticPodPath, "pod-manifest-path", c.StaticPodPath, "Path to the directory containing static pod files to run, or the path to a single static pod file. Files starting with dots will be ignored.")
   369  	fs.DurationVar(&c.SyncFrequency.Duration, "sync-frequency", c.SyncFrequency.Duration, "Max period between synchronizing running containers and config")
   370  	fs.DurationVar(&c.FileCheckFrequency.Duration, "file-check-frequency", c.FileCheckFrequency.Duration, "Duration between checking config files for new data")
   371  	fs.DurationVar(&c.HTTPCheckFrequency.Duration, "http-check-frequency", c.HTTPCheckFrequency.Duration, "Duration between checking http for new data")
   372  	fs.StringVar(&c.StaticPodURL, "manifest-url", c.StaticPodURL, "URL for accessing additional Pod specifications to run")
   373  	fs.Var(cliflag.NewColonSeparatedMultimapStringString(&c.StaticPodURLHeader), "manifest-url-header", "Comma-separated list of HTTP headers to use when accessing the url provided to --manifest-url. Multiple headers with the same name will be added in the same order provided. This flag can be repeatedly invoked. For example: --manifest-url-header 'a:hello,b:again,c:world' --manifest-url-header 'b:beautiful'")
   374  	fs.Var(&utilflag.IPVar{Val: &c.Address}, "address", "The IP address for the Kubelet to serve on (set to '0.0.0.0' or '::' for listening on all interfaces and IP address families)")
   375  	fs.Int32Var(&c.Port, "port", c.Port, "The port for the Kubelet to serve on.")
   376  	fs.Int32Var(&c.ReadOnlyPort, "read-only-port", c.ReadOnlyPort, "The read-only port for the Kubelet to serve on with no authentication/authorization (set to 0 to disable)")
   377  
   378  	// runtime flags
   379  	fs.StringVar(&c.ContainerRuntimeEndpoint, "container-runtime-endpoint", c.ContainerRuntimeEndpoint, "The endpoint of container runtime service. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'")
   380  	fs.StringVar(&c.ImageServiceEndpoint, "image-service-endpoint", c.ImageServiceEndpoint, "The endpoint of container image service. If not specified, it will be the same with --container-runtime-endpoint by default. Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'")
   381  
   382  	// Authentication
   383  	fs.BoolVar(&c.Authentication.Anonymous.Enabled, "anonymous-auth", c.Authentication.Anonymous.Enabled, ""+
   384  		"Enables anonymous requests to the Kubelet server. Requests that are not rejected by another "+
   385  		"authentication method are treated as anonymous requests. Anonymous requests have a username "+
   386  		"of system:anonymous, and a group name of system:unauthenticated.")
   387  	fs.BoolVar(&c.Authentication.Webhook.Enabled, "authentication-token-webhook", c.Authentication.Webhook.Enabled, ""+
   388  		"Use the TokenReview API to determine authentication for bearer tokens.")
   389  	fs.DurationVar(&c.Authentication.Webhook.CacheTTL.Duration, "authentication-token-webhook-cache-ttl", c.Authentication.Webhook.CacheTTL.Duration, ""+
   390  		"The duration to cache responses from the webhook token authenticator.")
   391  	fs.StringVar(&c.Authentication.X509.ClientCAFile, "client-ca-file", c.Authentication.X509.ClientCAFile, ""+
   392  		"If set, any request presenting a client certificate signed by one of the authorities in the client-ca-file "+
   393  		"is authenticated with an identity corresponding to the CommonName of the client certificate.")
   394  
   395  	// Authorization
   396  	fs.StringVar((*string)(&c.Authorization.Mode), "authorization-mode", string(c.Authorization.Mode), ""+
   397  		"Authorization mode for Kubelet server. Valid options are AlwaysAllow or Webhook. "+
   398  		"Webhook mode uses the SubjectAccessReview API to determine authorization.")
   399  	fs.DurationVar(&c.Authorization.Webhook.CacheAuthorizedTTL.Duration, "authorization-webhook-cache-authorized-ttl", c.Authorization.Webhook.CacheAuthorizedTTL.Duration, ""+
   400  		"The duration to cache 'authorized' responses from the webhook authorizer.")
   401  	fs.DurationVar(&c.Authorization.Webhook.CacheUnauthorizedTTL.Duration, "authorization-webhook-cache-unauthorized-ttl", c.Authorization.Webhook.CacheUnauthorizedTTL.Duration, ""+
   402  		"The duration to cache 'unauthorized' responses from the webhook authorizer.")
   403  
   404  	fs.StringVar(&c.TLSCertFile, "tls-cert-file", c.TLSCertFile, ""+
   405  		"File containing x509 Certificate used for serving HTTPS (with intermediate certs, if any, concatenated after server cert). "+
   406  		"If --tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate and key "+
   407  		"are generated for the public address and saved to the directory passed to --cert-dir.")
   408  	fs.StringVar(&c.TLSPrivateKeyFile, "tls-private-key-file", c.TLSPrivateKeyFile, "File containing x509 private key matching --tls-cert-file.")
   409  	fs.BoolVar(&c.ServerTLSBootstrap, "rotate-server-certificates", c.ServerTLSBootstrap, "Auto-request and rotate the kubelet serving certificates by requesting new certificates from the kube-apiserver when the certificate expiration approaches. Requires the RotateKubeletServerCertificate feature gate to be enabled, and approval of the submitted CertificateSigningRequest objects.")
   410  
   411  	tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames()
   412  	tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames()
   413  	fs.StringSliceVar(&c.TLSCipherSuites, "tls-cipher-suites", c.TLSCipherSuites,
   414  		"Comma-separated list of cipher suites for the server. "+
   415  			"If omitted, the default Go cipher suites will be used. \n"+
   416  			"Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+
   417  			"Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".")
   418  	tlsPossibleVersions := cliflag.TLSPossibleVersions()
   419  	fs.StringVar(&c.TLSMinVersion, "tls-min-version", c.TLSMinVersion,
   420  		"Minimum TLS version supported. "+
   421  			"Possible values: "+strings.Join(tlsPossibleVersions, ", "))
   422  	fs.BoolVar(&c.RotateCertificates, "rotate-certificates", c.RotateCertificates, "Auto rotate the kubelet client certificates by requesting new certificates from the kube-apiserver when the certificate expiration approaches.")
   423  
   424  	fs.Int32Var(&c.RegistryPullQPS, "registry-qps", c.RegistryPullQPS, "If > 0, limit registry pull QPS to this value.  If 0, unlimited.")
   425  	fs.Int32Var(&c.RegistryBurst, "registry-burst", c.RegistryBurst, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry-qps. Only used if --registry-qps > 0")
   426  	fs.Int32Var(&c.EventRecordQPS, "event-qps", c.EventRecordQPS, "QPS to limit event creations. The number must be >= 0. If 0 will use DefaultQPS: 5.")
   427  	fs.Int32Var(&c.EventBurst, "event-burst", c.EventBurst, "Maximum size of a bursty event records, temporarily allows event records to burst to this number, while still not exceeding event-qps. The number must be >= 0. If 0 will use DefaultBurst: 10.")
   428  
   429  	fs.BoolVar(&c.EnableDebuggingHandlers, "enable-debugging-handlers", c.EnableDebuggingHandlers, "Enables server endpoints for log collection and local running of containers and commands")
   430  	fs.BoolVar(&c.EnableContentionProfiling, "contention-profiling", c.EnableContentionProfiling, "Enable block profiling, if profiling is enabled")
   431  	fs.Int32Var(&c.HealthzPort, "healthz-port", c.HealthzPort, "The port of the localhost healthz endpoint (set to 0 to disable)")
   432  	fs.Var(&utilflag.IPVar{Val: &c.HealthzBindAddress}, "healthz-bind-address", "The IP address for the healthz server to serve on (set to '0.0.0.0' or '::' for listening on all interfaces and IP address families)")
   433  	fs.Int32Var(&c.OOMScoreAdj, "oom-score-adj", c.OOMScoreAdj, "The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]")
   434  	fs.StringVar(&c.ClusterDomain, "cluster-domain", c.ClusterDomain, "Domain for this cluster.  If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
   435  
   436  	fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, "The full path of the directory in which to search for additional third party volume plugins")
   437  	fs.StringSliceVar(&c.ClusterDNS, "cluster-dns", c.ClusterDNS, "Comma-separated list of DNS server IP address.  This value is used for containers DNS server in case of Pods with \"dnsPolicy=ClusterFirst\". Note: all DNS servers appearing in the list MUST serve the same set of records otherwise name resolution within the cluster may not work correctly. There is no guarantee as to which DNS server may be contacted for name resolution.")
   438  	fs.DurationVar(&c.StreamingConnectionIdleTimeout.Duration, "streaming-connection-idle-timeout", c.StreamingConnectionIdleTimeout.Duration, "Maximum time a streaming connection can be idle before the connection is automatically closed. 0 indicates no timeout. Example: '5m'. Note: All connections to the kubelet server have a maximum duration of 4 hours.")
   439  	fs.DurationVar(&c.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", c.NodeStatusUpdateFrequency.Duration, "Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller.")
   440  	fs.DurationVar(&c.ImageMinimumGCAge.Duration, "minimum-image-ttl-duration", c.ImageMinimumGCAge.Duration, "Minimum age for an unused image before it is garbage collected.  Examples: '300ms', '10s' or '2h45m'.")
   441  	fs.Int32Var(&c.ImageGCHighThresholdPercent, "image-gc-high-threshold", c.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run. Values must be within the range [0, 100], To disable image garbage collection, set to 100. ")
   442  	fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. Values must be within the range [0, 100] and must be less than that of --image-gc-high-threshold.")
   443  	fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes.  To disable volume calculations, set to a negative number.")
   444  	fs.Var(cliflag.NewMapStringBool(&c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
   445  		"Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n"))
   446  	fs.StringVar(&c.KubeletCgroups, "kubelet-cgroups", c.KubeletCgroups, "Optional absolute name of cgroups to create and run the Kubelet in.")
   447  	fs.StringVar(&c.SystemCgroups, "system-cgroups", c.SystemCgroups, "Optional absolute name of cgroups in which to place all non-kernel processes that are not already inside a cgroup under '/'. Empty for no container. Rolling back the flag requires a reboot.")
   448  
   449  	fs.StringVar(&c.ProviderID, "provider-id", c.ProviderID, "Unique identifier for identifying the node in a machine database, i.e cloudprovider")
   450  
   451  	fs.BoolVar(&c.CgroupsPerQOS, "cgroups-per-qos", c.CgroupsPerQOS, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.")
   452  	fs.StringVar(&c.CgroupDriver, "cgroup-driver", c.CgroupDriver, "Driver that the kubelet uses to manipulate cgroups on the host.  Possible values: 'cgroupfs', 'systemd'")
   453  	fs.StringVar(&c.CgroupRoot, "cgroup-root", c.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.")
   454  	fs.StringVar(&c.CPUManagerPolicy, "cpu-manager-policy", c.CPUManagerPolicy, "CPU Manager policy to use. Possible values: 'none', 'static'.")
   455  	fs.Var(cliflag.NewMapStringStringNoSplit(&c.CPUManagerPolicyOptions), "cpu-manager-policy-options", "A set of key=value CPU Manager policy options to use, to fine tune their behaviour. If not supplied, keep the default behaviour.")
   456  	fs.DurationVar(&c.CPUManagerReconcilePeriod.Duration, "cpu-manager-reconcile-period", c.CPUManagerReconcilePeriod.Duration, "<Warning: Alpha feature> CPU Manager reconciliation period. Examples: '10s', or '1m'. If not supplied, defaults to 'NodeStatusUpdateFrequency'")
   457  	fs.Var(cliflag.NewMapStringString(&c.QOSReserved), "qos-reserved", "<Warning: Alpha feature> A set of ResourceName=Percentage (e.g. memory=50%) pairs that describe how pod resource requests are reserved at the QoS level. Currently only memory is supported. Requires the QOSReserved feature gate to be enabled.")
   458  	fs.StringVar(&c.TopologyManagerPolicy, "topology-manager-policy", c.TopologyManagerPolicy, "Topology Manager policy to use. Possible values: 'none', 'best-effort', 'restricted', 'single-numa-node'.")
   459  	fs.DurationVar(&c.RuntimeRequestTimeout.Duration, "runtime-request-timeout", c.RuntimeRequestTimeout.Duration, "Timeout of all runtime requests except long running request - pull, logs, exec and attach. When timeout exceeded, kubelet will cancel the request, throw out an error and retry later.")
   460  	fs.StringVar(&c.HairpinMode, "hairpin-mode", c.HairpinMode, "How should the kubelet setup hairpin NAT. This allows endpoints of a Service to loadbalance back to themselves if they should try to access their own Service. Valid values are \"promiscuous-bridge\", \"hairpin-veth\" and \"none\".")
   461  	fs.Int32Var(&c.MaxPods, "max-pods", c.MaxPods, "Number of Pods that can run on this Kubelet.")
   462  
   463  	fs.StringVar(&c.PodCIDR, "pod-cidr", c.PodCIDR, "The CIDR to use for pod IP addresses, only used in standalone mode.  In cluster mode, this is obtained from the master. For IPv6, the maximum number of IP's allocated is 65536")
   464  	fs.Int64Var(&c.PodPidsLimit, "pod-max-pids", c.PodPidsLimit, "Set the maximum number of processes per pod.  If -1, the kubelet defaults to the node allocatable pid capacity.")
   465  
   466  	fs.StringVar(&c.ResolverConfig, "resolv-conf", c.ResolverConfig, "Resolver configuration file used as the basis for the container DNS resolution configuration.")
   467  
   468  	fs.BoolVar(&c.RunOnce, "runonce", c.RunOnce, "If true, exit after spawning pods from static pod files or remote urls. Exclusive with --enable-server")
   469  
   470  	fs.BoolVar(&c.CPUCFSQuota, "cpu-cfs-quota", c.CPUCFSQuota, "Enable CPU CFS quota enforcement for containers that specify CPU limits")
   471  	fs.DurationVar(&c.CPUCFSQuotaPeriod.Duration, "cpu-cfs-quota-period", c.CPUCFSQuotaPeriod.Duration, "Sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default")
   472  	fs.BoolVar(&c.EnableControllerAttachDetach, "enable-controller-attach-detach", c.EnableControllerAttachDetach, "Enables the Attach/Detach controller to manage attachment/detachment of volumes scheduled to this node, and disables kubelet from executing any attach/detach operations")
   473  	fs.BoolVar(&c.MakeIPTablesUtilChains, "make-iptables-util-chains", c.MakeIPTablesUtilChains, "If true, kubelet will ensure iptables utility rules are present on host.")
   474  	fs.Int32Var(&c.IPTablesMasqueradeBit, "iptables-masquerade-bit", c.IPTablesMasqueradeBit, "Has no effect; use kube-proxy parameters to configure the KUBE-MARK-MASQ chain.")
   475  	fs.MarkDeprecated("iptables-masquerade-bit", "This flag has no effect and will be removed in a future version.")
   476  	fs.Int32Var(&c.IPTablesDropBit, "iptables-drop-bit", c.IPTablesDropBit, "Has no effect; kubelet no longer creates a KUBE-MARK-DROP chain")
   477  	fs.MarkDeprecated("iptables-drop-bit", "This flag has no effect and will be removed in a future version.")
   478  	fs.StringVar(&c.ContainerLogMaxSize, "container-log-max-size", c.ContainerLogMaxSize, "<Warning: Beta feature> Set the maximum size (e.g. 10Mi) of container log file before it is rotated.")
   479  	fs.Int32Var(&c.ContainerLogMaxFiles, "container-log-max-files", c.ContainerLogMaxFiles, "<Warning: Beta feature> Set the maximum number of container log files that can be present for a container. The number must be >= 2.")
   480  	fs.StringSliceVar(&c.AllowedUnsafeSysctls, "allowed-unsafe-sysctls", c.AllowedUnsafeSysctls, "Comma-separated whitelist of unsafe sysctls or unsafe sysctl patterns (ending in *). Use these at your own risk.")
   481  
   482  	fs.Int32Var(&c.NodeStatusMaxImages, "node-status-max-images", c.NodeStatusMaxImages, "The maximum number of images to report in Node.Status.Images. If -1 is specified, no cap will be applied.")
   483  	fs.BoolVar(&c.KernelMemcgNotification, "kernel-memcg-notification", c.KernelMemcgNotification, "If enabled, the kubelet will integrate with the kernel memcg notification to determine if memory eviction thresholds are crossed rather than polling.")
   484  	fs.BoolVar(&c.LocalStorageCapacityIsolation, "local-storage-capacity-isolation", c.LocalStorageCapacityIsolation, "If true, local ephemeral storage isolation is enabled. Otherwise, local storage isolation feature will be disabled")
   485  
   486  	// Flags intended for testing, not recommended used in production environments.
   487  	fs.Int64Var(&c.MaxOpenFiles, "max-open-files", c.MaxOpenFiles, "Number of files that can be opened by Kubelet process.")
   488  
   489  	fs.StringVar(&c.ContentType, "kube-api-content-type", c.ContentType, "Content type of requests sent to apiserver.")
   490  	fs.Int32Var(&c.KubeAPIQPS, "kube-api-qps", c.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver. The number must be >= 0. If 0 will use DefaultQPS: 50. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags")
   491  	fs.Int32Var(&c.KubeAPIBurst, "kube-api-burst", c.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver. The number must be >= 0. If 0 will use DefaultBurst: 100. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags")
   492  	fs.BoolVar(&c.SerializeImagePulls, "serialize-image-pulls", c.SerializeImagePulls, "Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details.")
   493  
   494  	fs.Var(cliflag.NewLangleSeparatedMapStringString(&c.EvictionHard), "eviction-hard", "A set of eviction thresholds (e.g. memory.available<1Gi) that if met would trigger a pod eviction.")
   495  	fs.Var(cliflag.NewLangleSeparatedMapStringString(&c.EvictionSoft), "eviction-soft", "A set of eviction thresholds (e.g. memory.available<1.5Gi) that if met over a corresponding grace period would trigger a pod eviction.")
   496  	fs.Var(cliflag.NewMapStringString(&c.EvictionSoftGracePeriod), "eviction-soft-grace-period", "A set of eviction grace periods (e.g. memory.available=1m30s) that correspond to how long a soft eviction threshold must hold before triggering a pod eviction.")
   497  	fs.DurationVar(&c.EvictionPressureTransitionPeriod.Duration, "eviction-pressure-transition-period", c.EvictionPressureTransitionPeriod.Duration, "Duration for which the kubelet has to wait before transitioning out of an eviction pressure condition.")
   498  	fs.Int32Var(&c.EvictionMaxPodGracePeriod, "eviction-max-pod-grace-period", c.EvictionMaxPodGracePeriod, "Maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met.  If negative, defer to pod specified value.")
   499  	fs.Var(cliflag.NewMapStringString(&c.EvictionMinimumReclaim), "eviction-minimum-reclaim", "A set of minimum reclaims (e.g. imagefs.available=2Gi) that describes the minimum amount of resource the kubelet will reclaim when performing a pod eviction if that resource is under pressure.")
   500  	fs.Int32Var(&c.PodsPerCore, "pods-per-core", c.PodsPerCore, "Number of Pods per core that can run on this Kubelet. The total number of Pods on this Kubelet cannot exceed max-pods, so max-pods will be used if this calculation results in a larger number of Pods allowed on the Kubelet. A value of 0 disables this limit.")
   501  	fs.BoolVar(&c.ProtectKernelDefaults, "protect-kernel-defaults", c.ProtectKernelDefaults, "Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults.")
   502  	fs.StringVar(&c.ReservedSystemCPUs, "reserved-cpus", c.ReservedSystemCPUs, "A comma-separated list of CPUs or CPU ranges that are reserved for system and kubernetes usage. This specific list will supersede cpu counts in --system-reserved and --kube-reserved.")
   503  	fs.StringVar(&c.TopologyManagerScope, "topology-manager-scope", c.TopologyManagerScope, "Scope to which topology hints applied. Topology Manager collects hints from Hint Providers and applies them to defined scope to ensure the pod admission. Possible values: 'container', 'pod'.")
   504  	fs.Var(cliflag.NewMapStringStringNoSplit(&c.TopologyManagerPolicyOptions), "topology-manager-policy-options", "A set of key=value Topology Manager policy options to use, to fine tune their behaviour. If not supplied, keep the default behaviour.")
   505  	// Node Allocatable Flags
   506  	fs.Var(cliflag.NewMapStringString(&c.SystemReserved), "system-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=500Mi,ephemeral-storage=1Gi,pid=1000) pairs that describe resources reserved for non-kubernetes components. Currently only cpu, memory, pid and local ephemeral storage for root file system are supported. See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for more detail. [default=none]")
   507  	fs.Var(cliflag.NewMapStringString(&c.KubeReserved), "kube-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=500Mi,ephemeral-storage=1Gi,pid=1000) pairs that describe resources reserved for kubernetes system components. Currently only cpu, memory, pid and local ephemeral storage for root file system are supported. See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for more detail. [default=none]")
   508  	fs.StringSliceVar(&c.EnforceNodeAllocatable, "enforce-node-allocatable", c.EnforceNodeAllocatable, "A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. Acceptable options are 'none', 'pods', 'system-reserved', and 'kube-reserved'. If the latter two options are specified, '--system-reserved-cgroup' and '--kube-reserved-cgroup' must also be set, respectively. If 'none' is specified, no additional options should be set. See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ for more details.")
   509  	fs.StringVar(&c.SystemReservedCgroup, "system-reserved-cgroup", c.SystemReservedCgroup, "Absolute name of the top level cgroup that is used to manage non-kubernetes components for which compute resources were reserved via '--system-reserved' flag. Ex. '/system-reserved'. [default='']")
   510  	fs.StringVar(&c.KubeReservedCgroup, "kube-reserved-cgroup", c.KubeReservedCgroup, "Absolute name of the top level cgroup that is used to manage kubernetes components for which compute resources were reserved via '--kube-reserved' flag. Ex. '/kube-reserved'. [default='']")
   511  	logsapi.AddFlags(&c.Logging, fs)
   512  
   513  	// Memory Manager Flags
   514  	fs.StringVar(&c.MemoryManagerPolicy, "memory-manager-policy", c.MemoryManagerPolicy, "Memory Manager policy to use. Possible values: 'None', 'Static'.")
   515  	fs.Var(&utilflag.ReservedMemoryVar{Value: &c.ReservedMemory}, "reserved-memory", "A comma separated list of memory reservations for NUMA nodes. (e.g. --reserved-memory 0:memory=1Gi,hugepages-1M=2Gi --reserved-memory 1:memory=2Gi). The total sum for each memory type should be equal to the sum of kube-reserved, system-reserved and eviction-threshold. See https://kubernetes.io/docs/tasks/administer-cluster/memory-manager/#reserved-memory-flag for more details.")
   516  
   517  	fs.BoolVar(&c.RegisterNode, "register-node", c.RegisterNode, "Register the node with the apiserver. If --kubeconfig is not provided, this flag is irrelevant, as the Kubelet won't have an apiserver to register with.")
   518  
   519  	fs.Var(&utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"<key>=<value>:<effect>\"). No-op if register-node is false.")
   520  }
   521
View as plain text