options.go

Documentation: k8s.io/kubernetes/pkg/controlplane/apiserver/options

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package options contains flags and options for initializing an apiserver
    18  package options
    19  
    20  import (
    21  	"fmt"
    22  	"net"
    23  	"os"
    24  	"strings"
    25  	"time"
    26  
    27  	peerreconcilers "k8s.io/apiserver/pkg/reconcilers"
    28  	genericoptions "k8s.io/apiserver/pkg/server/options"
    29  	"k8s.io/apiserver/pkg/storage/storagebackend"
    30  	"k8s.io/client-go/util/keyutil"
    31  	cliflag "k8s.io/component-base/cli/flag"
    32  	"k8s.io/component-base/logs"
    33  	logsapi "k8s.io/component-base/logs/api/v1"
    34  	"k8s.io/component-base/metrics"
    35  	"k8s.io/klog/v2"
    36  	netutil "k8s.io/utils/net"
    37  
    38  	_ "k8s.io/kubernetes/pkg/features"
    39  	kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
    40  	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
    41  	"k8s.io/kubernetes/pkg/serviceaccount"
    42  )
    43  
    44  // Options define the flags and validation for a generic controlplane. If the
    45  // structs are nil, the options are not added to the command line and not validated.
    46  type Options struct {
    47  	GenericServerRunOptions *genericoptions.ServerRunOptions
    48  	Etcd                    *genericoptions.EtcdOptions
    49  	SecureServing           *genericoptions.SecureServingOptionsWithLoopback
    50  	Audit                   *genericoptions.AuditOptions
    51  	Features                *genericoptions.FeatureOptions
    52  	Admission               *kubeoptions.AdmissionOptions
    53  	Authentication          *kubeoptions.BuiltInAuthenticationOptions
    54  	Authorization           *kubeoptions.BuiltInAuthorizationOptions
    55  	APIEnablement           *genericoptions.APIEnablementOptions
    56  	EgressSelector          *genericoptions.EgressSelectorOptions
    57  	Metrics                 *metrics.Options
    58  	Logs                    *logs.Options
    59  	Traces                  *genericoptions.TracingOptions
    60  
    61  	EnableLogsHandler        bool
    62  	EventTTL                 time.Duration
    63  	MaxConnectionBytesPerSec int64
    64  
    65  	ProxyClientCertFile string
    66  	ProxyClientKeyFile  string
    67  
    68  	// PeerCAFile is the ca bundle used by this kube-apiserver to verify peer apiservers'
    69  	// serving certs when routing a request to the peer in the case the request can not be served
    70  	// locally due to version skew.
    71  	PeerCAFile string
    72  
    73  	// PeerAdvertiseAddress is the IP for this kube-apiserver which is used by peer apiservers to route a request
    74  	// to this apiserver. This happens in cases where the peer is not able to serve the request due to
    75  	// version skew.
    76  	PeerAdvertiseAddress peerreconcilers.PeerAdvertiseAddress
    77  
    78  	EnableAggregatorRouting             bool
    79  	AggregatorRejectForwardingRedirects bool
    80  
    81  	ServiceAccountSigningKeyFile     string
    82  	ServiceAccountIssuer             serviceaccount.TokenGenerator
    83  	ServiceAccountTokenMaxExpiration time.Duration
    84  
    85  	ShowHiddenMetricsForVersion string
    86  }
    87  
    88  // completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
    89  type completedOptions struct {
    90  	Options
    91  }
    92  
    93  type CompletedOptions struct {
    94  	// Embed a private pointer that cannot be instantiated outside of this package.
    95  	*completedOptions
    96  }
    97  
    98  // NewOptions creates a new ServerRunOptions object with default parameters
    99  func NewOptions() *Options {
   100  	s := Options{
   101  		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
   102  		Etcd:                    genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
   103  		SecureServing:           kubeoptions.NewSecureServingOptions(),
   104  		Audit:                   genericoptions.NewAuditOptions(),
   105  		Features:                genericoptions.NewFeatureOptions(),
   106  		Admission:               kubeoptions.NewAdmissionOptions(),
   107  		Authentication:          kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
   108  		Authorization:           kubeoptions.NewBuiltInAuthorizationOptions(),
   109  		APIEnablement:           genericoptions.NewAPIEnablementOptions(),
   110  		EgressSelector:          genericoptions.NewEgressSelectorOptions(),
   111  		Metrics:                 metrics.NewOptions(),
   112  		Logs:                    logs.NewOptions(),
   113  		Traces:                  genericoptions.NewTracingOptions(),
   114  
   115  		EnableLogsHandler:                   true,
   116  		EventTTL:                            1 * time.Hour,
   117  		AggregatorRejectForwardingRedirects: true,
   118  	}
   119  
   120  	// Overwrite the default for storage data format.
   121  	s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
   122  
   123  	return &s
   124  }
   125  
   126  func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) {
   127  	// Add the generic flags.
   128  	s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic"))
   129  	s.Etcd.AddFlags(fss.FlagSet("etcd"))
   130  	s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
   131  	s.Audit.AddFlags(fss.FlagSet("auditing"))
   132  	s.Features.AddFlags(fss.FlagSet("features"))
   133  	s.Authentication.AddFlags(fss.FlagSet("authentication"))
   134  	s.Authorization.AddFlags(fss.FlagSet("authorization"))
   135  	s.APIEnablement.AddFlags(fss.FlagSet("API enablement"))
   136  	s.EgressSelector.AddFlags(fss.FlagSet("egress selector"))
   137  	s.Admission.AddFlags(fss.FlagSet("admission"))
   138  	s.Metrics.AddFlags(fss.FlagSet("metrics"))
   139  	logsapi.AddFlags(s.Logs, fss.FlagSet("logs"))
   140  	s.Traces.AddFlags(fss.FlagSet("traces"))
   141  
   142  	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
   143  	// arrange these text blocks sensibly. Grrr.
   144  	fs := fss.FlagSet("misc")
   145  	fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL,
   146  		"Amount of time to retain events.")
   147  
   148  	fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler,
   149  		"If true, install a /logs handler for the apiserver logs.")
   150  	fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19")
   151  
   152  	fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+
   153  		"If non-zero, throttle each user connection to this number of bytes/sec. "+
   154  		"Currently only applies to long-running requests.")
   155  
   156  	fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
   157  		"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
   158  		"when it must call out during a request. This includes proxying requests to a user "+
   159  		"api-server and calling out to webhook admission plugins. It is expected that this "+
   160  		"cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+
   161  		"That CA is published in the 'extension-apiserver-authentication' configmap in "+
   162  		"the kube-system namespace. Components receiving calls from kube-aggregator should "+
   163  		"use that CA to perform their half of the mutual TLS verification.")
   164  	fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+
   165  		"Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+
   166  		"when it must call out during a request. This includes proxying requests to a user "+
   167  		"api-server and calling out to webhook admission plugins.")
   168  
   169  	fs.StringVar(&s.PeerCAFile, "peer-ca-file", s.PeerCAFile,
   170  		"If set and the UnknownVersionInteroperabilityProxy feature gate is enabled, this file will be used to verify serving certificates of peer kube-apiservers. "+
   171  			"This flag is only used in clusters configured with multiple kube-apiservers for high availability.")
   172  
   173  	fs.StringVar(&s.PeerAdvertiseAddress.PeerAdvertiseIP, "peer-advertise-ip", s.PeerAdvertiseAddress.PeerAdvertiseIP,
   174  		"If set and the UnknownVersionInteroperabilityProxy feature gate is enabled, this IP will be used by peer kube-apiservers to proxy requests to this kube-apiserver "+
   175  			"when the request cannot be handled by the peer due to version skew between the kube-apiservers. "+
   176  			"This flag is only used in clusters configured with multiple kube-apiservers for high availability. ")
   177  
   178  	fs.StringVar(&s.PeerAdvertiseAddress.PeerAdvertisePort, "peer-advertise-port", s.PeerAdvertiseAddress.PeerAdvertisePort,
   179  		"If set and the UnknownVersionInteroperabilityProxy feature gate is enabled, this port will be used by peer kube-apiservers to proxy requests to this kube-apiserver "+
   180  			"when the request cannot be handled by the peer due to version skew between the kube-apiservers. "+
   181  			"This flag is only used in clusters configured with multiple kube-apiservers for high availability. ")
   182  
   183  	fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting,
   184  		"Turns on aggregator routing requests to endpoints IP rather than cluster IP.")
   185  
   186  	fs.BoolVar(&s.AggregatorRejectForwardingRedirects, "aggregator-reject-forwarding-redirect", s.AggregatorRejectForwardingRedirects,
   187  		"Aggregator reject forwarding redirect response back to client.")
   188  
   189  	fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
   190  		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
   191  }
   192  
   193  func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
   194  	if o == nil {
   195  		return CompletedOptions{completedOptions: &completedOptions{}}, nil
   196  	}
   197  
   198  	completed := completedOptions{
   199  		Options: *o,
   200  	}
   201  
   202  	// set defaults
   203  	if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil {
   204  		return CompletedOptions{}, err
   205  	}
   206  
   207  	if err := completed.SecureServing.MaybeDefaultWithSelfSignedCerts(completed.GenericServerRunOptions.AdvertiseAddress.String(), alternateDNS, alternateIPs); err != nil {
   208  		return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err)
   209  	}
   210  
   211  	if len(completed.GenericServerRunOptions.ExternalHost) == 0 {
   212  		if len(completed.GenericServerRunOptions.AdvertiseAddress) > 0 {
   213  			completed.GenericServerRunOptions.ExternalHost = completed.GenericServerRunOptions.AdvertiseAddress.String()
   214  		} else {
   215  			hostname, err := os.Hostname()
   216  			if err != nil {
   217  				return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err)
   218  			}
   219  			completed.GenericServerRunOptions.ExternalHost = hostname
   220  		}
   221  		klog.Infof("external host was not specified, using %v", completed.GenericServerRunOptions.ExternalHost)
   222  	}
   223  
   224  	// put authorization options in final state
   225  	completed.Authorization.Complete()
   226  	// adjust authentication for completed authorization
   227  	completed.Authentication.ApplyAuthorization(completed.Authorization)
   228  
   229  	// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
   230  	// TokenRequest functionality. This defaulting was convenient, but messed up
   231  	// a lot of people when they rotated their serving cert with no idea it was
   232  	// connected to their service account keys. We are taking this opportunity to
   233  	// remove this problematic defaulting.
   234  	if completed.ServiceAccountSigningKeyFile == "" {
   235  		// Default to the private server key for service account token signing
   236  		if len(completed.Authentication.ServiceAccounts.KeyFiles) == 0 && completed.SecureServing.ServerCert.CertKey.KeyFile != "" {
   237  			if kubeauthenticator.IsValidServiceAccountKeyFile(completed.SecureServing.ServerCert.CertKey.KeyFile) {
   238  				completed.Authentication.ServiceAccounts.KeyFiles = []string{completed.SecureServing.ServerCert.CertKey.KeyFile}
   239  			} else {
   240  				klog.Warning("No TLS key provided, service account token authentication disabled")
   241  			}
   242  		}
   243  	}
   244  
   245  	if completed.ServiceAccountSigningKeyFile != "" && len(completed.Authentication.ServiceAccounts.Issuers) != 0 && completed.Authentication.ServiceAccounts.Issuers[0] != "" {
   246  		sk, err := keyutil.PrivateKeyFromFile(completed.ServiceAccountSigningKeyFile)
   247  		if err != nil {
   248  			return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
   249  		}
   250  		if completed.Authentication.ServiceAccounts.MaxExpiration != 0 {
   251  			lowBound := time.Hour
   252  			upBound := time.Duration(1<<32) * time.Second
   253  			if completed.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
   254  				completed.Authentication.ServiceAccounts.MaxExpiration > upBound {
   255  				return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
   256  			}
   257  			if completed.Authentication.ServiceAccounts.ExtendExpiration {
   258  				if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
   259  					klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
   260  				}
   261  				if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
   262  					klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
   263  				}
   264  			}
   265  		}
   266  
   267  		completed.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(completed.Authentication.ServiceAccounts.Issuers[0], sk)
   268  		if err != nil {
   269  			return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err)
   270  		}
   271  		completed.ServiceAccountTokenMaxExpiration = completed.Authentication.ServiceAccounts.MaxExpiration
   272  	}
   273  
   274  	for key, value := range completed.APIEnablement.RuntimeConfig {
   275  		if key == "v1" || strings.HasPrefix(key, "v1/") ||
   276  			key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
   277  			delete(completed.APIEnablement.RuntimeConfig, key)
   278  			completed.APIEnablement.RuntimeConfig["/v1"] = value
   279  		}
   280  		if key == "api/legacy" {
   281  			delete(completed.APIEnablement.RuntimeConfig, key)
   282  		}
   283  	}
   284  
   285  	return CompletedOptions{
   286  		completedOptions: &completed,
   287  	}, nil
   288  }
   289  
   290  // ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and
   291  // setting service ip range to the default value in kubeoptions.DefaultServiceIPCIDR
   292  // for now until the default is removed per the deprecation timeline guidelines.
   293  // Returns service ip range, api server service IP, and an error
   294  func ServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) {
   295  	serviceClusterIPRange := passedServiceClusterIPRange
   296  	if passedServiceClusterIPRange.IP == nil {
   297  		klog.Warningf("No CIDR for service cluster IPs specified. Default value which was %s is deprecated and will be removed in future releases. Please specify it using --service-cluster-ip-range on kube-apiserver.", kubeoptions.DefaultServiceIPCIDR.String())
   298  		serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
   299  	}
   300  
   301  	size := min(netutil.RangeSize(&serviceClusterIPRange), 1<<16)
   302  	if size < 8 {
   303  		return net.IPNet{}, net.IP{}, fmt.Errorf("the service cluster IP range must be at least %d IP addresses", 8)
   304  	}
   305  
   306  	// Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP.
   307  	apiServerServiceIP, err := netutil.GetIndexedIP(&serviceClusterIPRange, 1)
   308  	if err != nil {
   309  		return net.IPNet{}, net.IP{}, err
   310  	}
   311  	klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP)
   312  
   313  	return serviceClusterIPRange, apiServerServiceIP, nil
   314  }
   315
View as plain text