1 /* 2 Copyright 2021 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package controlplane 18 19 import ( 20 "fmt" 21 "io" 22 "net/url" 23 "os" 24 "path/filepath" 25 "strconv" 26 "time" 27 28 "sigs.k8s.io/controller-runtime/pkg/internal/testing/addr" 29 "sigs.k8s.io/controller-runtime/pkg/internal/testing/certs" 30 "sigs.k8s.io/controller-runtime/pkg/internal/testing/process" 31 ) 32 33 const ( 34 // saKeyFile is the name of the service account signing private key file. 35 saKeyFile = "sa-signer.key" 36 // saKeyFile is the name of the service account signing public key (cert) file. 37 saCertFile = "sa-signer.crt" 38 ) 39 40 // SecureServing provides/configures how the API server serves on the secure port. 41 type SecureServing struct { 42 // ListenAddr contains the host & port to serve on. 43 // 44 // Configurable. If unset, it will be defaulted. 45 process.ListenAddr 46 // CA contains the CA that signed the API server's serving certificates. 47 // 48 // Read-only. 49 CA []byte 50 // Authn can be used to provision users, and override what type of 51 // authentication is used to provision users. 52 // 53 // Configurable. If unset, it will be defaulted. 54 Authn 55 } 56 57 // APIServer knows how to run a kubernetes apiserver. 58 type APIServer struct { 59 // URL is the address the ApiServer should listen on for client 60 // connections. 61 // 62 // If set, this will configure the *insecure* serving details. 63 // If unset, it will contain the insecure port if insecure serving is enabled, 64 // and otherwise will contain the secure port. 65 // 66 // If this is not specified, we default to a random free port on localhost. 67 // 68 // Deprecated: use InsecureServing (for the insecure URL) or SecureServing, ideally. 69 URL *url.URL 70 71 // SecurePort is the additional secure port that the APIServer should listen on. 72 // 73 // If set, this will override SecureServing.Port. 74 // 75 // Deprecated: use SecureServing. 76 SecurePort int 77 78 // SecureServing indicates how the API server will serve on the secure port. 79 // 80 // Some parts are configurable. Will be defaulted if unset. 81 SecureServing 82 83 // InsecureServing indicates how the API server will serve on the insecure port. 84 // 85 // If unset, the insecure port will be disabled. Set to an empty struct to get 86 // default values. 87 // 88 // Deprecated: does not work with Kubernetes versions 1.20 and above. Use secure 89 // serving instead. 90 InsecureServing *process.ListenAddr 91 92 // Path is the path to the apiserver binary. 93 // 94 // If this is left as the empty string, we will attempt to locate a binary, 95 // by checking for the TEST_ASSET_KUBE_APISERVER environment variable, and 96 // the default test assets directory. See the "Binaries" section above (in 97 // doc.go) for details. 98 Path string 99 100 // Args is a list of arguments which will passed to the APIServer binary. 101 // Before they are passed on, they will be evaluated as go-template strings. 102 // This means you can use fields which are defined and exported on this 103 // APIServer struct (e.g. "--cert-dir={{ .Dir }}"). 104 // Those templates will be evaluated after the defaulting of the APIServer's 105 // fields has already happened and just before the binary actually gets 106 // started. Thus you have access to calculated fields like `URL` and others. 107 // 108 // If not specified, the minimal set of arguments to run the APIServer will 109 // be used. 110 // 111 // They will be loaded into the same argument set as Configure. Each flag 112 // will be Append-ed to the configured arguments just before launch. 113 // 114 // Deprecated: use Configure instead. 115 Args []string 116 117 // CertDir is a path to a directory containing whatever certificates the 118 // APIServer will need. 119 // 120 // If left unspecified, then the Start() method will create a fresh temporary 121 // directory, and the Stop() method will clean it up. 122 CertDir string 123 124 // EtcdURL is the URL of the Etcd the APIServer should use. 125 // 126 // If this is not specified, the Start() method will return an error. 127 EtcdURL *url.URL 128 129 // StartTimeout, StopTimeout specify the time the APIServer is allowed to 130 // take when starting and stoppping before an error is emitted. 131 // 132 // If not specified, these default to 20 seconds. 133 StartTimeout time.Duration 134 StopTimeout time.Duration 135 136 // Out, Err specify where APIServer should write its StdOut, StdErr to. 137 // 138 // If not specified, the output will be discarded. 139 Out io.Writer 140 Err io.Writer 141 142 processState *process.State 143 144 // args contains the structured arguments to use for running the API server 145 // Lazily initialized by .Configure(), Defaulted eventually with .defaultArgs() 146 args *process.Arguments 147 } 148 149 // Configure returns Arguments that may be used to customize the 150 // flags used to launch the API server. A set of defaults will 151 // be applied underneath. 152 func (s *APIServer) Configure() *process.Arguments { 153 if s.args == nil { 154 s.args = process.EmptyArguments() 155 } 156 return s.args 157 } 158 159 // Start starts the apiserver, waits for it to come up, and returns an error, 160 // if occurred. 161 func (s *APIServer) Start() error { 162 if err := s.prepare(); err != nil { 163 return err 164 } 165 return s.processState.Start(s.Out, s.Err) 166 } 167 168 func (s *APIServer) prepare() error { 169 if err := s.setProcessState(); err != nil { 170 return err 171 } 172 return s.Authn.Start() 173 } 174 175 // configurePorts configures the serving ports for this API server. 176 // 177 // Most of this method currently deals with making the deprecated fields 178 // take precedence over the new fields. 179 func (s *APIServer) configurePorts() error { 180 // prefer the old fields to the new fields if a user set one, 181 // otherwise, default the new fields and populate the old ones. 182 183 // Insecure: URL, InsecureServing 184 if s.URL != nil { 185 s.InsecureServing = &process.ListenAddr{ 186 Address: s.URL.Hostname(), 187 Port: s.URL.Port(), 188 } 189 } else if insec := s.InsecureServing; insec != nil { 190 if insec.Port == "" || insec.Address == "" { 191 port, host, err := addr.Suggest("") 192 if err != nil { 193 return fmt.Errorf("unable to provision unused insecure port: %w", err) 194 } 195 s.InsecureServing.Port = strconv.Itoa(port) 196 s.InsecureServing.Address = host 197 } 198 s.URL = s.InsecureServing.URL("http", "") 199 } 200 201 // Secure: SecurePort, SecureServing 202 if s.SecurePort != 0 { 203 s.SecureServing.Port = strconv.Itoa(s.SecurePort) 204 // if we don't have an address, try the insecure address, and otherwise 205 // default to loopback. 206 if s.SecureServing.Address == "" { 207 if s.InsecureServing != nil { 208 s.SecureServing.Address = s.InsecureServing.Address 209 } else { 210 s.SecureServing.Address = "127.0.0.1" 211 } 212 } 213 } else if s.SecureServing.Port == "" || s.SecureServing.Address == "" { 214 port, host, err := addr.Suggest("") 215 if err != nil { 216 return fmt.Errorf("unable to provision unused secure port: %w", err) 217 } 218 s.SecureServing.Port = strconv.Itoa(port) 219 s.SecureServing.Address = host 220 s.SecurePort = port 221 } 222 223 return nil 224 } 225 226 func (s *APIServer) setProcessState() error { 227 if s.EtcdURL == nil { 228 return fmt.Errorf("expected EtcdURL to be configured") 229 } 230 231 var err error 232 233 // unconditionally re-set this so we can successfully restart 234 // TODO(directxman12): we supported this in the past, but do we actually 235 // want to support re-using an API server object to restart? The loss 236 // of provisioned users is surprising to say the least. 237 s.processState = &process.State{ 238 Dir: s.CertDir, 239 Path: s.Path, 240 StartTimeout: s.StartTimeout, 241 StopTimeout: s.StopTimeout, 242 } 243 if err := s.processState.Init("kube-apiserver"); err != nil { 244 return err 245 } 246 247 if err := s.configurePorts(); err != nil { 248 return err 249 } 250 251 // the secure port will always be on, so use that 252 s.processState.HealthCheck.URL = *s.SecureServing.URL("https", "/healthz") 253 254 s.CertDir = s.processState.Dir 255 s.Path = s.processState.Path 256 s.StartTimeout = s.processState.StartTimeout 257 s.StopTimeout = s.processState.StopTimeout 258 259 if err := s.populateAPIServerCerts(); err != nil { 260 return err 261 } 262 263 if s.SecureServing.Authn == nil { 264 authn, err := NewCertAuthn() 265 if err != nil { 266 return err 267 } 268 s.SecureServing.Authn = authn 269 } 270 271 if err := s.Authn.Configure(s.CertDir, s.Configure()); err != nil { 272 return err 273 } 274 275 // NB(directxman12): insecure port is a mess: 276 // - 1.19 and below have the `--insecure-port` flag, and require it to be set to zero to 277 // disable it, otherwise the default will be used and we'll conflict. 278 // - 1.20 requires the flag to be unset or set to zero, and yells at you if you configure it 279 // - 1.24 won't have the flag at all... 280 // 281 // In an effort to automatically do the right thing during this mess, we do feature discovery 282 // on the flags, and hope that we've "parsed" them properly. 283 // 284 // TODO(directxman12): once we support 1.20 as the min version (might be when 1.24 comes out, 285 // might be around 1.25 or 1.26), remove this logic and the corresponding line in API server's 286 // default args. 287 if err := s.discoverFlags(); err != nil { 288 return err 289 } 290 291 s.processState.Args, s.Args, err = process.TemplateAndArguments(s.Args, s.Configure(), process.TemplateDefaults{ //nolint:staticcheck 292 Data: s, 293 Defaults: s.defaultArgs(), 294 MinimalDefaults: map[string][]string{ 295 // as per kubernetes-sigs/controller-runtime#641, we need this (we 296 // probably need other stuff too, but this is the only thing that was 297 // previously considered a "minimal default") 298 "service-cluster-ip-range": {"10.0.0.0/24"}, 299 300 // we need *some* authorization mode for health checks on the secure port, 301 // so default to RBAC unless the user set something else (in which case 302 // this'll be ignored due to SliceToArguments using AppendNoDefaults). 303 "authorization-mode": {"RBAC"}, 304 }, 305 }) 306 if err != nil { 307 return err 308 } 309 310 return nil 311 } 312 313 // discoverFlags checks for certain flags that *must* be set in certain 314 // versions, and *must not* be set in others. 315 func (s *APIServer) discoverFlags() error { 316 // Present: <1.24, Absent: >= 1.24 317 present, err := s.processState.CheckFlag("insecure-port") 318 if err != nil { 319 return err 320 } 321 322 if !present { 323 s.Configure().Disable("insecure-port") 324 } 325 326 return nil 327 } 328 329 func (s *APIServer) defaultArgs() map[string][]string { 330 args := map[string][]string{ 331 "service-cluster-ip-range": {"10.0.0.0/24"}, 332 "allow-privileged": {"true"}, 333 // we're keeping this disabled because if enabled, default SA is 334 // missing which would force all tests to create one in normal 335 // apiserver operation this SA is created by controller, but that is 336 // not run in integration environment 337 "disable-admission-plugins": {"ServiceAccount"}, 338 "cert-dir": {s.CertDir}, 339 "authorization-mode": {"RBAC"}, 340 "secure-port": {s.SecureServing.Port}, 341 // NB(directxman12): previously we didn't set the bind address for the secure 342 // port. It *shouldn't* make a difference unless people are doing something really 343 // funky, but if you start to get bug reports look here ;-) 344 "bind-address": {s.SecureServing.Address}, 345 346 // required on 1.20+, fine to leave on for <1.20 347 "service-account-issuer": {s.SecureServing.URL("https", "/").String()}, 348 "service-account-key-file": {filepath.Join(s.CertDir, saCertFile)}, 349 "service-account-signing-key-file": {filepath.Join(s.CertDir, saKeyFile)}, 350 } 351 if s.EtcdURL != nil { 352 args["etcd-servers"] = []string{s.EtcdURL.String()} 353 } 354 if s.URL != nil { 355 args["insecure-port"] = []string{s.URL.Port()} 356 args["insecure-bind-address"] = []string{s.URL.Hostname()} 357 } else { 358 // TODO(directxman12): remove this once 1.21 is the lowest version we support 359 // (this might be a while, but this line'll break as of 1.24, so see the comment 360 // in Start 361 args["insecure-port"] = []string{"0"} 362 } 363 return args 364 } 365 366 func (s *APIServer) populateAPIServerCerts() error { 367 _, statErr := os.Stat(filepath.Join(s.CertDir, "apiserver.crt")) 368 if !os.IsNotExist(statErr) { 369 return statErr 370 } 371 372 ca, err := certs.NewTinyCA() 373 if err != nil { 374 return err 375 } 376 377 servingCerts, err := ca.NewServingCert() 378 if err != nil { 379 return err 380 } 381 382 certData, keyData, err := servingCerts.AsBytes() 383 if err != nil { 384 return err 385 } 386 387 if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec 388 return err 389 } 390 if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec 391 return err 392 } 393 394 s.SecureServing.CA = ca.CA.CertBytes() 395 396 // service account signing files too 397 saCA, err := certs.NewTinyCA() 398 if err != nil { 399 return err 400 } 401 402 saCert, saKey, err := saCA.CA.AsBytes() 403 if err != nil { 404 return err 405 } 406 407 if err := os.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec 408 return err 409 } 410 return os.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0640) //nolint:gosec 411 } 412 413 // Stop stops this process gracefully, waits for its termination, and cleans up 414 // the CertDir if necessary. 415 func (s *APIServer) Stop() error { 416 if s.processState != nil { 417 if s.processState.DirNeedsCleaning { 418 s.CertDir = "" // reset the directory if it was randomly allocated, so that we can safely restart 419 } 420 if err := s.processState.Stop(); err != nil { 421 return err 422 } 423 } 424 return s.Authn.Stop() 425 } 426 427 // APIServerDefaultArgs exposes the default args for the APIServer so that you 428 // can use those to append your own additional arguments. 429 // 430 // Note that these arguments don't handle newer API servers well to due the more 431 // complex feature detection neeeded. It's recommended that you switch to .Configure 432 // as you upgrade API server versions. 433 // 434 // Deprecated: use APIServer.Configure(). 435 var APIServerDefaultArgs = []string{ 436 "--advertise-address=127.0.0.1", 437 "--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}", 438 "--cert-dir={{ .CertDir }}", 439 "--insecure-port={{ if .URL }}{{ .URL.Port }}{{else}}0{{ end }}", 440 "{{ if .URL }}--insecure-bind-address={{ .URL.Hostname }}{{ end }}", 441 "--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}", 442 // we're keeping this disabled because if enabled, default SA is missing which would force all tests to create one 443 // in normal apiserver operation this SA is created by controller, but that is not run in integration environment 444 "--disable-admission-plugins=ServiceAccount", 445 "--service-cluster-ip-range=10.0.0.0/24", 446 "--allow-privileged=true", 447 // NB(directxman12): we also enable RBAC if nothing else was enabled 448 } 449 450 // PrepareAPIServer is an internal-only (NEVER SHOULD BE EXPOSED) 451 // function that sets up the API server just before starting it, 452 // without actually starting it. This saves time on tests. 453 // 454 // NB(directxman12): do not expose this outside of internal -- it's unsafe to 455 // use, because things like port allocation could race even more than they 456 // currently do if you later call start! 457 func PrepareAPIServer(s *APIServer) error { 458 return s.prepare() 459 } 460 461 // APIServerArguments is an internal-only (NEVER SHOULD BE EXPOSED) 462 // function that sets up the API server just before starting it, 463 // without actually starting it. It's public to make testing easier. 464 // 465 // NB(directxman12): do not expose this outside of internal. 466 func APIServerArguments(s *APIServer) []string { 467 return s.processState.Args 468 } 469