webhook.go

Documentation: sigs.k8s.io/controller-runtime/pkg/envtest

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  Licensed under the Apache License, Version 2.0 (the "License");
     4  you may not use this file except in compliance with the License.
     5  You may obtain a copy of the License at
     6      http://www.apache.org/licenses/LICENSE-2.0
     7  Unless required by applicable law or agreed to in writing, software
     8  distributed under the License is distributed on an "AS IS" BASIS,
     9  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    10  See the License for the specific language governing permissions and
    11  limitations under the License.
    12  */
    13  
    14  package envtest
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"net"
    20  	"os"
    21  	"path/filepath"
    22  	"time"
    23  
    24  	admissionv1 "k8s.io/api/admissionregistration/v1"
    25  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    28  	"k8s.io/apimachinery/pkg/runtime/schema"
    29  	"k8s.io/apimachinery/pkg/util/sets"
    30  	"k8s.io/apimachinery/pkg/util/wait"
    31  	"k8s.io/client-go/kubernetes/scheme"
    32  	"k8s.io/client-go/rest"
    33  	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
    34  	"sigs.k8s.io/yaml"
    35  
    36  	"sigs.k8s.io/controller-runtime/pkg/client"
    37  	"sigs.k8s.io/controller-runtime/pkg/internal/testing/addr"
    38  	"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
    39  )
    40  
    41  // WebhookInstallOptions are the options for installing mutating or validating webhooks.
    42  type WebhookInstallOptions struct {
    43  	// Paths is a list of paths to the directories or files containing the mutating or validating webhooks yaml or json configs.
    44  	Paths []string
    45  
    46  	// MutatingWebhooks is a list of MutatingWebhookConfigurations to install
    47  	MutatingWebhooks []*admissionv1.MutatingWebhookConfiguration
    48  
    49  	// ValidatingWebhooks is a list of ValidatingWebhookConfigurations to install
    50  	ValidatingWebhooks []*admissionv1.ValidatingWebhookConfiguration
    51  
    52  	// IgnoreSchemeConvertible, will modify any CRD conversion webhook to use the local serving host and port,
    53  	// bypassing the need to have the types registered in the Scheme. This is useful for testing CRD conversion webhooks
    54  	// with unregistered or unstructured types.
    55  	IgnoreSchemeConvertible bool
    56  
    57  	// IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true
    58  	IgnoreErrorIfPathMissing bool
    59  
    60  	// LocalServingHost is the host for serving webhooks on.
    61  	// it will be automatically populated
    62  	LocalServingHost string
    63  
    64  	// LocalServingPort is the allocated port for serving webhooks on.
    65  	// it will be automatically populated by a random available local port
    66  	LocalServingPort int
    67  
    68  	// LocalServingCertDir is the allocated directory for serving certificates.
    69  	// it will be automatically populated by the local temp dir
    70  	LocalServingCertDir string
    71  
    72  	// CAData is the CA that can be used to trust the serving certificates in LocalServingCertDir.
    73  	LocalServingCAData []byte
    74  
    75  	// LocalServingHostExternalName is the hostname to use to reach the webhook server.
    76  	LocalServingHostExternalName string
    77  
    78  	// MaxTime is the max time to wait
    79  	MaxTime time.Duration
    80  
    81  	// PollInterval is the interval to check
    82  	PollInterval time.Duration
    83  }
    84  
    85  // ModifyWebhookDefinitions modifies webhook definitions by:
    86  // - applying CABundle based on the provided tinyca
    87  // - if webhook client config uses service spec, it's removed and replaced with direct url.
    88  func (o *WebhookInstallOptions) ModifyWebhookDefinitions() error {
    89  	caData := o.LocalServingCAData
    90  
    91  	// generate host port.
    92  	hostPort, err := o.generateHostPort()
    93  	if err != nil {
    94  		return err
    95  	}
    96  
    97  	for i := range o.MutatingWebhooks {
    98  		for j := range o.MutatingWebhooks[i].Webhooks {
    99  			updateClientConfig(&o.MutatingWebhooks[i].Webhooks[j].ClientConfig, hostPort, caData)
   100  		}
   101  	}
   102  
   103  	for i := range o.ValidatingWebhooks {
   104  		for j := range o.ValidatingWebhooks[i].Webhooks {
   105  			updateClientConfig(&o.ValidatingWebhooks[i].Webhooks[j].ClientConfig, hostPort, caData)
   106  		}
   107  	}
   108  	return nil
   109  }
   110  
   111  func updateClientConfig(cc *admissionv1.WebhookClientConfig, hostPort string, caData []byte) {
   112  	cc.CABundle = caData
   113  	if cc.Service != nil && cc.Service.Path != nil {
   114  		url := fmt.Sprintf("https://%s/%s", hostPort, *cc.Service.Path)
   115  		cc.URL = &url
   116  		cc.Service = nil
   117  	}
   118  }
   119  
   120  func (o *WebhookInstallOptions) generateHostPort() (string, error) {
   121  	if o.LocalServingPort == 0 {
   122  		port, host, err := addr.Suggest(o.LocalServingHost)
   123  		if err != nil {
   124  			return "", fmt.Errorf("unable to grab random port for serving webhooks on: %w", err)
   125  		}
   126  		o.LocalServingPort = port
   127  		o.LocalServingHost = host
   128  	}
   129  	host := o.LocalServingHostExternalName
   130  	if host == "" {
   131  		host = o.LocalServingHost
   132  	}
   133  	return net.JoinHostPort(host, fmt.Sprintf("%d", o.LocalServingPort)), nil
   134  }
   135  
   136  // PrepWithoutInstalling does the setup parts of Install (populating host-port,
   137  // setting up CAs, etc), without actually truing to do anything with webhook
   138  // definitions.  This is largely useful for internal testing of
   139  // controller-runtime, where we need a random host-port & caData for webhook
   140  // tests, but may be useful in similar scenarios.
   141  func (o *WebhookInstallOptions) PrepWithoutInstalling() error {
   142  	if err := o.setupCA(); err != nil {
   143  		return err
   144  	}
   145  
   146  	if err := parseWebhook(o); err != nil {
   147  		return err
   148  	}
   149  
   150  	return o.ModifyWebhookDefinitions()
   151  }
   152  
   153  // Install installs specified webhooks to the API server.
   154  func (o *WebhookInstallOptions) Install(config *rest.Config) error {
   155  	defaultWebhookOptions(o)
   156  
   157  	if len(o.LocalServingCAData) == 0 {
   158  		if err := o.PrepWithoutInstalling(); err != nil {
   159  			return err
   160  		}
   161  	}
   162  
   163  	if err := createWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks); err != nil {
   164  		return err
   165  	}
   166  
   167  	return WaitForWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks, *o)
   168  }
   169  
   170  // Cleanup cleans up cert directories.
   171  func (o *WebhookInstallOptions) Cleanup() error {
   172  	if o.LocalServingCertDir != "" {
   173  		return os.RemoveAll(o.LocalServingCertDir)
   174  	}
   175  	return nil
   176  }
   177  
   178  // defaultWebhookOptions sets the default values for Webhooks.
   179  func defaultWebhookOptions(o *WebhookInstallOptions) {
   180  	if o.MaxTime == 0 {
   181  		o.MaxTime = defaultMaxWait
   182  	}
   183  	if o.PollInterval == 0 {
   184  		o.PollInterval = defaultPollInterval
   185  	}
   186  }
   187  
   188  // WaitForWebhooks waits for the Webhooks to be available through API server.
   189  func WaitForWebhooks(config *rest.Config,
   190  	mutatingWebhooks []*admissionv1.MutatingWebhookConfiguration,
   191  	validatingWebhooks []*admissionv1.ValidatingWebhookConfiguration,
   192  	options WebhookInstallOptions,
   193  ) error {
   194  	waitingFor := map[schema.GroupVersionKind]*sets.Set[string]{}
   195  
   196  	for _, hook := range mutatingWebhooks {
   197  		h := hook
   198  		gvk, err := apiutil.GVKForObject(h, scheme.Scheme)
   199  		if err != nil {
   200  			return fmt.Errorf("unable to get gvk for MutatingWebhookConfiguration %s: %w", hook.GetName(), err)
   201  		}
   202  
   203  		if _, ok := waitingFor[gvk]; !ok {
   204  			waitingFor[gvk] = &sets.Set[string]{}
   205  		}
   206  		waitingFor[gvk].Insert(h.GetName())
   207  	}
   208  
   209  	for _, hook := range validatingWebhooks {
   210  		h := hook
   211  		gvk, err := apiutil.GVKForObject(h, scheme.Scheme)
   212  		if err != nil {
   213  			return fmt.Errorf("unable to get gvk for ValidatingWebhookConfiguration %s: %w", hook.GetName(), err)
   214  		}
   215  
   216  		if _, ok := waitingFor[gvk]; !ok {
   217  			waitingFor[gvk] = &sets.Set[string]{}
   218  		}
   219  		waitingFor[gvk].Insert(hook.GetName())
   220  	}
   221  
   222  	// Poll until all resources are found in discovery
   223  	p := &webhookPoller{config: config, waitingFor: waitingFor}
   224  	return wait.PollUntilContextTimeout(context.TODO(), options.PollInterval, options.MaxTime, true, p.poll)
   225  }
   226  
   227  // poller checks if all the resources have been found in discovery, and returns false if not.
   228  type webhookPoller struct {
   229  	// config is used to get discovery
   230  	config *rest.Config
   231  
   232  	// waitingFor is the map of resources keyed by group version that have not yet been found in discovery
   233  	waitingFor map[schema.GroupVersionKind]*sets.Set[string]
   234  }
   235  
   236  // poll checks if all the resources have been found in discovery, and returns false if not.
   237  func (p *webhookPoller) poll(ctx context.Context) (done bool, err error) {
   238  	// Create a new clientset to avoid any client caching of discovery
   239  	c, err := client.New(p.config, client.Options{})
   240  	if err != nil {
   241  		return false, err
   242  	}
   243  
   244  	allFound := true
   245  	for gvk, names := range p.waitingFor {
   246  		if names.Len() == 0 {
   247  			delete(p.waitingFor, gvk)
   248  			continue
   249  		}
   250  		for _, name := range names.UnsortedList() {
   251  			obj := &unstructured.Unstructured{}
   252  			obj.SetGroupVersionKind(gvk)
   253  			err := c.Get(context.Background(), client.ObjectKey{
   254  				Namespace: "",
   255  				Name:      name,
   256  			}, obj)
   257  
   258  			if err == nil {
   259  				names.Delete(name)
   260  			}
   261  
   262  			if apierrors.IsNotFound(err) {
   263  				allFound = false
   264  			}
   265  			if err != nil {
   266  				return false, err
   267  			}
   268  		}
   269  	}
   270  	return allFound, nil
   271  }
   272  
   273  // setupCA creates CA for testing and writes them to disk.
   274  func (o *WebhookInstallOptions) setupCA() error {
   275  	hookCA, err := certs.NewTinyCA()
   276  	if err != nil {
   277  		return fmt.Errorf("unable to set up webhook CA: %w", err)
   278  	}
   279  
   280  	names := []string{"localhost", o.LocalServingHost, o.LocalServingHostExternalName}
   281  	hookCert, err := hookCA.NewServingCert(names...)
   282  	if err != nil {
   283  		return fmt.Errorf("unable to set up webhook serving certs: %w", err)
   284  	}
   285  
   286  	localServingCertsDir, err := os.MkdirTemp("", "envtest-serving-certs-")
   287  	o.LocalServingCertDir = localServingCertsDir
   288  	if err != nil {
   289  		return fmt.Errorf("unable to create directory for webhook serving certs: %w", err)
   290  	}
   291  
   292  	certData, keyData, err := hookCert.AsBytes()
   293  	if err != nil {
   294  		return fmt.Errorf("unable to marshal webhook serving certs: %w", err)
   295  	}
   296  
   297  	if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { //nolint:gosec
   298  		return fmt.Errorf("unable to write webhook serving cert to disk: %w", err)
   299  	}
   300  	if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { //nolint:gosec
   301  		return fmt.Errorf("unable to write webhook serving key to disk: %w", err)
   302  	}
   303  
   304  	o.LocalServingCAData = certData
   305  	return err
   306  }
   307  
   308  func createWebhooks(config *rest.Config, mutHooks []*admissionv1.MutatingWebhookConfiguration, valHooks []*admissionv1.ValidatingWebhookConfiguration) error {
   309  	cs, err := client.New(config, client.Options{})
   310  	if err != nil {
   311  		return err
   312  	}
   313  
   314  	// Create each webhook
   315  	for _, hook := range mutHooks {
   316  		hook := hook
   317  		log.V(1).Info("installing mutating webhook", "webhook", hook.GetName())
   318  		if err := ensureCreated(cs, hook); err != nil {
   319  			return err
   320  		}
   321  	}
   322  	for _, hook := range valHooks {
   323  		hook := hook
   324  		log.V(1).Info("installing validating webhook", "webhook", hook.GetName())
   325  		if err := ensureCreated(cs, hook); err != nil {
   326  			return err
   327  		}
   328  	}
   329  	return nil
   330  }
   331  
   332  // ensureCreated creates or update object if already exists in the cluster.
   333  func ensureCreated(cs client.Client, obj client.Object) error {
   334  	existing := obj.DeepCopyObject().(client.Object)
   335  	err := cs.Get(context.Background(), client.ObjectKey{Name: obj.GetName()}, existing)
   336  	switch {
   337  	case apierrors.IsNotFound(err):
   338  		if err := cs.Create(context.Background(), obj); err != nil {
   339  			return err
   340  		}
   341  	case err != nil:
   342  		return err
   343  	default:
   344  		log.V(1).Info("Webhook configuration already exists, updating", "webhook", obj.GetName())
   345  		obj.SetResourceVersion(existing.GetResourceVersion())
   346  		if err := cs.Update(context.Background(), obj); err != nil {
   347  			return err
   348  		}
   349  	}
   350  	return nil
   351  }
   352  
   353  // parseWebhook reads the directories or files of Webhooks in options.Paths and adds the Webhook structs to options.
   354  func parseWebhook(options *WebhookInstallOptions) error {
   355  	if len(options.Paths) > 0 {
   356  		for _, path := range options.Paths {
   357  			_, err := os.Stat(path)
   358  			if options.IgnoreErrorIfPathMissing && os.IsNotExist(err) {
   359  				continue // skip this path
   360  			}
   361  			if !options.IgnoreErrorIfPathMissing && os.IsNotExist(err) {
   362  				return err // treat missing path as error
   363  			}
   364  			mutHooks, valHooks, err := readWebhooks(path)
   365  			if err != nil {
   366  				return err
   367  			}
   368  			options.MutatingWebhooks = append(options.MutatingWebhooks, mutHooks...)
   369  			options.ValidatingWebhooks = append(options.ValidatingWebhooks, valHooks...)
   370  		}
   371  	}
   372  	return nil
   373  }
   374  
   375  // readWebhooks reads the Webhooks from files and Unmarshals them into structs
   376  // returns slice of mutating and validating webhook configurations.
   377  func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []*admissionv1.ValidatingWebhookConfiguration, error) {
   378  	// Get the webhook files
   379  	var files []string
   380  	var err error
   381  	log.V(1).Info("reading Webhooks from path", "path", path)
   382  	info, err := os.Stat(path)
   383  	if err != nil {
   384  		return nil, nil, err
   385  	}
   386  	if !info.IsDir() {
   387  		path, files = filepath.Dir(path), []string{info.Name()}
   388  	} else {
   389  		entries, err := os.ReadDir(path)
   390  		if err != nil {
   391  			return nil, nil, err
   392  		}
   393  		for _, e := range entries {
   394  			files = append(files, e.Name())
   395  		}
   396  	}
   397  
   398  	// file extensions that may contain Webhooks
   399  	resourceExtensions := sets.NewString(".json", ".yaml", ".yml")
   400  
   401  	var mutHooks []*admissionv1.MutatingWebhookConfiguration
   402  	var valHooks []*admissionv1.ValidatingWebhookConfiguration
   403  	for _, file := range files {
   404  		// Only parse allowlisted file types
   405  		if !resourceExtensions.Has(filepath.Ext(file)) {
   406  			continue
   407  		}
   408  
   409  		// Unmarshal Webhooks from file into structs
   410  		docs, err := readDocuments(filepath.Join(path, file))
   411  		if err != nil {
   412  			return nil, nil, err
   413  		}
   414  
   415  		for _, doc := range docs {
   416  			var generic metav1.PartialObjectMetadata
   417  			if err = yaml.Unmarshal(doc, &generic); err != nil {
   418  				return nil, nil, err
   419  			}
   420  
   421  			const (
   422  				admissionregv1 = "admissionregistration.k8s.io/v1"
   423  			)
   424  			switch {
   425  			case generic.Kind == "MutatingWebhookConfiguration":
   426  				if generic.APIVersion != admissionregv1 {
   427  					return nil, nil, fmt.Errorf("only v1 is supported right now for MutatingWebhookConfiguration (name: %s)", generic.Name)
   428  				}
   429  				hook := &admissionv1.MutatingWebhookConfiguration{}
   430  				if err := yaml.Unmarshal(doc, hook); err != nil {
   431  					return nil, nil, err
   432  				}
   433  				mutHooks = append(mutHooks, hook)
   434  			case generic.Kind == "ValidatingWebhookConfiguration":
   435  				if generic.APIVersion != admissionregv1 {
   436  					return nil, nil, fmt.Errorf("only v1 is supported right now for ValidatingWebhookConfiguration (name: %s)", generic.Name)
   437  				}
   438  				hook := &admissionv1.ValidatingWebhookConfiguration{}
   439  				if err := yaml.Unmarshal(doc, hook); err != nil {
   440  					return nil, nil, err
   441  				}
   442  				valHooks = append(valHooks, hook)
   443  			default:
   444  				continue
   445  			}
   446  		}
   447  
   448  		log.V(1).Info("read webhooks from file", "file", file)
   449  	}
   450  	return mutHooks, valHooks, nil
   451  }
   452
View as plain text