...

Source file src/k8s.io/kubernetes/test/integration/controlplane/transformation/transformation_test.go

Documentation: k8s.io/kubernetes/test/integration/controlplane/transformation

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package transformation
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"encoding/json"
    23  	"fmt"
    24  	"os"
    25  	"path/filepath"
    26  	"strconv"
    27  	"strings"
    28  	"testing"
    29  	"time"
    30  
    31  	clientv3 "go.etcd.io/etcd/client/v3"
    32  
    33  	appsv1 "k8s.io/api/apps/v1"
    34  	batchv1 "k8s.io/api/batch/v1"
    35  	corev1 "k8s.io/api/core/v1"
    36  	"k8s.io/apimachinery/pkg/api/errors"
    37  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    38  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    39  	"k8s.io/apimachinery/pkg/runtime/schema"
    40  	"k8s.io/apimachinery/pkg/util/wait"
    41  	apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
    42  	"k8s.io/apiserver/pkg/storage/storagebackend"
    43  	"k8s.io/apiserver/pkg/storage/value"
    44  	"k8s.io/client-go/dynamic"
    45  	"k8s.io/client-go/kubernetes"
    46  	"k8s.io/client-go/rest"
    47  	"k8s.io/component-base/metrics/legacyregistry"
    48  	"k8s.io/klog/v2"
    49  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    50  	"k8s.io/kubernetes/test/integration"
    51  	"k8s.io/kubernetes/test/integration/etcd"
    52  	"k8s.io/kubernetes/test/integration/framework"
    53  	"k8s.io/utils/pointer"
    54  	"sigs.k8s.io/yaml"
    55  )
    56  
    57  const (
    58  	secretKey                = "api_key"
    59  	secretVal                = "086a7ffc-0225-11e8-ba89-0ed5f89f718b" // Fake value for testing.
    60  	encryptionConfigFileName = "encryption.conf"
    61  	testNamespace            = "secret-encryption-test"
    62  	testSecret               = "test-secret"
    63  	testConfigmap            = "test-configmap"
    64  	metricsPrefix            = "apiserver_storage_"
    65  	configMapKey             = "foo"
    66  	configMapVal             = "bar"
    67  
    68  	// precomputed key and secret for use with AES CBC
    69  	// this looks exactly the same as the AES GCM secret but with a different value
    70  	oldAESCBCKey = "e0/+tts8FS254BZimFZWtUsOCOUDSkvzB72PyimMlkY="
    71  	oldSecret    = "azhzAAoMCgJ2MRIGU2VjcmV0En4KXwoLdGVzdC1zZWNyZXQSABoWc2VjcmV0LWVuY3J5cHRpb24tdGVzdCIAKiQ3MmRmZTVjNC0xNDU2LTQyMzktYjFlZC1hZGZmYTJmMWY3YmEyADgAQggI5Jy/7wUQAHoAEhMKB2FwaV9rZXkSCPCfpJfwn5C8GgZPcGFxdWUaACIA"
    72  	oldSecretVal = "\xf0\x9f\xa4\x97\xf0\x9f\x90\xbc"
    73  )
    74  
    75  type unSealSecret func(ctx context.Context, cipherText []byte, dataCtx value.Context, config apiserverv1.ProviderConfiguration) ([]byte, error)
    76  
    77  type transformTest struct {
    78  	logger            kubeapiservertesting.Logger
    79  	storageConfig     *storagebackend.Config
    80  	configDir         string
    81  	transformerConfig string
    82  	kubeAPIServer     kubeapiservertesting.TestServer
    83  	restClient        *kubernetes.Clientset
    84  	ns                *corev1.Namespace
    85  	secret            *corev1.Secret
    86  }
    87  
    88  func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string, reload bool, configDir string, storageConfig *storagebackend.Config) (*transformTest, error) {
    89  	if storageConfig == nil {
    90  		storageConfig = framework.SharedEtcd()
    91  	}
    92  	e := transformTest{
    93  		logger:            l,
    94  		transformerConfig: transformerConfigYAML,
    95  		storageConfig:     storageConfig,
    96  	}
    97  
    98  	var err error
    99  	// create config dir with provided config yaml
   100  	if transformerConfigYAML != "" && configDir == "" {
   101  		if e.configDir, err = e.createEncryptionConfig(); err != nil {
   102  			e.cleanUp()
   103  			return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %w", err)
   104  		}
   105  	} else {
   106  		// configDir already exists. api-server must be restarting with existing encryption config
   107  		e.configDir = configDir
   108  	}
   109  	configFile := filepath.Join(e.configDir, encryptionConfigFileName)
   110  	_, err = os.ReadFile(configFile)
   111  	if err != nil {
   112  		e.cleanUp()
   113  		return nil, fmt.Errorf("failed to read config file: %w", err)
   114  	}
   115  
   116  	if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil {
   117  		e.cleanUp()
   118  		return nil, fmt.Errorf("failed to start KubeAPI server: %w", err)
   119  	}
   120  	klog.Infof("Started kube-apiserver %v", e.kubeAPIServer.ClientConfig.Host)
   121  
   122  	if e.restClient, err = kubernetes.NewForConfig(e.kubeAPIServer.ClientConfig); err != nil {
   123  		e.cleanUp()
   124  		return nil, fmt.Errorf("error while creating rest client: %w", err)
   125  	}
   126  
   127  	if e.ns, err = e.createNamespace(testNamespace); err != nil {
   128  		e.cleanUp()
   129  		return nil, err
   130  	}
   131  
   132  	if transformerConfigYAML != "" && reload {
   133  		// when reloading is enabled, this healthz endpoint is always present
   134  		mustBeHealthy(l, "/kms-providers", "ok", e.kubeAPIServer.ClientConfig)
   135  		mustNotHaveLivez(l, "/kms-providers", "404 page not found", e.kubeAPIServer.ClientConfig)
   136  
   137  		// excluding healthz endpoints even if they do not exist should work
   138  		mustBeHealthy(l, "", `warn: some health checks cannot be excluded: no matches for "kms-provider-0","kms-provider-1","kms-provider-2","kms-provider-3"`,
   139  			e.kubeAPIServer.ClientConfig, "kms-provider-0", "kms-provider-1", "kms-provider-2", "kms-provider-3")
   140  	}
   141  
   142  	return &e, nil
   143  }
   144  
   145  func (e *transformTest) cleanUp() {
   146  	if e.configDir != "" {
   147  		os.RemoveAll(e.configDir)
   148  	}
   149  
   150  	if e.kubeAPIServer.ClientConfig != nil {
   151  		e.shutdownAPIServer()
   152  	}
   153  }
   154  
   155  func (e *transformTest) shutdownAPIServer() {
   156  	e.kubeAPIServer.TearDownFn()
   157  }
   158  
   159  func (e *transformTest) runResource(l kubeapiservertesting.Logger, unSealSecretFunc unSealSecret, expectedEnvelopePrefix,
   160  	group,
   161  	version,
   162  	resource,
   163  	name,
   164  	namespaceName string,
   165  ) {
   166  	response, err := e.readRawRecordFromETCD(e.getETCDPathForResource(e.storageConfig.Prefix, group, resource, name, namespaceName))
   167  	if err != nil {
   168  		l.Errorf("failed to read from etcd: %v", err)
   169  		return
   170  	}
   171  
   172  	if !bytes.HasPrefix(response.Kvs[0].Value, []byte(expectedEnvelopePrefix)) {
   173  		l.Errorf("expected data to be prefixed with %s, but got %s",
   174  			expectedEnvelopePrefix, response.Kvs[0].Value)
   175  		return
   176  	}
   177  
   178  	// etcd path of the key is used as the authenticated context - need to pass it to decrypt
   179  	ctx := context.Background()
   180  	dataCtx := value.DefaultContext(e.getETCDPathForResource(e.storageConfig.Prefix, group, resource, name, namespaceName))
   181  	// Envelope header precedes the cipherTextPayload
   182  	sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):]
   183  	transformerConfig, err := e.getEncryptionConfig()
   184  	if err != nil {
   185  		l.Errorf("failed to parse transformer config: %v", err)
   186  	}
   187  	v, err := unSealSecretFunc(ctx, sealedData, dataCtx, *transformerConfig)
   188  	if err != nil {
   189  		l.Errorf("failed to unseal secret: %v", err)
   190  		return
   191  	}
   192  	if resource == "secrets" {
   193  		if !strings.Contains(string(v), secretVal) {
   194  			l.Errorf("expected %q after decryption, but got %q", secretVal, string(v))
   195  		}
   196  	} else if resource == "configmaps" {
   197  		if !strings.Contains(string(v), configMapVal) {
   198  			l.Errorf("expected %q after decryption, but got %q", configMapVal, string(v))
   199  		}
   200  	} else {
   201  		if !strings.Contains(string(v), name) {
   202  			l.Errorf("expected %q after decryption, but got %q", name, string(v))
   203  		}
   204  	}
   205  
   206  	// Data should be un-enveloped on direct reads from Kube API Server.
   207  	if resource == "secrets" {
   208  		s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), name, metav1.GetOptions{})
   209  		if err != nil {
   210  			l.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err)
   211  		}
   212  		if secretVal != string(s.Data[secretKey]) {
   213  			l.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
   214  		}
   215  	} else if resource == "configmaps" {
   216  		s, err := e.restClient.CoreV1().ConfigMaps(namespaceName).Get(context.TODO(), name, metav1.GetOptions{})
   217  		if err != nil {
   218  			l.Fatalf("failed to get ConfigMap from %s, err: %v", namespaceName, err)
   219  		}
   220  		if configMapVal != string(s.Data[configMapKey]) {
   221  			l.Errorf("expected %s from KubeAPI, but got %s", configMapVal, string(s.Data[configMapKey]))
   222  		}
   223  	} else if resource == "pods" {
   224  		p, err := e.restClient.CoreV1().Pods(namespaceName).Get(context.TODO(), name, metav1.GetOptions{})
   225  		if err != nil {
   226  			l.Fatalf("failed to get Pod from %s, err: %v", namespaceName, err)
   227  		}
   228  		if p.Name != name {
   229  			l.Errorf("expected %s from KubeAPI, but got %s", name, p.Name)
   230  		}
   231  	} else {
   232  		l.Logf("Get object with dynamic client")
   233  		fooResource := schema.GroupVersionResource{Group: group, Version: version, Resource: resource}
   234  		obj, err := dynamic.NewForConfigOrDie(e.kubeAPIServer.ClientConfig).Resource(fooResource).Namespace(namespaceName).Get(context.TODO(), name, metav1.GetOptions{})
   235  		if err != nil {
   236  			l.Fatalf("Failed to get test instance: %v, name: %s", err, name)
   237  		}
   238  		if obj.GetObjectKind().GroupVersionKind().Group == group && obj.GroupVersionKind().Version == version && obj.GetKind() == resource && obj.GetNamespace() == namespaceName && obj.GetName() != name {
   239  			l.Errorf("expected %s from KubeAPI, but got %s", name, obj.GetName())
   240  		}
   241  	}
   242  }
   243  
   244  func (e *transformTest) benchmark(b *testing.B) {
   245  	for i := 0; i < b.N; i++ {
   246  		_, err := e.createSecret(e.secret.Name+strconv.Itoa(i), e.ns.Name)
   247  		if err != nil {
   248  			b.Fatalf("failed to create a secret: %v", err)
   249  		}
   250  	}
   251  }
   252  
   253  func (e *transformTest) getETCDPathForResource(storagePrefix, group, resource, name, namespaceName string) string {
   254  	groupResource := resource
   255  	if group != "" {
   256  		groupResource = fmt.Sprintf("%s/%s", group, resource)
   257  	}
   258  	if namespaceName == "" {
   259  		return fmt.Sprintf("/%s/%s/%s", storagePrefix, groupResource, name)
   260  	}
   261  	return fmt.Sprintf("/%s/%s/%s/%s", storagePrefix, groupResource, namespaceName, name)
   262  }
   263  
   264  func (e *transformTest) getRawSecretFromETCD() ([]byte, error) {
   265  	secretETCDPath := e.getETCDPathForResource(e.storageConfig.Prefix, "", "secrets", e.secret.Name, e.secret.Namespace)
   266  	etcdResponse, err := e.readRawRecordFromETCD(secretETCDPath)
   267  	if err != nil {
   268  		return nil, fmt.Errorf("failed to read %s from etcd: %v", secretETCDPath, err)
   269  	}
   270  	return etcdResponse.Kvs[0].Value, nil
   271  }
   272  
   273  func (e *transformTest) getEncryptionOptions(reload bool) []string {
   274  	if e.transformerConfig != "" {
   275  		return []string{
   276  			"--encryption-provider-config", filepath.Join(e.configDir, encryptionConfigFileName),
   277  			fmt.Sprintf("--encryption-provider-config-automatic-reload=%v", reload),
   278  			"--disable-admission-plugins", "ServiceAccount"}
   279  	}
   280  
   281  	return nil
   282  }
   283  
   284  func (e *transformTest) createEncryptionConfig() (
   285  	filePathForEncryptionConfig string,
   286  	err error,
   287  ) {
   288  	tempDir, err := os.MkdirTemp("", "secrets-encryption-test")
   289  	if err != nil {
   290  		return "", fmt.Errorf("failed to create temp directory: %v", err)
   291  	}
   292  
   293  	if err = os.WriteFile(filepath.Join(tempDir, encryptionConfigFileName), []byte(e.transformerConfig), 0644); err != nil {
   294  		os.RemoveAll(tempDir)
   295  		return tempDir, fmt.Errorf("error while writing encryption config: %v", err)
   296  	}
   297  
   298  	return tempDir, nil
   299  }
   300  
   301  func (e *transformTest) getEncryptionConfig() (*apiserverv1.ProviderConfiguration, error) {
   302  	var config apiserverv1.EncryptionConfiguration
   303  	err := yaml.Unmarshal([]byte(e.transformerConfig), &config)
   304  	if err != nil {
   305  		return nil, fmt.Errorf("failed to extract transformer key: %v", err)
   306  	}
   307  
   308  	return &config.Resources[0].Providers[0], nil
   309  }
   310  
   311  func (e *transformTest) createNamespace(name string) (*corev1.Namespace, error) {
   312  	ns := &corev1.Namespace{
   313  		ObjectMeta: metav1.ObjectMeta{
   314  			Name: name,
   315  		},
   316  	}
   317  
   318  	if _, err := e.restClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}); err != nil {
   319  		if errors.IsAlreadyExists(err) {
   320  			existingNs, err := e.restClient.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{})
   321  			if err != nil {
   322  				return nil, fmt.Errorf("unable to get testing namespace, err: [%v]", err)
   323  			}
   324  			return existingNs, nil
   325  		}
   326  		return nil, fmt.Errorf("unable to create testing namespace, err: [%v]", err)
   327  	}
   328  
   329  	return ns, nil
   330  }
   331  
   332  func (e *transformTest) createSecret(name, namespace string) (*corev1.Secret, error) {
   333  	secret := &corev1.Secret{
   334  		ObjectMeta: metav1.ObjectMeta{
   335  			Name:      name,
   336  			Namespace: namespace,
   337  		},
   338  		Data: map[string][]byte{
   339  			secretKey: []byte(secretVal),
   340  		},
   341  	}
   342  	if _, err := e.restClient.CoreV1().Secrets(secret.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil {
   343  		return nil, fmt.Errorf("error while writing secret: %v", err)
   344  	}
   345  
   346  	return secret, nil
   347  }
   348  
   349  func (e *transformTest) createConfigMap(name, namespace string) (*corev1.ConfigMap, error) {
   350  	cm := &corev1.ConfigMap{
   351  		ObjectMeta: metav1.ObjectMeta{
   352  			Name:      name,
   353  			Namespace: namespace,
   354  		},
   355  		Data: map[string]string{
   356  			configMapKey: configMapVal,
   357  		},
   358  	}
   359  	if _, err := e.restClient.CoreV1().ConfigMaps(cm.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil {
   360  		return nil, fmt.Errorf("error while writing configmap: %v", err)
   361  	}
   362  
   363  	return cm, nil
   364  }
   365  
   366  // create jobs
   367  func (e *transformTest) createJob(name, namespace string) (*batchv1.Job, error) {
   368  	job := &batchv1.Job{
   369  		ObjectMeta: metav1.ObjectMeta{
   370  			Name:      name,
   371  			Namespace: namespace,
   372  		},
   373  		Spec: batchv1.JobSpec{
   374  			Template: corev1.PodTemplateSpec{
   375  				Spec: corev1.PodSpec{
   376  					Containers: []corev1.Container{
   377  						{
   378  							Name:  "test",
   379  							Image: "test",
   380  						},
   381  					},
   382  					RestartPolicy: corev1.RestartPolicyNever,
   383  				},
   384  			},
   385  		},
   386  	}
   387  	if _, err := e.restClient.BatchV1().Jobs(job.Namespace).Create(context.TODO(), job, metav1.CreateOptions{}); err != nil {
   388  		return nil, fmt.Errorf("error while creating job: %v", err)
   389  	}
   390  
   391  	return job, nil
   392  }
   393  
   394  // create deployment
   395  func (e *transformTest) createDeployment(name, namespace string) (*appsv1.Deployment, error) {
   396  	deployment := &appsv1.Deployment{
   397  		ObjectMeta: metav1.ObjectMeta{
   398  			Name:      name,
   399  			Namespace: namespace,
   400  		},
   401  		Spec: appsv1.DeploymentSpec{
   402  			Replicas: pointer.Int32(2),
   403  			Selector: &metav1.LabelSelector{
   404  				MatchLabels: map[string]string{
   405  					"app": "nginx",
   406  				},
   407  			},
   408  			Template: corev1.PodTemplateSpec{
   409  				ObjectMeta: metav1.ObjectMeta{
   410  					Labels: map[string]string{
   411  						"app": "nginx",
   412  					},
   413  				},
   414  				Spec: corev1.PodSpec{
   415  					Containers: []corev1.Container{
   416  						{
   417  							Name:  "nginx",
   418  							Image: "nginx:1.17",
   419  							Ports: []corev1.ContainerPort{
   420  								{
   421  									Name:          "http",
   422  									Protocol:      corev1.ProtocolTCP,
   423  									ContainerPort: 80,
   424  								},
   425  							},
   426  						},
   427  					},
   428  				},
   429  			},
   430  		},
   431  	}
   432  	if _, err := e.restClient.AppsV1().Deployments(deployment.Namespace).Create(context.TODO(), deployment, metav1.CreateOptions{}); err != nil {
   433  		return nil, fmt.Errorf("error while creating deployment: %v", err)
   434  	}
   435  
   436  	return deployment, nil
   437  }
   438  
   439  func gvr(group, version, resource string) schema.GroupVersionResource {
   440  	return schema.GroupVersionResource{Group: group, Version: version, Resource: resource}
   441  }
   442  
   443  func createResource(client dynamic.Interface, gvr schema.GroupVersionResource, ns string) (*unstructured.Unstructured, error) {
   444  	stubObj, err := getStubObj(gvr)
   445  	if err != nil {
   446  		return nil, err
   447  	}
   448  	return client.Resource(gvr).Namespace(ns).Create(context.TODO(), stubObj, metav1.CreateOptions{})
   449  }
   450  
   451  func inplaceUpdateResource(client dynamic.Interface, gvr schema.GroupVersionResource, ns string, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
   452  	return client.Resource(gvr).Namespace(ns).Update(context.TODO(), obj, metav1.UpdateOptions{})
   453  }
   454  
   455  func getStubObj(gvr schema.GroupVersionResource) (*unstructured.Unstructured, error) {
   456  	stub := ""
   457  	if data, ok := etcd.GetEtcdStorageDataForNamespace(testNamespace)[gvr]; ok {
   458  		stub = data.Stub
   459  	}
   460  	if len(stub) == 0 {
   461  		return nil, fmt.Errorf("no stub data for %#v", gvr)
   462  	}
   463  
   464  	stubObj := &unstructured.Unstructured{Object: map[string]interface{}{}}
   465  	if err := json.Unmarshal([]byte(stub), &stubObj.Object); err != nil {
   466  		return nil, fmt.Errorf("error unmarshaling stub for %#v: %v", gvr, err)
   467  	}
   468  	return stubObj, nil
   469  }
   470  
   471  func (e *transformTest) createPod(namespace string, dynamicInterface dynamic.Interface) (*unstructured.Unstructured, error) {
   472  	podGVR := gvr("", "v1", "pods")
   473  	pod, err := createResource(dynamicInterface, podGVR, namespace)
   474  	if err != nil {
   475  		return nil, fmt.Errorf("error while writing pod: %v", err)
   476  	}
   477  	return pod, nil
   478  }
   479  
   480  func (e *transformTest) deletePod(namespace string, dynamicInterface dynamic.Interface) error {
   481  	podGVR := gvr("", "v1", "pods")
   482  	stubObj, err := getStubObj(podGVR)
   483  	if err != nil {
   484  		return err
   485  	}
   486  	return dynamicInterface.Resource(podGVR).Namespace(namespace).Delete(context.TODO(), stubObj.GetName(), metav1.DeleteOptions{})
   487  }
   488  
   489  func (e *transformTest) inplaceUpdatePod(namespace string, obj *unstructured.Unstructured, dynamicInterface dynamic.Interface) (*unstructured.Unstructured, error) {
   490  	podGVR := gvr("", "v1", "pods")
   491  	pod, err := inplaceUpdateResource(dynamicInterface, podGVR, namespace, obj)
   492  	if err != nil {
   493  		return nil, fmt.Errorf("error while writing pod: %v", err)
   494  	}
   495  	return pod, nil
   496  }
   497  
   498  func (e *transformTest) readRawRecordFromETCD(path string) (*clientv3.GetResponse, error) {
   499  	rawClient, etcdClient, err := integration.GetEtcdClients(e.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
   500  	if err != nil {
   501  		return nil, fmt.Errorf("failed to create etcd client: %v", err)
   502  	}
   503  	// kvClient is a wrapper around rawClient and to avoid leaking goroutines we need to
   504  	// close the client (which we can do by closing rawClient).
   505  	defer rawClient.Close()
   506  
   507  	response, err := etcdClient.Get(context.Background(), path, clientv3.WithPrefix())
   508  	if err != nil {
   509  		return nil, fmt.Errorf("failed to retrieve secret from etcd %v", err)
   510  	}
   511  
   512  	return response, nil
   513  }
   514  
   515  func (e *transformTest) writeRawRecordToETCD(path string, data []byte) (*clientv3.PutResponse, error) {
   516  	rawClient, etcdClient, err := integration.GetEtcdClients(e.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
   517  	if err != nil {
   518  		return nil, fmt.Errorf("failed to create etcd client: %v", err)
   519  	}
   520  	// kvClient is a wrapper around rawClient and to avoid leaking goroutines we need to
   521  	// close the client (which we can do by closing rawClient).
   522  	defer rawClient.Close()
   523  
   524  	response, err := etcdClient.Put(context.Background(), path, string(data))
   525  	if err != nil {
   526  		return nil, fmt.Errorf("failed to write secret to etcd %v", err)
   527  	}
   528  
   529  	return response, nil
   530  }
   531  
   532  func (e *transformTest) printMetrics() error {
   533  	e.logger.Logf("Transformation Metrics:")
   534  	metrics, err := legacyregistry.DefaultGatherer.Gather()
   535  	if err != nil {
   536  		return fmt.Errorf("failed to gather metrics: %s", err)
   537  	}
   538  
   539  	for _, mf := range metrics {
   540  		if strings.HasPrefix(*mf.Name, metricsPrefix) {
   541  			e.logger.Logf("%s", *mf.Name)
   542  			for _, metric := range mf.GetMetric() {
   543  				e.logger.Logf("%v", metric)
   544  			}
   545  		}
   546  	}
   547  
   548  	return nil
   549  }
   550  
   551  func mustBeHealthy(t kubeapiservertesting.Logger, checkName, wantBodyContains string, clientConfig *rest.Config, excludes ...string) {
   552  	t.Helper()
   553  	var restErr error
   554  	pollErr := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
   555  		body, ok, err := getHealthz(checkName, clientConfig, excludes...)
   556  		restErr = err
   557  		if err != nil {
   558  			return false, err
   559  		}
   560  		done := ok && strings.Contains(body, wantBodyContains)
   561  		if !done {
   562  			t.Logf("expected server check %q to be healthy with message %q but it is not: %s", checkName, wantBodyContains, body)
   563  		}
   564  		return done, nil
   565  	})
   566  
   567  	if pollErr != nil {
   568  		t.Fatalf("failed to get the expected healthz status of OK for check: %s, error: %v, debug inner error: %v", checkName, pollErr, restErr)
   569  	}
   570  }
   571  
   572  func mustBeUnHealthy(t kubeapiservertesting.Logger, checkName, wantBodyContains string, clientConfig *rest.Config, excludes ...string) {
   573  	t.Helper()
   574  	var restErr error
   575  	pollErr := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
   576  		body, ok, err := getHealthz(checkName, clientConfig, excludes...)
   577  		restErr = err
   578  		if err != nil {
   579  			return false, err
   580  		}
   581  		done := !ok && strings.Contains(body, wantBodyContains)
   582  		if !done {
   583  			t.Logf("expected server check %q to be unhealthy with message %q but it is not: %s", checkName, wantBodyContains, body)
   584  		}
   585  		return done, nil
   586  	})
   587  
   588  	if pollErr != nil {
   589  		t.Fatalf("failed to get the expected healthz status of !OK for check: %s, error: %v, debug inner error: %v", checkName, pollErr, restErr)
   590  	}
   591  }
   592  
   593  func mustNotHaveLivez(t kubeapiservertesting.Logger, checkName, wantBodyContains string, clientConfig *rest.Config, excludes ...string) {
   594  	t.Helper()
   595  	var restErr error
   596  	pollErr := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
   597  		body, ok, err := getLivez(checkName, clientConfig, excludes...)
   598  		restErr = err
   599  		if err != nil {
   600  			return false, err
   601  		}
   602  		done := !ok && strings.Contains(body, wantBodyContains)
   603  		if !done {
   604  			t.Logf("expected server check %q with message %q but it is not: %s", checkName, wantBodyContains, body)
   605  		}
   606  		return done, nil
   607  	})
   608  
   609  	if pollErr != nil {
   610  		t.Fatalf("failed to get the expected livez status of !OK for check: %s, error: %v, debug inner error: %v", checkName, pollErr, restErr)
   611  	}
   612  }
   613  
   614  func getHealthz(checkName string, clientConfig *rest.Config, excludes ...string) (string, bool, error) {
   615  	client, err := kubernetes.NewForConfig(clientConfig)
   616  	if err != nil {
   617  		return "", false, fmt.Errorf("failed to create a client: %v", err)
   618  	}
   619  
   620  	req := client.CoreV1().RESTClient().Get().AbsPath(fmt.Sprintf("/healthz%v", checkName)).Param("verbose", "true")
   621  	for _, exclude := range excludes {
   622  		req.Param("exclude", exclude)
   623  	}
   624  	body, err := req.DoRaw(context.TODO()) // we can still have a response body during an error case
   625  	return string(body), err == nil, nil
   626  }
   627  
   628  func getLivez(checkName string, clientConfig *rest.Config, excludes ...string) (string, bool, error) {
   629  	client, err := kubernetes.NewForConfig(clientConfig)
   630  	if err != nil {
   631  		return "", false, fmt.Errorf("failed to create a client: %v", err)
   632  	}
   633  
   634  	req := client.CoreV1().RESTClient().Get().AbsPath(fmt.Sprintf("/livez%v", checkName)).Param("verbose", "true")
   635  	for _, exclude := range excludes {
   636  		req.Param("exclude", exclude)
   637  	}
   638  	body, err := req.DoRaw(context.TODO()) // we can still have a response body during an error case
   639  	return string(body), err == nil, nil
   640  }
   641  

View as plain text