Source file src/k8s.io/kubernetes/test/integration/controlplane/transformation/kms_transformation_test.go

Documentation: k8s.io/kubernetes/test/integration/controlplane/transformation

     1  //go:build !windows
     2  // +build !windows
     3  
     4  /*
     5  Copyright 2017 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package transformation
    21  
    22  import (
    23  	"bytes"
    24  	"context"
    25  	"crypto/aes"
    26  	"encoding/base64"
    27  	"encoding/binary"
    28  	"fmt"
    29  	"math/rand"
    30  	"os"
    31  	"path/filepath"
    32  	"regexp"
    33  	"strings"
    34  	"testing"
    35  	"time"
    36  
    37  	"github.com/google/go-cmp/cmp"
    38  	clientv3 "go.etcd.io/etcd/client/v3"
    39  	"golang.org/x/crypto/cryptobyte"
    40  
    41  	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    42  	"k8s.io/apimachinery/pkg/api/meta"
    43  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    44  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
    45  	"k8s.io/apimachinery/pkg/runtime/schema"
    46  	"k8s.io/apimachinery/pkg/util/sets"
    47  	"k8s.io/apimachinery/pkg/util/wait"
    48  	"k8s.io/apiserver/pkg/features"
    49  	genericapiserver "k8s.io/apiserver/pkg/server"
    50  	encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
    51  	"k8s.io/apiserver/pkg/storage/value"
    52  	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
    53  	mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1"
    54  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    55  	"k8s.io/client-go/dynamic"
    56  	"k8s.io/client-go/rest"
    57  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    58  	kmsapi "k8s.io/kms/apis/v1beta1"
    59  	"k8s.io/kubernetes/test/integration"
    60  	"k8s.io/kubernetes/test/integration/etcd"
    61  	"k8s.io/kubernetes/test/integration/framework"
    62  )
    63  
    64  const (
    65  	dekKeySizeLen = 2
    66  	kmsAPIVersion = "v1beta1"
    67  )
    68  
    69  type envelope struct {
    70  	providerName string
    71  	rawEnvelope  []byte
    72  	plainTextDEK []byte
    73  }
    74  
    75  func (r envelope) prefix() string {
    76  	return fmt.Sprintf("k8s:enc:kms:v1:%s:", r.providerName)
    77  }
    78  
    79  func (r envelope) prefixLen() int {
    80  	return len(r.prefix())
    81  }
    82  
    83  func (r envelope) dekLen() int {
    84  	// DEK's length is stored in the two bytes that follow the prefix.
    85  	return int(binary.BigEndian.Uint16(r.rawEnvelope[r.prefixLen() : r.prefixLen()+dekKeySizeLen]))
    86  }
    87  
    88  func (r envelope) cipherTextDEK() []byte {
    89  	return r.rawEnvelope[r.prefixLen()+dekKeySizeLen : r.prefixLen()+dekKeySizeLen+r.dekLen()]
    90  }
    91  
    92  func (r envelope) startOfPayload(providerName string) int {
    93  	return r.prefixLen() + dekKeySizeLen + r.dekLen()
    94  }
    95  
    96  func (r envelope) cipherTextPayload() []byte {
    97  	return r.rawEnvelope[r.startOfPayload(r.providerName):]
    98  }
    99  
   100  func (r envelope) plainTextPayload(secretETCDPath string) ([]byte, error) {
   101  	block, err := aes.NewCipher(r.plainTextDEK)
   102  	if err != nil {
   103  		return nil, fmt.Errorf("failed to initialize AES Cipher: %v", err)
   104  	}
   105  	// etcd path of the key is used as the authenticated context - need to pass it to decrypt
   106  	ctx := context.Background()
   107  	dataCtx := value.DefaultContext([]byte(secretETCDPath))
   108  	aesgcmTransformer, err := aestransformer.NewGCMTransformer(block)
   109  	if err != nil {
   110  		return nil, fmt.Errorf("failed to create transformer from block: %v", err)
   111  	}
   112  	plainSecret, _, err := aesgcmTransformer.TransformFromStorage(ctx, r.cipherTextPayload(), dataCtx)
   113  	if err != nil {
   114  		return nil, fmt.Errorf("failed to transform from storage via AESGCM, err: %w", err)
   115  	}
   116  
   117  	return plainSecret, nil
   118  }
   119  
   120  // TestKMSProvider is an integration test between KubeAPI, ETCD and KMS Plugin
   121  // Concretely, this test verifies the following integration contracts:
   122  // 1. Raw records in ETCD that were processed by KMS Provider should be prefixed with k8s:enc:kms:v1:grpc-kms-provider-name:
   123  // 2. Data Encryption Key (DEK) should be generated by envelopeTransformer and passed to KMS gRPC Plugin
   124  // 3. KMS gRPC Plugin should encrypt the DEK with a Key Encryption Key (KEK) and pass it back to envelopeTransformer
   125  // 4. The cipherTextPayload (ex. Secret) should be encrypted via AES CBC transform
   126  // 5. Prefix-EncryptedDEK-EncryptedPayload structure should be deposited to ETCD
   127  // 6. Direct AES GCM decryption of the cipherTextPayload written with AES CBC transform does not work
   128  // 7. Existing AES CBC secrets should be un-enveloped on direct reads from Kube API Server
   129  // 8. No-op updates to the secret should cause new AES GCM key to be used
   130  // 9. Direct AES GCM decryption works after the new AES GCM key is used
   131  func TestKMSProvider(t *testing.T) {
   132  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
   133  
   134  	encryptionConfig := `
   135  kind: EncryptionConfiguration
   136  apiVersion: apiserver.config.k8s.io/v1
   137  resources:
   138    - resources:
   139      - secrets
   140      providers:
   141      - kms:
   142         name: kms-provider
   143         cachesize: 1000
   144         endpoint: unix:///@kms-provider.sock
   145  `
   146  	providerName := "kms-provider"
   147  	pluginMock := mock.NewBase64Plugin(t, "@kms-provider.sock")
   148  	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
   149  	if err != nil {
   150  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   151  	}
   152  	defer test.cleanUp()
   153  
   154  	test.secret, err = test.createSecret(testSecret, testNamespace)
   155  	if err != nil {
   156  		t.Fatalf("Failed to create test secret, error: %v", err)
   157  	}
   158  
   159  	// Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it.
   160  	plainTextDEK := pluginMock.LastEncryptRequest()
   161  
   162  	secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace)
   163  	rawEnvelope, err := test.getRawSecretFromETCD()
   164  	if err != nil {
   165  		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
   166  	}
   167  	envelopeData := envelope{
   168  		providerName: providerName,
   169  		rawEnvelope:  rawEnvelope,
   170  		plainTextDEK: plainTextDEK,
   171  	}
   172  
   173  	wantPrefix := "k8s:enc:kms:v1:kms-provider:"
   174  	if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) {
   175  		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope)
   176  	}
   177  
   178  	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   179  	defer cancel()
   180  	decryptResponse, err := pluginMock.Decrypt(ctx, &kmsapi.DecryptRequest{Version: kmsAPIVersion, Cipher: envelopeData.cipherTextDEK()})
   181  	if err != nil {
   182  		t.Fatalf("failed to decrypt DEK, %v", err)
   183  	}
   184  	dekPlainAsWouldBeSeenByETCD := decryptResponse.Plain
   185  
   186  	if !bytes.Equal(plainTextDEK, dekPlainAsWouldBeSeenByETCD) {
   187  		t.Fatalf("expected plainTextDEK %v to be passed to KMS Plugin, but got %s",
   188  			plainTextDEK, dekPlainAsWouldBeSeenByETCD)
   189  	}
   190  
   191  	plainSecret, err := envelopeData.plainTextPayload(secretETCDPath)
   192  	if err != nil {
   193  		t.Fatalf("failed to transform from storage via AESCBC, err: %v", err)
   194  	}
   195  
   196  	if !strings.Contains(string(plainSecret), secretVal) {
   197  		t.Fatalf("expected %q after decryption, but got %q", secretVal, string(plainSecret))
   198  	}
   199  
   200  	secretClient := test.restClient.CoreV1().Secrets(testNamespace)
   201  	// Secrets should be un-enveloped on direct reads from Kube API Server.
   202  	s, err := secretClient.Get(ctx, testSecret, metav1.GetOptions{})
   203  	if err != nil {
   204  		t.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err)
   205  	}
   206  	if secretVal != string(s.Data[secretKey]) {
   207  		t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
   208  	}
   209  
   210  	// write data using AES CBC to simulate a downgrade
   211  	oldSecretBytes, err := base64.StdEncoding.DecodeString(oldSecret)
   212  	if err != nil {
   213  		t.Fatalf("failed to base64 decode old secret, err: %v", err)
   214  	}
   215  	oldKeyBytes, err := base64.StdEncoding.DecodeString(oldAESCBCKey)
   216  	if err != nil {
   217  		t.Fatalf("failed to base64 decode old key, err: %v", err)
   218  	}
   219  	block, err := aes.NewCipher(oldKeyBytes)
   220  	if err != nil {
   221  		t.Fatalf("invalid key, err: %v", err)
   222  	}
   223  
   224  	oldEncryptedSecretBytes, err := aestransformer.NewCBCTransformer(block).TransformToStorage(ctx, oldSecretBytes, value.DefaultContext(secretETCDPath))
   225  	if err != nil {
   226  		t.Fatalf("failed to encrypt old secret, err: %v", err)
   227  	}
   228  
   229  	oldEncryptedSecretBuf := cryptobyte.NewBuilder(nil)
   230  	oldEncryptedSecretBuf.AddBytes([]byte(wantPrefix))
   231  	oldEncryptedSecretBuf.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
   232  		b.AddBytes([]byte(oldAESCBCKey))
   233  	})
   234  	oldEncryptedSecretBuf.AddBytes(oldEncryptedSecretBytes)
   235  
   236  	_, err = test.writeRawRecordToETCD(secretETCDPath, oldEncryptedSecretBuf.BytesOrPanic())
   237  	if err != nil {
   238  		t.Fatalf("failed to write old encrypted secret, err: %v", err)
   239  	}
   240  
   241  	// confirm that direct AES GCM decryption does not work
   242  	failingRawEnvelope, err := test.getRawSecretFromETCD()
   243  	if err != nil {
   244  		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
   245  	}
   246  	failingOldEnvelope := envelope{
   247  		providerName: providerName,
   248  		rawEnvelope:  failingRawEnvelope,
   249  		plainTextDEK: oldKeyBytes,
   250  	}
   251  	failingOldPlainSecret, err := failingOldEnvelope.plainTextPayload(secretETCDPath)
   252  	if err == nil {
   253  		t.Fatalf("AESGCM decryption failure not seen, data: %s", string(failingOldPlainSecret))
   254  	}
   255  
   256  	// Existing AES CBC secrets should be un-enveloped on direct reads from Kube API Server.
   257  	oldSecretObj, err := secretClient.Get(ctx, testSecret, metav1.GetOptions{})
   258  	if err != nil {
   259  		t.Fatalf("failed to read old secret via Kube API, err: %v", err)
   260  	}
   261  	if oldSecretVal != string(oldSecretObj.Data[secretKey]) {
   262  		t.Fatalf("expected %s from KubeAPI, but got %s", oldSecretVal, string(oldSecretObj.Data[secretKey]))
   263  	}
   264  
   265  	// no-op update should cause new AES GCM key to be used
   266  	oldSecretUpdated, err := secretClient.Update(ctx, oldSecretObj, metav1.UpdateOptions{})
   267  	if err != nil {
   268  		t.Fatalf("failed to update old secret via Kube API, err: %v", err)
   269  	}
   270  	if oldSecretObj.ResourceVersion == oldSecretUpdated.ResourceVersion {
   271  		t.Fatalf("old secret not updated on no-op write: %s", oldSecretObj.ResourceVersion)
   272  	}
   273  
   274  	// confirm that direct AES GCM decryption works
   275  	oldRawEnvelope, err := test.getRawSecretFromETCD()
   276  	if err != nil {
   277  		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
   278  	}
   279  	oldEnvelope := envelope{
   280  		providerName: providerName,
   281  		rawEnvelope:  oldRawEnvelope,
   282  		plainTextDEK: pluginMock.LastEncryptRequest(),
   283  	}
   284  	if !bytes.HasPrefix(oldRawEnvelope, []byte(wantPrefix)) {
   285  		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, oldRawEnvelope)
   286  	}
   287  	oldPlainSecret, err := oldEnvelope.plainTextPayload(secretETCDPath)
   288  	if err != nil {
   289  		t.Fatalf("failed to transform from storage via AESGCM, err: %v", err)
   290  	}
   291  	if !strings.Contains(string(oldPlainSecret), oldSecretVal) {
   292  		t.Fatalf("expected %q after decryption, but got %q", oldSecretVal, string(oldPlainSecret))
   293  	}
   294  }
   295  
   296  // TestECHotReload is an integration test that verifies hot reload of KMS encryption config works.
   297  // This test asserts following scenarios:
   298  // 1. start at 'kms-provider'
   299  // 2. create some secrets
   300  // 3. add 'new-kms-provider' as write KMS (this is okay because we only have 1 API server)
   301  // 4. wait for config to be observed
   302  // 5. run storage migration on secrets
   303  // 6. confirm that secrets have the new prefix
   304  // 7. remove 'kms-provider'
   305  // 8. wait for config to be observed
   306  // 9. confirm that reads still work
   307  // 10. confirm that cluster wide secret read still works
   308  // 11. confirm that api server can restart with last applied encryption config
   309  func TestEncryptionConfigHotReload(t *testing.T) {
   310  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
   311  
   312  	// this makes the test super responsive. It's set to a default of 1 minute.
   313  	encryptionconfigcontroller.EncryptionConfigFileChangePollDuration = time.Second
   314  
   315  	storageConfig := framework.SharedEtcd()
   316  	encryptionConfig := `
   317  kind: EncryptionConfiguration
   318  apiVersion: apiserver.config.k8s.io/v1
   319  resources:
   320    - resources:
   321      - secrets
   322      providers:
   323      - kms:
   324         name: kms-provider
   325         cachesize: 1000
   326         endpoint: unix:///@kms-provider.sock
   327  `
   328  
   329  	genericapiserver.SetHostnameFuncForTests("testAPIServerID")
   330  	_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
   331  	var restarted bool
   332  	test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig)
   333  	if err != nil {
   334  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   335  	}
   336  	defer func() {
   337  		if !restarted {
   338  			test.cleanUp()
   339  		}
   340  	}()
   341  	ctx := testContext(t)
   342  
   343  	// the global metrics registry persists across test runs - reset it here so we can make assertions
   344  	copyConfig := rest.CopyConfig(test.kubeAPIServer.ClientConfig)
   345  	copyConfig.GroupVersion = &schema.GroupVersion{}
   346  	copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
   347  	rc, err := rest.RESTClientFor(copyConfig)
   348  	if err != nil {
   349  		t.Fatal(err)
   350  	}
   351  	if err := rc.Delete().AbsPath("/metrics").Do(ctx).Error(); err != nil {
   352  		t.Fatal(err)
   353  	}
   354  
   355  	// assert that the metrics we collect during the test run match expectations
   356  	// NOTE: 2 successful automatic reload resulted from 2 config file updates
   357  	wantMetricStrings := []string{
   358  		`apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
   359  		`apiserver_encryption_config_controller_automatic_reload_success_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795"} 2`,
   360  		`apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 2`,
   361  	}
   362  
   363  	test.secret, err = test.createSecret(testSecret, testNamespace)
   364  	if err != nil {
   365  		t.Fatalf("Failed to create test secret, error: %v", err)
   366  	}
   367  
   368  	// create a new secret in default namespace. This is to assert cluster wide read works after hot reload.
   369  	_, err = test.createSecret(fmt.Sprintf("%s-%s", testSecret, "1"), "default")
   370  	if err != nil {
   371  		t.Fatalf("Failed to create test secret in default namespace, error: %v", err)
   372  	}
   373  
   374  	_, err = test.createConfigMap(testConfigmap, testNamespace)
   375  	if err != nil {
   376  		t.Fatalf("Failed to create test configmap, error: %v", err)
   377  	}
   378  
   379  	// test if hot reload controller is healthy
   380  	mustBeHealthy(t, "/poststarthook/start-encryption-provider-config-automatic-reload", "ok", test.kubeAPIServer.ClientConfig)
   381  
   382  	encryptionConfigWithNewProvider := `
   383  kind: EncryptionConfiguration
   384  apiVersion: apiserver.config.k8s.io/v1
   385  resources:
   386    - resources:
   387      - secrets
   388      providers:
   389      - kms:
   390         name: new-kms-provider-for-secrets
   391         cachesize: 1000
   392         endpoint: unix:///@new-kms-provider.sock
   393      - kms:
   394         name: kms-provider
   395         cachesize: 1000
   396         endpoint: unix:///@kms-provider.sock
   397    - resources:
   398      - configmaps
   399      providers:
   400      - kms:
   401         name: new-kms-provider-for-configmaps
   402         cachesize: 1000
   403         endpoint: unix:///@new-kms-provider.sock
   404      - identity: {}
   405  `
   406  	// start new KMS Plugin
   407  	_ = mock.NewBase64Plugin(t, "@new-kms-provider.sock")
   408  	// update encryption config
   409  	updateFile(t, test.configDir, encryptionConfigFileName, []byte(encryptionConfigWithNewProvider))
   410  
   411  	wantPrefixForSecrets := "k8s:enc:kms:v1:new-kms-provider-for-secrets:"
   412  
   413  	// implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod.
   414  	// wait for config to be observed
   415  	verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, "", test)
   416  
   417  	// run storage migration
   418  	// get secrets
   419  
   420  	secretsList, err := test.restClient.CoreV1().Secrets("").List(
   421  		ctx,
   422  		metav1.ListOptions{},
   423  	)
   424  	if err != nil {
   425  		t.Fatalf("failed to list secrets, err: %v", err)
   426  	}
   427  
   428  	for _, secret := range secretsList.Items {
   429  		// update secret
   430  		_, err = test.restClient.CoreV1().Secrets(secret.Namespace).Update(
   431  			ctx,
   432  			&secret,
   433  			metav1.UpdateOptions{},
   434  		)
   435  		if err != nil {
   436  			t.Fatalf("failed to update secret, err: %v", err)
   437  		}
   438  	}
   439  
   440  	// get configmaps
   441  	configmapsList, err := test.restClient.CoreV1().ConfigMaps("").List(
   442  		ctx,
   443  		metav1.ListOptions{},
   444  	)
   445  	if err != nil {
   446  		t.Fatalf("failed to list configmaps, err: %v", err)
   447  	}
   448  
   449  	for _, configmap := range configmapsList.Items {
   450  		// update configmap
   451  		_, err = test.restClient.CoreV1().ConfigMaps(configmap.Namespace).Update(
   452  			ctx,
   453  			&configmap,
   454  			metav1.UpdateOptions{},
   455  		)
   456  		if err != nil {
   457  			t.Fatalf("failed to update configmap, err: %v", err)
   458  		}
   459  	}
   460  
   461  	// assert that resources has new prefix
   462  	secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace)
   463  	rawEnvelope, err := test.getRawSecretFromETCD()
   464  	if err != nil {
   465  		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
   466  	}
   467  
   468  	// assert secret
   469  	if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefixForSecrets)) {
   470  		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefixForSecrets, rawEnvelope)
   471  	}
   472  
   473  	rawConfigmapEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace))
   474  	if err != nil {
   475  		t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace), err)
   476  	}
   477  
   478  	// assert prefix for configmap
   479  	wantPrefixForConfigmaps := "k8s:enc:kms:v1:new-kms-provider-for-configmaps:"
   480  	if !bytes.HasPrefix(rawConfigmapEnvelope.Kvs[0].Value, []byte(wantPrefixForConfigmaps)) {
   481  		t.Fatalf("expected configmap to be prefixed with %s, but got %s", wantPrefixForConfigmaps, rawConfigmapEnvelope.Kvs[0].Value)
   482  	}
   483  
   484  	// remove old KMS provider
   485  	// verifyIfKMSTransformersSwapped sometimes passes even before the changes in the encryption config file are observed.
   486  	// this causes the metrics tests to fail, which validate two config changes.
   487  	// this may happen when an existing KMS provider is already running (e.g., new-kms-provider-for-secrets in this case).
   488  	// to ensure that the changes are observed, we added one more provider (kms-provider-to-encrypt-all) and are validating it in verifyIfKMSTransformersSwapped.
   489  	encryptionConfigWithoutOldProvider := `
   490  kind: EncryptionConfiguration
   491  apiVersion: apiserver.config.k8s.io/v1
   492  resources:
   493    - resources:
   494      - secrets
   495      providers:
   496      - kms:
   497         name: new-kms-provider-for-secrets
   498         cachesize: 1000
   499         endpoint: unix:///@new-kms-provider.sock
   500    - resources:
   501      - configmaps
   502      providers:
   503      - kms:
   504         name: new-kms-provider-for-configmaps
   505         cachesize: 1000
   506         endpoint: unix:///@new-kms-provider.sock
   507    - resources:
   508      - '*.*'
   509      providers:
   510      - kms:
   511          name: kms-provider-to-encrypt-all
   512          cachesize: 1000
   513          endpoint: unix:///@new-encrypt-all-kms-provider.sock
   514      - identity: {}
   515  `
   516  	// start new KMS Plugin
   517  	_ = mock.NewBase64Plugin(t, "@new-encrypt-all-kms-provider.sock")
   518  
   519  	// update encryption config and wait for hot reload
   520  	updateFile(t, test.configDir, encryptionConfigFileName, []byte(encryptionConfigWithoutOldProvider))
   521  
   522  	wantPrefixForEncryptAll := "k8s:enc:kms:v1:kms-provider-to-encrypt-all:"
   523  
   524  	// wait for config to be observed
   525  	verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, wantPrefixForEncryptAll, test)
   526  
   527  	// confirm that reading secrets still works
   528  	_, err = test.restClient.CoreV1().Secrets(testNamespace).Get(
   529  		ctx,
   530  		testSecret,
   531  		metav1.GetOptions{},
   532  	)
   533  	if err != nil {
   534  		t.Fatalf("failed to read secret, err: %v", err)
   535  	}
   536  
   537  	// make sure cluster wide secrets read still works
   538  	_, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{})
   539  	if err != nil {
   540  		t.Fatalf("failed to list secrets, err: %v", err)
   541  	}
   542  
   543  	// make sure cluster wide configmaps read still works
   544  	_, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{})
   545  	if err != nil {
   546  		t.Fatalf("failed to list configmaps, err: %v", err)
   547  	}
   548  
   549  	// restart kube-apiserver with last applied encryption config and assert that server can start
   550  	previousConfigDir := test.configDir
   551  	test.shutdownAPIServer()
   552  	restarted = true
   553  	test, err = newTransformTest(t, test.transformerConfig, true, previousConfigDir, storageConfig)
   554  	if err != nil {
   555  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   556  	}
   557  	defer test.cleanUp()
   558  
   559  	_, err = test.restClient.CoreV1().Secrets(testNamespace).Get(
   560  		ctx,
   561  		testSecret,
   562  		metav1.GetOptions{},
   563  	)
   564  	if err != nil {
   565  		t.Fatalf("failed to read secret, err: %v", err)
   566  	}
   567  
   568  	// confirm that reading cluster wide secrets still works after restart
   569  	if _, err = test.restClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{}); err != nil {
   570  		t.Fatalf("failed to list secrets, err: %v", err)
   571  	}
   572  
   573  	// make sure cluster wide configmaps read still works
   574  	if _, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{}); err != nil {
   575  		t.Fatalf("failed to list configmaps, err: %v", err)
   576  	}
   577  
   578  	// recreate rest client with the new transformTest
   579  	copyConfig = rest.CopyConfig(test.kubeAPIServer.ClientConfig)
   580  	copyConfig.GroupVersion = &schema.GroupVersion{}
   581  	copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
   582  	rc, err = rest.RESTClientFor(copyConfig)
   583  	if err != nil {
   584  		t.Fatal(err)
   585  	}
   586  	defer func() {
   587  		body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
   588  		if err != nil {
   589  			t.Fatal(err)
   590  		}
   591  		var gotMetricStrings []string
   592  		trimFP := regexp.MustCompile(`(.*)(} \d+\.\d+.*)`)
   593  		for _, line := range strings.Split(string(body), "\n") {
   594  			if strings.HasPrefix(line, "apiserver_encryption_config_controller_") {
   595  				if strings.Contains(line, "_seconds") {
   596  					line = trimFP.ReplaceAllString(line, `$1`) + "} FP" // ignore floating point metric values
   597  				}
   598  				gotMetricStrings = append(gotMetricStrings, line)
   599  			}
   600  		}
   601  		if diff := cmp.Diff(wantMetricStrings, gotMetricStrings); diff != "" {
   602  			t.Errorf("unexpected metrics diff (-want +got): %s", diff)
   603  		}
   604  	}()
   605  }
   606  
   607  func TestEncryptAll(t *testing.T) {
   608  	encryptionConfig := `
   609  kind: EncryptionConfiguration
   610  apiVersion: apiserver.config.k8s.io/v1
   611  resources:
   612    - resources:
   613      - '*.*'
   614      providers:
   615      - kms:
   616          name: encrypt-all-kms-provider
   617          cachesize: 1000
   618          endpoint: unix:///@encrypt-all-kms-provider.sock
   619  `
   620  
   621  	t.Run("encrypt all resources", func(t *testing.T) {
   622  		_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
   623  		// To ensure we are checking all REST resources
   624  		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllAlpha", true)()
   625  		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllBeta", true)()
   626  		// Need to enable this explicitly as the feature is deprecated
   627  		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
   628  
   629  		test, err := newTransformTest(t, encryptionConfig, false, "", nil)
   630  		if err != nil {
   631  			t.Fatalf("failed to start KUBE API Server with encryptionConfig")
   632  		}
   633  		defer test.cleanUp()
   634  
   635  		etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(test.kubeAPIServer.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...)
   636  
   637  		_, serverResources, err := test.restClient.Discovery().ServerGroupsAndResources()
   638  		if err != nil {
   639  			t.Fatal(err)
   640  		}
   641  		resources := etcd.GetResources(t, serverResources)
   642  		client := dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
   643  
   644  		etcdStorageData := etcd.GetEtcdStorageDataForNamespace(testNamespace)
   645  		restResourceSet := sets.New[schema.GroupVersionResource]()
   646  		stubResourceSet := sets.New[schema.GroupVersionResource]()
   647  		for _, resource := range resources {
   648  			gvr := resource.Mapping.Resource
   649  			stub := etcdStorageData[gvr].Stub
   650  
   651  			// continue if stub is empty
   652  			if stub == "" {
   653  				t.Errorf("skipping resource %s because stub is empty", gvr)
   654  				continue
   655  			}
   656  			restResourceSet.Insert(gvr)
   657  			dynamicClient, obj, err := etcd.JSONToUnstructured(stub, testNamespace, &meta.RESTMapping{
   658  				Resource:         gvr,
   659  				GroupVersionKind: gvr.GroupVersion().WithKind(resource.Mapping.GroupVersionKind.Kind),
   660  				Scope:            resource.Mapping.Scope,
   661  			}, client)
   662  			if err != nil {
   663  				t.Fatal(err)
   664  			}
   665  
   666  			_, err = dynamicClient.Create(context.TODO(), obj, metav1.CreateOptions{})
   667  			if err != nil {
   668  				t.Fatal(err)
   669  			}
   670  		}
   671  		for gvr, data := range etcdStorageData {
   672  			if data.Stub == "" {
   673  				continue
   674  			}
   675  			stubResourceSet.Insert(gvr)
   676  		}
   677  		if !restResourceSet.Equal(stubResourceSet) {
   678  			t.Errorf("failed to check all REST resources: %q", restResourceSet.SymmetricDifference(stubResourceSet).UnsortedList())
   679  		}
   680  		rawClient, etcdClient, err := integration.GetEtcdClients(test.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
   681  		if err != nil {
   682  			t.Fatalf("failed to create etcd client: %v", err)
   683  		}
   684  		// kvClient is a wrapper around rawClient and to avoid leaking goroutines we need to
   685  		// close the client (which we can do by closing rawClient).
   686  		defer rawClient.Close()
   687  
   688  		response, err := etcdClient.Get(context.TODO(), "/"+test.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Prefix, clientv3.WithPrefix())
   689  		if err != nil {
   690  			t.Fatalf("failed to retrieve secret from etcd %v", err)
   691  		}
   692  
   693  		// assert that total key values in response in greater than 0
   694  		if len(response.Kvs) == 0 {
   695  			t.Fatalf("expected total number of keys to be greater than 0, but got %d", len(response.Kvs))
   696  		}
   697  
   698  		// assert that total response keys are greater or equal to total resources
   699  		if len(response.Kvs) < len(resources) {
   700  			t.Fatalf("expected total number of keys to be greater or equal to total resources, but got %d", len(response.Kvs))
   701  		}
   702  
   703  		wantPrefix := "k8s:enc:kms:v1:encrypt-all-kms-provider:"
   704  		for _, kv := range response.Kvs {
   705  			// the following resources are not encrypted as they are not REST APIs and hence are not expected
   706  			// to be encrypted because it would be impossible to perform a storage migration on them
   707  			if strings.Contains(kv.String(), "masterleases") ||
   708  				strings.Contains(kv.String(), "peerserverleases") ||
   709  				strings.Contains(kv.String(), "serviceips") ||
   710  				strings.Contains(kv.String(), "servicenodeports") {
   711  				// assert that these resources are not encrypted with any provider
   712  				if bytes.HasPrefix(kv.Value, []byte("k8s:enc:")) {
   713  					t.Errorf("expected resource %s to not be prefixed with %s, but got %s", kv.Key, "k8s:enc:", kv.Value)
   714  				}
   715  				continue
   716  			}
   717  
   718  			// assert that all other resources are encrypted
   719  			if !bytes.HasPrefix(kv.Value, []byte(wantPrefix)) {
   720  				t.Errorf("expected resource %s to be prefixed with %s, but got %s", kv.Key, wantPrefix, kv.Value)
   721  			}
   722  		}
   723  	})
   724  }
   725  
   726  func TestEncryptAllWithWildcard(t *testing.T) {
   727  	encryptionConfig := `
   728  kind: EncryptionConfiguration
   729  apiVersion: apiserver.config.k8s.io/v1
   730  resources:
   731    - resources:
   732      - configmaps
   733      providers:
   734      - identity: {}
   735    - resources:
   736      - '*.batch'
   737      providers:
   738      - kms:
   739          name: kms-provider
   740          cachesize: 1000
   741          endpoint: unix:///@kms-provider.sock
   742    - resources:
   743      - '*.*'
   744      providers:
   745      - kms:
   746          name: encrypt-all-kms-provider
   747          cachesize: 1000
   748          endpoint: unix:///@encrypt-all-kms-provider.sock
   749  `
   750  	_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
   751  	_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
   752  
   753  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
   754  
   755  	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
   756  	if err != nil {
   757  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   758  	}
   759  	defer test.cleanUp()
   760  
   761  	wantPrefix := "k8s:enc:kms:v1:kms-provider:"
   762  	wantPrefixForEncryptAll := "k8s:enc:kms:v1:encrypt-all-kms-provider:"
   763  
   764  	_, err = test.createJob("test-job", "default")
   765  	if err != nil {
   766  		t.Fatalf("failed to create job: %v", err)
   767  	}
   768  
   769  	rawJobsEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "jobs", "test-job", "default"))
   770  	if err != nil {
   771  		t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "jobs", "test-job", "default"), err)
   772  	}
   773  
   774  	// assert prefix for jobs
   775  	if !bytes.HasPrefix(rawJobsEnvelope.Kvs[0].Value, []byte(wantPrefix)) {
   776  		t.Fatalf("expected jobs to be prefixed with %s, but got %s", wantPrefix, rawJobsEnvelope.Kvs[0].Value)
   777  	}
   778  
   779  	_, err = test.createDeployment("test-deployment", "default")
   780  	if err != nil {
   781  		t.Fatalf("failed to create deployment: %v", err)
   782  	}
   783  
   784  	rawDeploymentsEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", "test-deployment", "default"))
   785  	if err != nil {
   786  		t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", "test-deployment", "default"), err)
   787  	}
   788  
   789  	// assert prefix for deployments
   790  	if !bytes.HasPrefix(rawDeploymentsEnvelope.Kvs[0].Value, []byte(wantPrefixForEncryptAll)) {
   791  		t.Fatalf("expected deployments to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawDeploymentsEnvelope.Kvs[0].Value)
   792  	}
   793  
   794  	test.secret, err = test.createSecret(testSecret, testNamespace)
   795  	if err != nil {
   796  		t.Fatalf("Failed to create test secret, error: %v", err)
   797  	}
   798  
   799  	rawSecretEnvelope, err := test.getRawSecretFromETCD()
   800  	if err != nil {
   801  		t.Fatalf("failed to read secrets from etcd: %v", err)
   802  	}
   803  
   804  	// assert prefix for secrets
   805  	if !bytes.HasPrefix(rawSecretEnvelope, []byte(wantPrefixForEncryptAll)) {
   806  		t.Fatalf("expected secrets to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawSecretEnvelope)
   807  	}
   808  
   809  	_, err = test.createConfigMap(testConfigmap, testNamespace)
   810  	if err != nil {
   811  		t.Fatalf("Failed to create test configmap, error: %v", err)
   812  	}
   813  
   814  	rawConfigMapEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace))
   815  	if err != nil {
   816  		t.Fatalf("failed to read configmaps from etcd: %v", err)
   817  	}
   818  
   819  	// assert configmaps do not have the encrypted data prefix
   820  	if bytes.HasPrefix(rawConfigMapEnvelope.Kvs[0].Value, []byte("k8s:enc:")) {
   821  		t.Fatalf("expected configmaps to be not encrypted, got %s", rawConfigMapEnvelope.Kvs[0].Value)
   822  	}
   823  }
   824  
   825  func TestEncryptionConfigHotReloadFilePolling(t *testing.T) {
   826  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
   827  
   828  	// this makes the test super responsive. It's set to a default of 1 minute.
   829  	encryptionconfigcontroller.EncryptionConfigFileChangePollDuration = time.Second
   830  
   831  	testCases := []struct {
   832  		sleep      time.Duration
   833  		name       string
   834  		updateFile func(filePath, fileContent string) error
   835  	}{
   836  		{
   837  			name: "truncate file",
   838  			updateFile: func(filePath string, fileContent string) error {
   839  				// os.WriteFile truncates the file before writing
   840  				return os.WriteFile(filePath, []byte(fileContent), 0644)
   841  			},
   842  			// significantly longer than KMSCloseGracePeriod
   843  			sleep: 20 * time.Second,
   844  		},
   845  		{
   846  			name: "delete and create file",
   847  			updateFile: func(filePath, fileContent string) error {
   848  				// os.Remove deletes the file before creating a new one
   849  				if err := os.Remove(filePath); err != nil {
   850  					return fmt.Errorf("failed to remove encryption config, err: %w", err)
   851  				}
   852  
   853  				file, err := os.Create(filePath)
   854  				if err != nil {
   855  					return fmt.Errorf("failed to create encryption config, err: %w", err)
   856  				}
   857  				defer file.Close()
   858  
   859  				if _, err := file.Write([]byte(fileContent)); err != nil {
   860  					return fmt.Errorf("failed to write encryption config, err: %w", err)
   861  				}
   862  
   863  				return nil
   864  			},
   865  		},
   866  		{
   867  			name: "move file",
   868  			updateFile: func(filePath, fileContent string) error {
   869  				// write new config to a temp file
   870  				tmpFilePath := filePath + ".tmp"
   871  				if err := os.WriteFile(tmpFilePath, []byte(fileContent), 0644); err != nil {
   872  					return fmt.Errorf("failed to write config to tmp file, err: %w", err)
   873  				}
   874  
   875  				// move the temp file to the original file
   876  				if err := os.Rename(tmpFilePath, filePath); err != nil {
   877  					return fmt.Errorf("failed to move encryption config, err: %w", err)
   878  				}
   879  
   880  				return nil
   881  			},
   882  		},
   883  	}
   884  
   885  	for _, tc := range testCases {
   886  		t.Run(tc.name, func(t *testing.T) {
   887  			encryptionConfig := `
   888  kind: EncryptionConfiguration
   889  apiVersion: apiserver.config.k8s.io/v1
   890  resources:
   891    - resources:
   892      - secrets
   893      providers:
   894      - kms:
   895         name: kms-provider
   896         cachesize: 1000
   897         endpoint: unix:///@kms-provider.sock
   898         timeout: 1s
   899  `
   900  			_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
   901  
   902  			test, err := newTransformTest(t, encryptionConfig, true, "", nil)
   903  			if err != nil {
   904  				t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   905  			}
   906  			defer test.cleanUp()
   907  
   908  			test.secret, err = test.createSecret(testSecret, testNamespace)
   909  			if err != nil {
   910  				t.Fatalf("Failed to create test secret, error: %v", err)
   911  			}
   912  
   913  			// test if hot reload controller is healthy
   914  			mustBeHealthy(t, "/poststarthook/start-encryption-provider-config-automatic-reload", "ok", test.kubeAPIServer.ClientConfig)
   915  
   916  			encryptionConfigWithNewProvider := `
   917  kind: EncryptionConfiguration
   918  apiVersion: apiserver.config.k8s.io/v1
   919  resources:
   920    - resources:
   921      - secrets
   922      providers:
   923      - kms:
   924         name: new-kms-provider-for-secrets
   925         cachesize: 1000
   926         endpoint: unix:///@new-kms-provider.sock
   927         timeout: 1s
   928      - kms:
   929         name: kms-provider
   930         cachesize: 1000
   931         endpoint: unix:///@kms-provider.sock
   932         timeout: 1s
   933    - resources:
   934      - configmaps
   935      providers:
   936      - kms:
   937         name: new-kms-provider-for-configmaps
   938         cachesize: 1000
   939         endpoint: unix:///@new-kms-provider.sock
   940         timeout: 1s
   941      - identity: {}
   942  `
   943  			// start new KMS Plugin
   944  			_ = mock.NewBase64Plugin(t, "@new-kms-provider.sock")
   945  			// update encryption config
   946  			if err := tc.updateFile(filepath.Join(test.configDir, encryptionConfigFileName), encryptionConfigWithNewProvider); err != nil {
   947  				t.Fatalf("failed to update encryption config, err: %v", err)
   948  			}
   949  
   950  			wantPrefix := "k8s:enc:kms:v1:new-kms-provider-for-secrets:"
   951  			verifyPrefixOfSecretResource(t, wantPrefix, test)
   952  
   953  			// make sure things still work at a "later" time
   954  			if tc.sleep != 0 {
   955  				time.Sleep(tc.sleep)
   956  			}
   957  			_, err = test.createSecret(fmt.Sprintf("secret-%d", rand.Intn(100000)), "default")
   958  			if err != nil {
   959  				t.Fatalf("Failed to create test secret, error: %v", err)
   960  			}
   961  			_, err = test.restClient.CoreV1().Secrets("").List(
   962  				context.TODO(),
   963  				metav1.ListOptions{},
   964  			)
   965  			if err != nil {
   966  				t.Fatalf("failed to re-list secrets, err: %v", err)
   967  			}
   968  		})
   969  	}
   970  }
   971  
   972  func verifyPrefixOfSecretResource(t *testing.T, wantPrefix string, test *transformTest) {
   973  	// implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod.
   974  	// wait for config to be observed
   975  	verifyIfKMSTransformersSwapped(t, wantPrefix, "", test)
   976  
   977  	// run storage migration
   978  	secretsList, err := test.restClient.CoreV1().Secrets("").List(
   979  		context.TODO(),
   980  		metav1.ListOptions{},
   981  	)
   982  	if err != nil {
   983  		t.Fatalf("failed to list secrets, err: %v", err)
   984  	}
   985  
   986  	for _, secret := range secretsList.Items {
   987  		_, err = test.restClient.CoreV1().Secrets(secret.Namespace).Update(
   988  			context.TODO(),
   989  			&secret,
   990  			metav1.UpdateOptions{},
   991  		)
   992  		if err != nil {
   993  			t.Fatalf("failed to update secret, err: %v", err)
   994  		}
   995  	}
   996  
   997  	secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace)
   998  	rawEnvelope, err := test.getRawSecretFromETCD()
   999  	if err != nil {
  1000  		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
  1001  	}
  1002  
  1003  	// assert that resources has new prefix
  1004  	if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) {
  1005  		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope)
  1006  	}
  1007  }
  1008  
  1009  func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix, wantPrefixForEncryptAll string, test *transformTest) {
  1010  	t.Helper()
  1011  
  1012  	var swapErr error
  1013  	// delete and recreate same secret flakes, so create a new secret with a different index until new prefix is observed
  1014  	// generate a random int to be used in secret name
  1015  	idx := rand.Intn(100000)
  1016  
  1017  	pollErr := wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
  1018  		// create secret
  1019  		secretName := fmt.Sprintf("secret-%d", idx)
  1020  		_, err := test.createSecret(secretName, "default")
  1021  		if err != nil {
  1022  			t.Fatalf("Failed to create test secret, error: %v", err)
  1023  		}
  1024  
  1025  		rawEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", secretName, "default"))
  1026  		if err != nil {
  1027  			t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", secretName, "default"), err)
  1028  		}
  1029  
  1030  		// check prefix
  1031  		if !bytes.HasPrefix(rawEnvelope.Kvs[0].Value, []byte(wantPrefix)) {
  1032  			idx++
  1033  
  1034  			swapErr = fmt.Errorf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope.Kvs[0].Value)
  1035  
  1036  			// return nil error to continue polling till timeout
  1037  			return false, nil
  1038  		}
  1039  
  1040  		if wantPrefixForEncryptAll != "" {
  1041  			deploymentName := fmt.Sprintf("deployment-%d", idx)
  1042  			_, err := test.createDeployment(deploymentName, "default")
  1043  			if err != nil {
  1044  				t.Fatalf("Failed to create test secret, error: %v", err)
  1045  			}
  1046  
  1047  			rawEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default"))
  1048  			if err != nil {
  1049  				t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default"), err)
  1050  			}
  1051  
  1052  			// check prefix
  1053  			if !bytes.HasPrefix(rawEnvelope.Kvs[0].Value, []byte(wantPrefixForEncryptAll)) {
  1054  				idx++
  1055  
  1056  				swapErr = fmt.Errorf("expected deployment to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawEnvelope.Kvs[0].Value)
  1057  
  1058  				// return nil error to continue polling till timeout
  1059  				return false, nil
  1060  			}
  1061  		}
  1062  
  1063  		return true, nil
  1064  	})
  1065  	if pollErr == wait.ErrWaitTimeout {
  1066  		t.Fatalf("failed to verify if kms transformers swapped, err: %v", swapErr)
  1067  	}
  1068  }
  1069  
  1070  func updateFile(t *testing.T, configDir, filename string, newContent []byte) {
  1071  	t.Helper()
  1072  
  1073  	// Create a temporary file
  1074  	tempFile, err := os.CreateTemp(configDir, "tempfile")
  1075  	if err != nil {
  1076  		t.Fatal(err)
  1077  	}
  1078  	defer tempFile.Close()
  1079  
  1080  	// Write the new content to the temporary file
  1081  	_, err = tempFile.Write(newContent)
  1082  	if err != nil {
  1083  		t.Fatal(err)
  1084  	}
  1085  
  1086  	// Atomically replace the original file with the temporary file
  1087  	err = os.Rename(tempFile.Name(), filepath.Join(configDir, filename))
  1088  	if err != nil {
  1089  		t.Fatal(err)
  1090  	}
  1091  }
  1092  
  1093  func TestKMSHealthz(t *testing.T) {
  1094  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
  1095  
  1096  	encryptionConfig := `
  1097  kind: EncryptionConfiguration
  1098  apiVersion: apiserver.config.k8s.io/v1
  1099  resources:
  1100    - resources:
  1101      - secrets
  1102      providers:
  1103      - kms:
  1104         name: provider-1
  1105         endpoint: unix:///@kms-provider-1.sock
  1106      - kms:
  1107         name: provider-2
  1108         endpoint: unix:///@kms-provider-2.sock
  1109  `
  1110  
  1111  	pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock")
  1112  	pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock")
  1113  
  1114  	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
  1115  	if err != nil {
  1116  		t.Fatalf("failed to start kube-apiserver, error: %v", err)
  1117  	}
  1118  	defer test.cleanUp()
  1119  
  1120  	// Name of the healthz check is always "kms-provider-0" and it covers all kms plugins.
  1121  
  1122  	// Stage 1 - Since all kms-plugins are guaranteed to be up, healthz checks for:
  1123  	// healthz/kms-provider-0 and /healthz/kms-provider-1 should be OK.
  1124  	mustBeHealthy(t, "/kms-provider-0", "ok", test.kubeAPIServer.ClientConfig)
  1125  	mustBeHealthy(t, "/kms-provider-1", "ok", test.kubeAPIServer.ClientConfig)
  1126  	mustNotHaveLivez(t, "/kms-provider-0", "404 page not found", test.kubeAPIServer.ClientConfig)
  1127  	mustNotHaveLivez(t, "/kms-provider-1", "404 page not found", test.kubeAPIServer.ClientConfig)
  1128  
  1129  	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the healthz check
  1130  	// to fail and report that provider-1 is down
  1131  	pluginMock1.EnterFailedState()
  1132  	mustBeUnHealthy(t, "/kms-provider-0",
  1133  		"internal server error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
  1134  		test.kubeAPIServer.ClientConfig)
  1135  
  1136  	mustNotHaveLivez(t, "/kms-provider-0", "404 page not found", test.kubeAPIServer.ClientConfig)
  1137  	mustBeHealthy(t, "/kms-provider-1", "ok", test.kubeAPIServer.ClientConfig)
  1138  	mustNotHaveLivez(t, "/kms-provider-1", "404 page not found", test.kubeAPIServer.ClientConfig)
  1139  	pluginMock1.ExitFailedState()
  1140  
  1141  	// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
  1142  	// to succeed now, but provider-2 is now down.
  1143  	pluginMock2.EnterFailedState()
  1144  	mustBeHealthy(t, "/kms-provider-0", "ok", test.kubeAPIServer.ClientConfig)
  1145  	mustBeUnHealthy(t, "/kms-provider-1",
  1146  		"internal server error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
  1147  		test.kubeAPIServer.ClientConfig)
  1148  	pluginMock2.ExitFailedState()
  1149  
  1150  	// Stage 4 - All kms-plugins are once again up,
  1151  	// the healthz check should be OK.
  1152  	mustBeHealthy(t, "/kms-provider-0", "ok", test.kubeAPIServer.ClientConfig)
  1153  	mustBeHealthy(t, "/kms-provider-1", "ok", test.kubeAPIServer.ClientConfig)
  1154  }
  1155  
  1156  func TestKMSHealthzWithReload(t *testing.T) {
  1157  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
  1158  
  1159  	encryptionConfig := `
  1160  kind: EncryptionConfiguration
  1161  apiVersion: apiserver.config.k8s.io/v1
  1162  resources:
  1163    - resources:
  1164      - secrets
  1165      providers:
  1166      - kms:
  1167         name: provider-1
  1168         endpoint: unix:///@kms-provider-1.sock
  1169      - kms:
  1170         name: provider-2
  1171         endpoint: unix:///@kms-provider-2.sock
  1172  `
  1173  
  1174  	pluginMock1 := mock.NewBase64Plugin(t, "@kms-provider-1.sock")
  1175  	pluginMock2 := mock.NewBase64Plugin(t, "@kms-provider-2.sock")
  1176  
  1177  	test, err := newTransformTest(t, encryptionConfig, true, "", nil)
  1178  	if err != nil {
  1179  		t.Fatalf("Failed to start kube-apiserver, error: %v", err)
  1180  	}
  1181  	defer test.cleanUp()
  1182  
  1183  	// Name of the healthz check is always "kms-provider-0" and it covers all kms plugins.
  1184  
  1185  	// Stage 1 - Since all kms-plugins are guaranteed to be up,
  1186  	// the healthz check should be OK.
  1187  	mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
  1188  
  1189  	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the healthz check
  1190  	// to fail and report that provider-1 is down
  1191  	pluginMock1.EnterFailedState()
  1192  	mustBeUnHealthy(t, "/kms-providers",
  1193  		"internal server error: kms-provider-0: failed to perform encrypt section of the healthz check for KMS Provider provider-1, error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
  1194  		test.kubeAPIServer.ClientConfig)
  1195  	pluginMock1.ExitFailedState()
  1196  
  1197  	// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
  1198  	// to succeed now, but provider-2 is now down.
  1199  	pluginMock2.EnterFailedState()
  1200  	mustBeUnHealthy(t, "/kms-providers",
  1201  		"internal server error: kms-provider-1: failed to perform encrypt section of the healthz check for KMS Provider provider-2, error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
  1202  		test.kubeAPIServer.ClientConfig)
  1203  	pluginMock2.ExitFailedState()
  1204  
  1205  	// Stage 4 - All kms-plugins are once again up,
  1206  	// the healthz check should be OK.
  1207  	mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
  1208  }
  1209  

View as plain text