Source file src/k8s.io/kubernetes/test/integration/controlplane/transformation/kmsv2_transformation_test.go

Documentation: k8s.io/kubernetes/test/integration/controlplane/transformation

     1  //go:build !windows
     2  // +build !windows
     3  
     4  /*
     5  Copyright 2022 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package transformation
    21  
    22  import (
    23  	"bytes"
    24  	"context"
    25  	"crypto/aes"
    26  	"crypto/cipher"
    27  	"encoding/binary"
    28  	"fmt"
    29  	"io"
    30  	"path"
    31  	"regexp"
    32  	"strings"
    33  	"testing"
    34  	"time"
    35  
    36  	"github.com/gogo/protobuf/proto"
    37  	"github.com/google/go-cmp/cmp"
    38  	"github.com/google/go-cmp/cmp/cmpopts"
    39  	clientv3 "go.etcd.io/etcd/client/v3"
    40  
    41  	corev1 "k8s.io/api/core/v1"
    42  	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    43  	"k8s.io/apimachinery/pkg/api/meta"
    44  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    45  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
    46  	"k8s.io/apimachinery/pkg/runtime"
    47  	"k8s.io/apimachinery/pkg/runtime/schema"
    48  	utilrand "k8s.io/apimachinery/pkg/util/rand"
    49  	"k8s.io/apimachinery/pkg/util/sets"
    50  	"k8s.io/apimachinery/pkg/util/uuid"
    51  	"k8s.io/apimachinery/pkg/util/wait"
    52  	"k8s.io/apiserver/pkg/endpoints/request"
    53  	"k8s.io/apiserver/pkg/features"
    54  	"k8s.io/apiserver/pkg/registry/generic"
    55  	genericapiserver "k8s.io/apiserver/pkg/server"
    56  	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
    57  	"k8s.io/apiserver/pkg/storage/storagebackend"
    58  	"k8s.io/apiserver/pkg/storage/value"
    59  	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
    60  	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
    61  	kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
    62  	kmsv2mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v2"
    63  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    64  	"k8s.io/client-go/dynamic"
    65  	"k8s.io/client-go/kubernetes"
    66  	"k8s.io/client-go/rest"
    67  	"k8s.io/klog/v2"
    68  	kmsv2api "k8s.io/kms/apis/v2"
    69  	kmsv2svc "k8s.io/kms/pkg/service"
    70  	"k8s.io/kubernetes/pkg/api/legacyscheme"
    71  	api "k8s.io/kubernetes/pkg/apis/core"
    72  	"k8s.io/kubernetes/pkg/controlplane"
    73  	"k8s.io/kubernetes/pkg/kubeapiserver"
    74  	secretstore "k8s.io/kubernetes/pkg/registry/core/secret/storage"
    75  	"k8s.io/kubernetes/test/integration"
    76  	"k8s.io/kubernetes/test/integration/etcd"
    77  	"k8s.io/kubernetes/test/integration/framework"
    78  )
    79  
    80  type envelopekmsv2 struct {
    81  	providerName       string
    82  	rawEnvelope        []byte
    83  	plainTextDEKSource []byte
    84  	useSeed            bool
    85  }
    86  
    87  func (r envelopekmsv2) prefix() string {
    88  	return fmt.Sprintf("k8s:enc:kms:v2:%s:", r.providerName)
    89  }
    90  
    91  func (r envelopekmsv2) prefixLen() int {
    92  	return len(r.prefix())
    93  }
    94  
    95  func (r envelopekmsv2) cipherTextDEKSource() ([]byte, error) {
    96  	o := &kmstypes.EncryptedObject{}
    97  	if err := proto.Unmarshal(r.rawEnvelope[r.startOfPayload(r.providerName):], o); err != nil {
    98  		return nil, err
    99  	}
   100  
   101  	if err := kmsv2.ValidateEncryptedObject(o); err != nil {
   102  		return nil, err
   103  	}
   104  
   105  	if r.useSeed && o.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED {
   106  		return nil, fmt.Errorf("wrong type used with useSeed=true")
   107  	}
   108  
   109  	if !r.useSeed && o.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_AES_GCM_KEY {
   110  		return nil, fmt.Errorf("wrong type used with useSeed=false")
   111  	}
   112  
   113  	return o.EncryptedDEKSource, nil
   114  }
   115  
   116  func (r envelopekmsv2) startOfPayload(_ string) int {
   117  	return r.prefixLen()
   118  }
   119  
   120  func (r envelopekmsv2) cipherTextPayload() ([]byte, error) {
   121  	o := &kmstypes.EncryptedObject{}
   122  	if err := proto.Unmarshal(r.rawEnvelope[r.startOfPayload(r.providerName):], o); err != nil {
   123  		return nil, err
   124  	}
   125  
   126  	if err := kmsv2.ValidateEncryptedObject(o); err != nil {
   127  		return nil, err
   128  	}
   129  
   130  	return o.EncryptedData, nil
   131  }
   132  
   133  func (r envelopekmsv2) plainTextPayload(secretETCDPath string) ([]byte, error) {
   134  	var transformer value.Read
   135  	var err error
   136  	if r.useSeed {
   137  		transformer, err = aestransformer.NewHKDFExtendedNonceGCMTransformer(r.plainTextDEKSource)
   138  	} else {
   139  		var block cipher.Block
   140  		block, err = aes.NewCipher(r.plainTextDEKSource)
   141  		if err != nil {
   142  			return nil, err
   143  		}
   144  		transformer, err = aestransformer.NewGCMTransformer(block)
   145  	}
   146  	if err != nil {
   147  		return nil, err
   148  	}
   149  
   150  	ctx := context.Background()
   151  	dataCtx := value.DefaultContext(secretETCDPath)
   152  
   153  	data, err := r.cipherTextPayload()
   154  	if err != nil {
   155  		return nil, fmt.Errorf("failed to get cipher text payload: %v", err)
   156  	}
   157  	plainSecret, _, err := transformer.TransformFromStorage(ctx, data, dataCtx)
   158  	if err != nil {
   159  		return nil, fmt.Errorf("failed to transform from storage via AESGCM, err: %w", err)
   160  	}
   161  
   162  	return plainSecret, nil
   163  }
   164  
   165  // TestDefaultValues tests default flag values without setting any of the feature flags or
   166  // calling SetKDFForTests, and assert that the data stored in etcd is using KDF
   167  func TestDefaultValues(t *testing.T) {
   168  	if encryptionconfig.GetKDF() != true {
   169  		t.Fatalf("without updating the feature flags, default value of KMSv2KDF should be enabled.")
   170  	}
   171  	if utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) != true {
   172  		t.Fatalf("without updating the feature flags, default value of KMSv2 should be enabled.")
   173  	}
   174  	if utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) != false {
   175  		t.Fatalf("without updating the feature flags, default value of KMSv1 should be disabled.")
   176  	}
   177  	// since encryptionconfig.GetKDF() is true by default, following test should verify if
   178  	// object.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
   179  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
   180  	t.Cleanup(cancel)
   181  
   182  	encryptionConfig := `
   183  kind: EncryptionConfiguration
   184  apiVersion: apiserver.config.k8s.io/v1
   185  resources:
   186    - resources:
   187      - pods
   188      providers:
   189      - kms:
   190         apiVersion: v2
   191         name: kms-provider
   192         endpoint: unix:///@kms-provider.sock
   193  `
   194  	_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
   195  
   196  	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
   197  	if err != nil {
   198  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   199  	}
   200  	t.Cleanup(test.cleanUp)
   201  
   202  	client := kubernetes.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
   203  	if _, err := client.CoreV1().Pods(testNamespace).Create(ctx, &corev1.Pod{
   204  		ObjectMeta: metav1.ObjectMeta{
   205  			Name: "test",
   206  		},
   207  		Spec: corev1.PodSpec{
   208  			Containers: []corev1.Container{
   209  				{
   210  					Name:  "busybox",
   211  					Image: "busybox",
   212  				},
   213  			},
   214  		},
   215  	}, metav1.CreateOptions{}); err != nil {
   216  		t.Fatal(err)
   217  	}
   218  
   219  	config := test.kubeAPIServer.ServerOpts.Etcd.StorageConfig
   220  	rawClient, etcdClient, err := integration.GetEtcdClients(config.Transport)
   221  	if err != nil {
   222  		t.Fatalf("failed to create etcd client: %v", err)
   223  	}
   224  	t.Cleanup(func() { _ = rawClient.Close() })
   225  
   226  	response, err := etcdClient.Get(ctx, "/"+config.Prefix+"/pods/"+testNamespace+"/", clientv3.WithPrefix())
   227  	if err != nil {
   228  		t.Fatal(err)
   229  	}
   230  	if len(response.Kvs) != 1 {
   231  		t.Fatalf("expected 1 KVs, but got %d", len(response.Kvs))
   232  	}
   233  	object := kmstypes.EncryptedObject{}
   234  	v := bytes.TrimPrefix(response.Kvs[0].Value, []byte("k8s:enc:kms:v2:kms-provider:"))
   235  	if err := proto.Unmarshal(v, &object); err != nil {
   236  		t.Fatal(err)
   237  	}
   238  	if object.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED {
   239  		t.Errorf("invalid type: %d", object.EncryptedDEKSourceType)
   240  	}
   241  }
   242  
   243  // TestKMSv2Provider is an integration test between KubeAPI, ETCD and KMSv2 Plugin
   244  // Concretely, this test verifies the following integration contracts:
   245  // 1. Raw records in ETCD that were processed by KMSv2 Provider should be prefixed with k8s:enc:kms:v2:<plugin name>:
   246  // 2. Data Encryption Key (DEK) / DEK seed should be generated by envelopeTransformer and passed to KMS gRPC Plugin
   247  // 3. KMS gRPC Plugin should encrypt the DEK/seed with a Key Encryption Key (KEK) and pass it back to envelopeTransformer
   248  // 4. The cipherTextPayload (ex. Secret) should be encrypted via AES GCM transform / extended nonce GCM
   249  // 5. kmstypes.EncryptedObject structure should be serialized and deposited in ETCD
   250  func TestKMSv2Provider(t *testing.T) {
   251  	defaultUseSeed := encryptionconfig.GetKDF()
   252  
   253  	t.Run("regular gcm", func(t *testing.T) {
   254  		defer encryptionconfig.SetKDFForTests(false)()
   255  		testKMSv2Provider(t, !defaultUseSeed)
   256  	})
   257  	t.Run("extended nonce gcm", func(t *testing.T) {
   258  		defer encryptionconfig.SetKDFForTests(true)()
   259  		testKMSv2Provider(t, defaultUseSeed)
   260  	})
   261  }
   262  
   263  func testKMSv2Provider(t *testing.T, useSeed bool) {
   264  	encryptionConfig := `
   265  kind: EncryptionConfiguration
   266  apiVersion: apiserver.config.k8s.io/v1
   267  resources:
   268    - resources:
   269      - secrets
   270      providers:
   271      - kms:
   272         apiVersion: v2
   273         name: kms-provider
   274         endpoint: unix:///@kms-provider.sock
   275  `
   276  	genericapiserver.SetHostnameFuncForTests("testAPIServerID")
   277  	providerName := "kms-provider"
   278  	pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
   279  
   280  	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
   281  	if err != nil {
   282  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   283  	}
   284  	defer test.cleanUp()
   285  
   286  	ctx := testContext(t)
   287  
   288  	// the global metrics registry persists across test runs - reset it here so we can make assertions
   289  	copyConfig := rest.CopyConfig(test.kubeAPIServer.ClientConfig)
   290  	copyConfig.GroupVersion = &schema.GroupVersion{}
   291  	copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
   292  	rc, err := rest.RESTClientFor(copyConfig)
   293  	if err != nil {
   294  		t.Fatal(err)
   295  	}
   296  	if err := rc.Delete().AbsPath("/metrics").Do(ctx).Error(); err != nil {
   297  		t.Fatal(err)
   298  	}
   299  
   300  	// assert that the metrics we collect during the test run match expectations
   301  	wantMetricStrings := []string{
   302  		`apiserver_envelope_encryption_dek_source_cache_size{provider_name="kms-provider"} 1`,
   303  		`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} FP`,
   304  		`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`,
   305  		`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`,
   306  		`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`,
   307  	}
   308  	defer func() {
   309  		body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
   310  		if err != nil {
   311  			t.Fatal(err)
   312  		}
   313  		var gotMetricStrings []string
   314  		trimFP := regexp.MustCompile(`(.*)(} \d+\.\d+.*)`)
   315  		for _, line := range strings.Split(string(body), "\n") {
   316  			if strings.HasPrefix(line, "apiserver_envelope_") {
   317  				if strings.HasPrefix(line, "apiserver_envelope_encryption_dek_cache_fill_percent") {
   318  					continue // this can be ignored as it is KMS v1 only
   319  				}
   320  
   321  				if strings.Contains(line, "_seconds") {
   322  					line = trimFP.ReplaceAllString(line, `$1`) + "} FP" // ignore floating point metric values
   323  				}
   324  
   325  				gotMetricStrings = append(gotMetricStrings, line)
   326  			}
   327  		}
   328  		if diff := cmp.Diff(wantMetricStrings, gotMetricStrings); diff != "" {
   329  			t.Errorf("unexpected metrics diff (-want +got): %s", diff)
   330  		}
   331  	}()
   332  
   333  	test.secret, err = test.createSecret(testSecret, testNamespace)
   334  	if err != nil {
   335  		t.Fatalf("Failed to create test secret, error: %v", err)
   336  	}
   337  
   338  	plainTextDEKSource := pluginMock.LastEncryptRequest()
   339  
   340  	secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace)
   341  	rawEnvelope, err := test.getRawSecretFromETCD()
   342  	if err != nil {
   343  		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
   344  	}
   345  
   346  	envelopeData := envelopekmsv2{
   347  		providerName:       providerName,
   348  		rawEnvelope:        rawEnvelope,
   349  		plainTextDEKSource: plainTextDEKSource,
   350  		useSeed:            useSeed,
   351  	}
   352  
   353  	wantPrefix := envelopeData.prefix()
   354  	if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) {
   355  		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope)
   356  	}
   357  
   358  	ciphertext, err := envelopeData.cipherTextDEKSource()
   359  	if err != nil {
   360  		t.Fatalf("failed to get ciphertext DEK/seed from KMSv2 Plugin: %v", err)
   361  	}
   362  	decryptResponse, err := pluginMock.Decrypt(ctx, &kmsv2api.DecryptRequest{Uid: string(uuid.NewUUID()), Ciphertext: ciphertext})
   363  	if err != nil {
   364  		t.Fatalf("failed to decrypt DEK, %v", err)
   365  	}
   366  	dekSourcePlainAsWouldBeSeenByETCD := decryptResponse.Plaintext
   367  
   368  	if !bytes.Equal(plainTextDEKSource, dekSourcePlainAsWouldBeSeenByETCD) {
   369  		t.Fatalf("expected plainTextDEKSource %v to be passed to KMS Plugin, but got %s",
   370  			plainTextDEKSource, dekSourcePlainAsWouldBeSeenByETCD)
   371  	}
   372  
   373  	plainSecret, err := envelopeData.plainTextPayload(secretETCDPath)
   374  	if err != nil {
   375  		t.Fatalf("failed to transform from storage via AESGCM, err: %v", err)
   376  	}
   377  
   378  	if !strings.Contains(string(plainSecret), secretVal) {
   379  		t.Fatalf("expected %q after decryption, but got %q", secretVal, string(plainSecret))
   380  	}
   381  
   382  	secretClient := test.restClient.CoreV1().Secrets(testNamespace)
   383  	// Secrets should be un-enveloped on direct reads from Kube API Server.
   384  	s, err := secretClient.Get(ctx, testSecret, metav1.GetOptions{})
   385  	if err != nil {
   386  		t.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err)
   387  	}
   388  	if secretVal != string(s.Data[secretKey]) {
   389  		t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
   390  	}
   391  }
   392  
   393  // TestKMSv2ProviderKeyIDStaleness is an integration test between KubeAPI and KMSv2 Plugin
   394  // Concretely, this test verifies the following contracts for no-op updates:
   395  // 1. When the key ID is unchanged, the resource version must not change
   396  // 2. When the key ID changes, the resource version changes (but only once)
   397  // 3. For all subsequent updates, the resource version must not change
   398  // 4. When kms-plugin is down, expect creation of new pod and encryption to succeed while the DEK/seed is valid
   399  // 5. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change while the DEK/seed is valid
   400  // 6. When kms-plugin is down, expect creation of new pod and encryption to fail once the DEK/seed is invalid
   401  // 7. when kms-plugin is down, no-op update for a pod should succeed and not result in RV change even once the DEK/seed is valid
   402  func TestKMSv2ProviderKeyIDStaleness(t *testing.T) {
   403  	t.Run("regular gcm", func(t *testing.T) {
   404  		defer encryptionconfig.SetKDFForTests(false)()
   405  		testKMSv2ProviderKeyIDStaleness(t)
   406  	})
   407  	t.Run("extended nonce gcm", func(t *testing.T) {
   408  		testKMSv2ProviderKeyIDStaleness(t)
   409  	})
   410  }
   411  
   412  func testKMSv2ProviderKeyIDStaleness(t *testing.T) {
   413  	encryptionConfig := `
   414  kind: EncryptionConfiguration
   415  apiVersion: apiserver.config.k8s.io/v1
   416  resources:
   417    - resources:
   418      - pods
   419      - deployments.apps
   420      providers:
   421      - kms:
   422         apiVersion: v2
   423         name: kms-provider
   424         endpoint: unix:///@kms-provider.sock
   425  `
   426  	pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
   427  
   428  	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
   429  	if err != nil {
   430  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   431  	}
   432  	defer test.cleanUp()
   433  
   434  	dynamicClient := dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
   435  
   436  	testPod, err := test.createPod(testNamespace, dynamicClient)
   437  	if err != nil {
   438  		t.Fatalf("Failed to create test pod, error: %v, ns: %s", err, testNamespace)
   439  	}
   440  	version1 := testPod.GetResourceVersion()
   441  
   442  	// 1. no-op update for the test pod should not result in any RV change
   443  	updatedPod, err := test.inplaceUpdatePod(testNamespace, testPod, dynamicClient)
   444  	if err != nil {
   445  		t.Fatalf("Failed to update test pod, error: %v, ns: %s", err, testNamespace)
   446  	}
   447  	version2 := updatedPod.GetResourceVersion()
   448  	if version1 != version2 {
   449  		t.Fatalf("Resource version should not have changed. old pod: %v, new pod: %v", testPod, updatedPod)
   450  	}
   451  
   452  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
   453  	t.Cleanup(cancel)
   454  
   455  	useSeed := encryptionconfig.GetKDF()
   456  
   457  	var firstEncryptedDEKSource []byte
   458  	var f checkFunc
   459  	if useSeed {
   460  		f = func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) {
   461  			firstEncryptedDEKSource = obj.EncryptedDEKSource
   462  
   463  			if obj.KeyID != "1" {
   464  				t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
   465  			}
   466  		}
   467  	} else {
   468  		f = func(_ int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
   469  			firstEncryptedDEKSource = obj.EncryptedDEKSource
   470  
   471  			if obj.KeyID != "1" {
   472  				t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
   473  			}
   474  
   475  			// with the first key we perform encryption during the following steps:
   476  			// - create
   477  			const want = 1_000_000_000 + 1 // zero value of counter is one billion
   478  			if want != counter {
   479  				t.Errorf("key %s: counter nonce is invalid: want %d, got %d", etcdKey, want, counter)
   480  			}
   481  		}
   482  	}
   483  	assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
   484  		1, 1, "k8s:enc:kms:v2:kms-provider:", f,
   485  	)
   486  	if len(firstEncryptedDEKSource) == 0 {
   487  		t.Fatal("unexpected empty DEK or seed")
   488  	}
   489  
   490  	// 2. no-op update for the test pod with keyID update should result in RV change
   491  	pluginMock.UpdateKeyID()
   492  	if err := kmsv2mock.WaitForBase64PluginToBeUpdated(pluginMock); err != nil {
   493  		t.Fatalf("Failed to update keyID for plugin, err: %v", err)
   494  	}
   495  	// Wait 1 sec (poll interval to check resource version) until a resource version change is detected or timeout at 1 minute.
   496  
   497  	version3 := ""
   498  	err = wait.Poll(time.Second, time.Minute,
   499  		func() (bool, error) {
   500  			t.Log("polling for in-place update rv change")
   501  			updatedPod, err = test.inplaceUpdatePod(testNamespace, updatedPod, dynamicClient)
   502  			if err != nil {
   503  				return false, err
   504  			}
   505  			version3 = updatedPod.GetResourceVersion()
   506  			if version1 != version3 {
   507  				return true, nil
   508  			}
   509  			return false, nil
   510  		})
   511  	if err != nil {
   512  		t.Fatalf("Failed to detect one resource version update within the allotted time after keyID is updated and pod has been inplace updated, err: %v, ns: %s", err, testNamespace)
   513  	}
   514  
   515  	if version1 == version3 {
   516  		t.Fatalf("Resource version should have changed after keyID update. old pod: %v, new pod: %v", testPod, updatedPod)
   517  	}
   518  
   519  	var wantCount uint64 = 1_000_000_000 // zero value of counter is one billion
   520  	wantCount++                          // in place update with RV change
   521  
   522  	// with the second key we perform encryption during the following steps:
   523  	// - in place update with RV change
   524  	// - delete (which does an update to set deletion timestamp)
   525  	// - create
   526  	var checkDEK checkFunc
   527  	if useSeed {
   528  		checkDEK = func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) {
   529  			if len(obj.EncryptedDEKSource) == 0 {
   530  				t.Error("unexpected empty DEK source")
   531  			}
   532  
   533  			if bytes.Equal(obj.EncryptedDEKSource, firstEncryptedDEKSource) {
   534  				t.Errorf("key %s: incorrectly has the same ESEED", etcdKey)
   535  			}
   536  
   537  			if obj.KeyID != "2" {
   538  				t.Errorf("key %s: want key ID %s, got %s", etcdKey, "2", obj.KeyID)
   539  			}
   540  		}
   541  	} else {
   542  		checkDEK = func(_ int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
   543  			if len(obj.EncryptedDEKSource) == 0 {
   544  				t.Error("unexpected empty DEK source")
   545  			}
   546  
   547  			if bytes.Equal(obj.EncryptedDEKSource, firstEncryptedDEKSource) {
   548  				t.Errorf("key %s: incorrectly has the same EDEK", etcdKey)
   549  			}
   550  
   551  			if obj.KeyID != "2" {
   552  				t.Errorf("key %s: want key ID %s, got %s", etcdKey, "2", obj.KeyID)
   553  			}
   554  
   555  			if wantCount != counter {
   556  				t.Errorf("key %s: counter nonce is invalid: want %d, got %d", etcdKey, wantCount, counter)
   557  			}
   558  		}
   559  	}
   560  
   561  	// 3. no-op update for the updated pod should not result in RV change
   562  	updatedPod, err = test.inplaceUpdatePod(testNamespace, updatedPod, dynamicClient)
   563  	if err != nil {
   564  		t.Fatalf("Failed to update test pod, error: %v, ns: %s", err, testNamespace)
   565  	}
   566  	version4 := updatedPod.GetResourceVersion()
   567  	if version3 != version4 {
   568  		t.Fatalf("Resource version should not have changed again after the initial version updated as a result of the keyID update. old pod: %v, new pod: %v", testPod, updatedPod)
   569  	}
   570  
   571  	// delete the pod so that it can be recreated
   572  	if err := test.deletePod(testNamespace, dynamicClient); err != nil {
   573  		t.Fatalf("failed to delete test pod: %v", err)
   574  	}
   575  	wantCount++ // we cannot assert against the counter being 2 since the pod gets deleted
   576  
   577  	// 4. when kms-plugin is down, expect creation of new pod and encryption to succeed because the DEK is still valid
   578  	pluginMock.EnterFailedState()
   579  	mustBeUnHealthy(t, "/kms-providers",
   580  		"internal server error: kms-provider-0: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
   581  		test.kubeAPIServer.ClientConfig)
   582  
   583  	newPod, err := test.createPod(testNamespace, dynamicClient)
   584  	if err != nil {
   585  		t.Fatalf("Create test pod should have succeeded due to valid DEK, ns: %s, got: %v", testNamespace, err)
   586  	}
   587  	wantCount++
   588  	version5 := newPod.GetResourceVersion()
   589  
   590  	// 5. when kms-plugin is down and DEK is valid, no-op update for a pod should succeed and not result in RV change
   591  	updatedPod, err = test.inplaceUpdatePod(testNamespace, newPod, dynamicClient)
   592  	if err != nil {
   593  		t.Fatalf("Failed to perform no-op update on pod when kms-plugin is down, error: %v, ns: %s", err, testNamespace)
   594  	}
   595  	version6 := updatedPod.GetResourceVersion()
   596  	if version5 != version6 {
   597  		t.Fatalf("Resource version should not have changed again after the initial version updated as a result of the keyID update. old pod: %v, new pod: %v", newPod, updatedPod)
   598  	}
   599  
   600  	// Invalidate the DEK by moving the current time forward
   601  	origNowFunc := kmsv2.NowFunc
   602  	t.Cleanup(func() { kmsv2.NowFunc = origNowFunc })
   603  	kmsv2.NowFunc = func() time.Time { return origNowFunc().Add(5 * time.Minute) }
   604  
   605  	// 6. when kms-plugin is down, expect creation of new pod and encryption to fail because the DEK is invalid
   606  	_, err = test.createPod(testNamespace, dynamicClient)
   607  	if err == nil || !strings.Contains(err.Error(), `encryptedDEKSource with keyID hash "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" expired at 2`) {
   608  		t.Fatalf("Create test pod should have failed due to encryption, ns: %s, got: %v", testNamespace, err)
   609  	}
   610  
   611  	// 7. when kms-plugin is down and DEK is invalid, no-op update for a pod should succeed and not result in RV change
   612  	updatedNewPod, err := test.inplaceUpdatePod(testNamespace, newPod, dynamicClient)
   613  	if err != nil {
   614  		t.Fatalf("Failed to perform no-op update on pod when kms-plugin is down, error: %v, ns: %s", err, testNamespace)
   615  	}
   616  	version7 := updatedNewPod.GetResourceVersion()
   617  	if version5 != version7 {
   618  		t.Fatalf("Resource version should not have changed again after the initial version updated as a result of the keyID update. old pod: %v, new pod: %v", newPod, updatedNewPod)
   619  	}
   620  
   621  	assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
   622  		1, 1, "k8s:enc:kms:v2:kms-provider:", checkDEK,
   623  	)
   624  
   625  	// fix plugin and wait for new writes to start working again
   626  	kmsv2.NowFunc = origNowFunc
   627  	pluginMock.ExitFailedState()
   628  	err = wait.Poll(time.Second, 3*time.Minute,
   629  		func() (bool, error) {
   630  			t.Log("polling for plugin to be functional")
   631  			_, err = test.createDeployment("panda", testNamespace)
   632  			if err != nil {
   633  				t.Logf("failed to create deployment, plugin is likely still unhealthy: %v", err)
   634  			}
   635  			return err == nil, nil
   636  		})
   637  	if err != nil {
   638  		t.Fatalf("failed to restore plugin health, err: %v, ns: %s", err, testNamespace)
   639  	}
   640  
   641  	// 8. confirm that no-op update for a pod succeeds and still does not result in RV change
   642  	updatedNewPod2, err := test.inplaceUpdatePod(testNamespace, updatedNewPod, dynamicClient)
   643  	if err != nil {
   644  		t.Fatalf("Failed to perform no-op update on pod when kms-plugin is up, error: %v, ns: %s", err, testNamespace)
   645  	}
   646  	version8 := updatedNewPod2.GetResourceVersion()
   647  	if version7 != version8 {
   648  		t.Fatalf("Resource version should not have changed after plugin health is restored. old pod: %v, new pod: %v", updatedNewPod, updatedNewPod2)
   649  	}
   650  	// flip the current config
   651  	defer encryptionconfig.SetKDFForTests(!useSeed)()
   652  
   653  	// 9. confirm that no-op update for a pod results in RV change due to KDF config change
   654  	var version9 string
   655  	err = wait.Poll(time.Second, 3*time.Minute,
   656  		func() (bool, error) {
   657  			t.Log("polling for in-place update rv change due to KDF config change")
   658  			updatedNewPod2, err = test.inplaceUpdatePod(testNamespace, updatedNewPod2, dynamicClient)
   659  			if err != nil {
   660  				return false, err
   661  			}
   662  			version9 = updatedNewPod2.GetResourceVersion()
   663  			if version8 != version9 {
   664  				return true, nil
   665  			}
   666  			return false, nil
   667  		})
   668  	if err != nil {
   669  		t.Fatalf("Failed to detect one resource version update within the allotted time after KDF config change and pod has been inplace updated, err: %v, ns: %s", err, testNamespace)
   670  	}
   671  }
   672  
   673  func TestKMSv2ProviderDEKSourceReuse(t *testing.T) {
   674  	t.Run("regular gcm", func(t *testing.T) {
   675  		defer encryptionconfig.SetKDFForTests(false)()
   676  		testKMSv2ProviderDEKSourceReuse(t,
   677  			func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject) {
   678  				if obj.KeyID != "1" {
   679  					t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
   680  				}
   681  
   682  				// zero value of counter is one billion so the first value will be one billion plus one
   683  				// hence we add that to our zero based index to calculate the expected nonce
   684  				if uint64(i+1_000_000_000+1) != counter {
   685  					t.Errorf("key %s: counter nonce is invalid: want %d, got %d", etcdKey, i+1, counter)
   686  				}
   687  			},
   688  		)
   689  	})
   690  	t.Run("extended nonce gcm", func(t *testing.T) {
   691  		defer encryptionconfig.SetKDFForTests(true)()
   692  		testKMSv2ProviderDEKSourceReuse(t,
   693  			func(_ int, _ uint64, etcdKey string, obj kmstypes.EncryptedObject) {
   694  				if obj.KeyID != "1" {
   695  					t.Errorf("key %s: want key ID %s, got %s", etcdKey, "1", obj.KeyID)
   696  				}
   697  			},
   698  		)
   699  	})
   700  }
   701  
   702  func testKMSv2ProviderDEKSourceReuse(t *testing.T, f checkFunc) {
   703  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
   704  	t.Cleanup(cancel)
   705  
   706  	encryptionConfig := `
   707  kind: EncryptionConfiguration
   708  apiVersion: apiserver.config.k8s.io/v1
   709  resources:
   710    - resources:
   711      - pods
   712      providers:
   713      - kms:
   714         apiVersion: v2
   715         name: kms-provider
   716         endpoint: unix:///@kms-provider.sock
   717  `
   718  	_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
   719  
   720  	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
   721  	if err != nil {
   722  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   723  	}
   724  	t.Cleanup(test.cleanUp)
   725  
   726  	client := kubernetes.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
   727  
   728  	const podCount = 1_000
   729  
   730  	for i := 0; i < podCount; i++ {
   731  		if _, err := client.CoreV1().Pods(testNamespace).Create(ctx, &corev1.Pod{
   732  			ObjectMeta: metav1.ObjectMeta{
   733  				Name: fmt.Sprintf("dek-reuse-%04d", i+1), // making creation order match returned list order / nonce counter
   734  			},
   735  			Spec: corev1.PodSpec{
   736  				Containers: []corev1.Container{
   737  					{
   738  						Name:  "busybox",
   739  						Image: "busybox",
   740  					},
   741  				},
   742  			},
   743  		}, metav1.CreateOptions{}); err != nil {
   744  			t.Fatal(err)
   745  		}
   746  	}
   747  
   748  	assertPodDEKSources(ctx, t, test.kubeAPIServer.ServerOpts.Etcd.StorageConfig,
   749  		podCount, 1, // key ID does not change during the test so we should only have a single DEK
   750  		"k8s:enc:kms:v2:kms-provider:", f,
   751  	)
   752  }
   753  
   754  type checkFunc func(i int, counter uint64, etcdKey string, obj kmstypes.EncryptedObject)
   755  
   756  func assertPodDEKSources(ctx context.Context, t *testing.T, config storagebackend.Config, podCount, dekSourcesCount int, kmsPrefix string, f checkFunc) {
   757  	t.Helper()
   758  
   759  	rawClient, etcdClient, err := integration.GetEtcdClients(config.Transport)
   760  	if err != nil {
   761  		t.Fatalf("failed to create etcd client: %v", err)
   762  	}
   763  	t.Cleanup(func() { _ = rawClient.Close() })
   764  
   765  	response, err := etcdClient.Get(ctx, "/"+config.Prefix+"/pods/"+testNamespace+"/", clientv3.WithPrefix())
   766  	if err != nil {
   767  		t.Fatal(err)
   768  	}
   769  
   770  	if len(response.Kvs) != podCount {
   771  		t.Fatalf("expected %d KVs, but got %d", podCount, len(response.Kvs))
   772  	}
   773  
   774  	useSeed := encryptionconfig.GetKDF()
   775  
   776  	out := make([]kmstypes.EncryptedObject, len(response.Kvs))
   777  	for i, kv := range response.Kvs {
   778  		v := bytes.TrimPrefix(kv.Value, []byte(kmsPrefix))
   779  		if err := proto.Unmarshal(v, &out[i]); err != nil {
   780  			t.Fatal(err)
   781  		}
   782  
   783  		if err := kmsv2.ValidateEncryptedObject(&out[i]); err != nil {
   784  			t.Fatal(err)
   785  		}
   786  
   787  		var infoLen int
   788  		if useSeed {
   789  			infoLen = 32
   790  		}
   791  
   792  		info := out[i].EncryptedData[:infoLen]
   793  		nonce := out[i].EncryptedData[infoLen : 12+infoLen]
   794  		randN := nonce[:4]
   795  		count := nonce[4:]
   796  
   797  		if bytes.Equal(randN, make([]byte, len(randN))) {
   798  			t.Errorf("key %s: got all zeros for first four bytes", string(kv.Key))
   799  		}
   800  
   801  		if useSeed {
   802  			if bytes.Equal(info, make([]byte, infoLen)) {
   803  				t.Errorf("key %s: got all zeros for info", string(kv.Key))
   804  			}
   805  		}
   806  
   807  		counter := binary.LittleEndian.Uint64(count)
   808  		f(i, counter, string(kv.Key), out[i])
   809  	}
   810  
   811  	uniqueDEKSources := sets.NewString()
   812  	for _, object := range out {
   813  		object := object
   814  		uniqueDEKSources.Insert(string(object.EncryptedDEKSource))
   815  		if useSeed {
   816  			if object.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED {
   817  				t.Errorf("invalid type: %d", object.EncryptedDEKSourceType)
   818  			}
   819  		} else {
   820  			if object.EncryptedDEKSourceType != kmstypes.EncryptedDEKSourceType_AES_GCM_KEY {
   821  				t.Errorf("invalid type: %d", object.EncryptedDEKSourceType)
   822  			}
   823  		}
   824  	}
   825  
   826  	if uniqueDEKSources.Has("") {
   827  		t.Error("unexpected empty DEK source seen")
   828  	}
   829  
   830  	if uniqueDEKSources.Len() != dekSourcesCount {
   831  		t.Errorf("expected %d DEK sources, got: %d", dekSourcesCount, uniqueDEKSources.Len())
   832  	}
   833  }
   834  
   835  func TestKMSv2Healthz(t *testing.T) {
   836  	defer encryptionconfig.SetKDFForTests(randomBool())()
   837  
   838  	encryptionConfig := `
   839  kind: EncryptionConfiguration
   840  apiVersion: apiserver.config.k8s.io/v1
   841  resources:
   842    - resources:
   843      - secrets
   844      providers:
   845      - kms:
   846         apiVersion: v2
   847         name: provider-1
   848         endpoint: unix:///@kms-provider-1.sock
   849      - kms:
   850         apiVersion: v2
   851         name: provider-2
   852         endpoint: unix:///@kms-provider-2.sock
   853  `
   854  
   855  	pluginMock1 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-1.sock")
   856  	pluginMock2 := kmsv2mock.NewBase64Plugin(t, "@kms-provider-2.sock")
   857  
   858  	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
   859  	if err != nil {
   860  		t.Fatalf("Failed to start kube-apiserver, error: %v", err)
   861  	}
   862  	defer test.cleanUp()
   863  
   864  	// Name of the healthz check is always "kms-provider-0" and it covers all kms plugins.
   865  
   866  	// Stage 1 - Since all kms-plugins are guaranteed to be up,
   867  	// the healthz check should be OK.
   868  	mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
   869  
   870  	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the healthz check
   871  	// to fail and report that provider-1 is down
   872  	pluginMock1.EnterFailedState()
   873  	mustBeUnHealthy(t, "/kms-providers",
   874  		"internal server error: kms-provider-0: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
   875  		test.kubeAPIServer.ClientConfig)
   876  	pluginMock1.ExitFailedState()
   877  
   878  	// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
   879  	// to succeed now, but provider-2 is now down.
   880  	pluginMock2.EnterFailedState()
   881  	mustBeUnHealthy(t, "/kms-providers",
   882  		"internal server error: kms-provider-1: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
   883  		test.kubeAPIServer.ClientConfig)
   884  	pluginMock2.ExitFailedState()
   885  
   886  	// Stage 4 - All kms-plugins are once again up,
   887  	// the healthz check should be OK.
   888  	mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
   889  
   890  	// Stage 5 - All kms-plugins are unhealthy at the same time and we can observe both failures.
   891  	pluginMock1.EnterFailedState()
   892  	pluginMock2.EnterFailedState()
   893  	mustBeUnHealthy(t, "/kms-providers",
   894  		"internal server error: "+
   895  			"[kms-provider-0: failed to perform status section of the healthz check for KMS Provider provider-1, error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled,"+
   896  			" kms-provider-1: failed to perform status section of the healthz check for KMS Provider provider-2, error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled]",
   897  		test.kubeAPIServer.ClientConfig)
   898  }
   899  
   900  func TestKMSv2SingleService(t *testing.T) {
   901  	defer encryptionconfig.SetKDFForTests(randomBool())()
   902  
   903  	var kmsv2Calls int
   904  	origEnvelopeKMSv2ServiceFactory := encryptionconfig.EnvelopeKMSv2ServiceFactory
   905  	encryptionconfig.EnvelopeKMSv2ServiceFactory = func(ctx context.Context, endpoint, providerName string, callTimeout time.Duration) (kmsv2svc.Service, error) {
   906  		kmsv2Calls++
   907  		return origEnvelopeKMSv2ServiceFactory(ctx, endpoint, providerName, callTimeout)
   908  	}
   909  	t.Cleanup(func() {
   910  		encryptionconfig.EnvelopeKMSv2ServiceFactory = origEnvelopeKMSv2ServiceFactory
   911  	})
   912  
   913  	// check resources provided by the three servers that we have wired together
   914  	// - pods and config maps from KAS
   915  	// - CRDs and CRs from API extensions
   916  	// - API services from aggregator
   917  	encryptionConfig := `
   918  kind: EncryptionConfiguration
   919  apiVersion: apiserver.config.k8s.io/v1
   920  resources:
   921    - resources:
   922      - pods
   923      - configmaps
   924      - customresourcedefinitions.apiextensions.k8s.io
   925      - pandas.awesome.bears.com
   926      - apiservices.apiregistration.k8s.io
   927      providers:
   928      - kms:
   929         apiVersion: v2
   930         name: kms-provider
   931         endpoint: unix:///@kms-provider.sock
   932  `
   933  
   934  	_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
   935  
   936  	test, err := newTransformTest(t, encryptionConfig, false, "", nil)
   937  	if err != nil {
   938  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   939  	}
   940  	t.Cleanup(test.cleanUp)
   941  
   942  	// the storage registry for CRs is dynamic so create one to exercise the wiring
   943  	etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(test.kubeAPIServer.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...)
   944  
   945  	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   946  	t.Cleanup(cancel)
   947  
   948  	gvr := schema.GroupVersionResource{Group: "awesome.bears.com", Version: "v1", Resource: "pandas"}
   949  	stub := etcd.GetEtcdStorageData()[gvr].Stub
   950  	dynamicClient, obj, err := etcd.JSONToUnstructured(stub, "", &meta.RESTMapping{
   951  		Resource:         gvr,
   952  		GroupVersionKind: gvr.GroupVersion().WithKind("Panda"),
   953  		Scope:            meta.RESTScopeRoot,
   954  	}, dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig))
   955  	if err != nil {
   956  		t.Fatal(err)
   957  	}
   958  	_, err = dynamicClient.Create(ctx, obj, metav1.CreateOptions{})
   959  	if err != nil {
   960  		t.Fatal(err)
   961  	}
   962  
   963  	if kmsv2Calls != 1 {
   964  		t.Fatalf("expected a single call to KMS v2 service factory: %v", kmsv2Calls)
   965  	}
   966  }
   967  
   968  // TestKMSv2FeatureFlag is an integration test between KubeAPI and ETCD
   969  // Concretely, this test verifies the following:
   970  // 1. When feature flag is enabled, loading a encryptionConfig with KMSv2 should work
   971  // 2. After a restart, loading a encryptionConfig with the same KMSv2 plugin from 1 should work,
   972  // decryption of data encrypted with v2 should work
   973  func TestKMSv2FeatureFlag(t *testing.T) {
   974  	encryptionConfig := `
   975  kind: EncryptionConfiguration
   976  apiVersion: apiserver.config.k8s.io/v1
   977  resources:
   978    - resources:
   979      - secrets
   980      providers:
   981      - kms:
   982         apiVersion: v2
   983         name: kms-provider
   984         endpoint: unix:///@kms-provider.sock
   985  `
   986  	providerName := "kms-provider"
   987  	pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
   988  	storageConfig := framework.SharedEtcd()
   989  
   990  	// KMSv2 is enabled by default. Loading a encryptionConfig with KMSv2 should work
   991  	test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig)
   992  	if err != nil {
   993  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
   994  	}
   995  	defer func() {
   996  		test.cleanUp()
   997  	}()
   998  
   999  	test.secret, err = test.createSecret(testSecret, testNamespace)
  1000  	if err != nil {
  1001  		t.Fatalf("Failed to create test secret, error: %v", err)
  1002  	}
  1003  
  1004  	// Since Data Encryption Key (DEK) is randomly generated, we need to ask KMS Mock for it.
  1005  	plainTextDEKSource := pluginMock.LastEncryptRequest()
  1006  
  1007  	secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace)
  1008  	rawEnvelope, err := test.getRawSecretFromETCD()
  1009  	if err != nil {
  1010  		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
  1011  	}
  1012  
  1013  	envelopeData := envelopekmsv2{
  1014  		providerName:       providerName,
  1015  		rawEnvelope:        rawEnvelope,
  1016  		plainTextDEKSource: plainTextDEKSource,
  1017  		useSeed:            true, // expect KMSv2KDF to be enabled by default for this test
  1018  	}
  1019  
  1020  	wantPrefix := envelopeData.prefix()
  1021  	if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) {
  1022  		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope)
  1023  	}
  1024  
  1025  	ctx := testContext(t)
  1026  	ciphertext, err := envelopeData.cipherTextDEKSource()
  1027  	if err != nil {
  1028  		t.Fatalf("failed to get ciphertext DEK from KMSv2 Plugin: %v", err)
  1029  	}
  1030  	decryptResponse, err := pluginMock.Decrypt(ctx, &kmsv2api.DecryptRequest{Uid: string(uuid.NewUUID()), Ciphertext: ciphertext})
  1031  	if err != nil {
  1032  		t.Fatalf("failed to decrypt DEK, %v", err)
  1033  	}
  1034  	dekPlainAsWouldBeSeenByETCD := decryptResponse.Plaintext
  1035  
  1036  	if !bytes.Equal(plainTextDEKSource, dekPlainAsWouldBeSeenByETCD) {
  1037  		t.Fatalf("expected plainTextDEKSource %v to be passed to KMS Plugin, but got %s",
  1038  			plainTextDEKSource, dekPlainAsWouldBeSeenByETCD)
  1039  	}
  1040  
  1041  	plainSecret, err := envelopeData.plainTextPayload(secretETCDPath)
  1042  	if err != nil {
  1043  		t.Fatalf("failed to transform from storage via AESGCM, err: %v", err)
  1044  	}
  1045  
  1046  	if !strings.Contains(string(plainSecret), secretVal) {
  1047  		t.Fatalf("expected %q after decryption, but got %q", secretVal, string(plainSecret))
  1048  	}
  1049  
  1050  	secretClient := test.restClient.CoreV1().Secrets(testNamespace)
  1051  	// Secrets should be un-enveloped on direct reads from Kube API Server.
  1052  	s, err := secretClient.Get(ctx, testSecret, metav1.GetOptions{})
  1053  	if err != nil {
  1054  		t.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err)
  1055  	}
  1056  	if secretVal != string(s.Data[secretKey]) {
  1057  		t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
  1058  	}
  1059  	test.shutdownAPIServer()
  1060  
  1061  	// After a restart, loading a encryptionConfig with the same KMSv2 plugin before the restart should work, decryption of data encrypted with v2 should work
  1062  
  1063  	test, err = newTransformTest(t, encryptionConfig, false, "", storageConfig)
  1064  	if err != nil {
  1065  		t.Fatalf("Failed to restart api server, error: %v", err)
  1066  	}
  1067  
  1068  	// Getting an old secret that was encrypted by the same plugin should not fail.
  1069  	s, err = test.restClient.CoreV1().Secrets(testNamespace).Get(
  1070  		ctx,
  1071  		testSecret,
  1072  		metav1.GetOptions{},
  1073  	)
  1074  	if err != nil {
  1075  		t.Fatalf("failed to read secret, err: %v", err)
  1076  	}
  1077  	if secretVal != string(s.Data[secretKey]) {
  1078  		t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
  1079  	}
  1080  }
  1081  
  1082  var benchSecret *api.Secret
  1083  
  1084  func BenchmarkKMSv2KDF(b *testing.B) {
  1085  	b.StopTimer()
  1086  
  1087  	klog.SetOutput(io.Discard)
  1088  	klog.LogToStderr(false)
  1089  
  1090  	defer encryptionconfig.SetKDFForTests(false)()
  1091  
  1092  	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
  1093  	b.Cleanup(cancel)
  1094  
  1095  	ctx = request.WithNamespace(ctx, testNamespace)
  1096  
  1097  	encryptionConfig := `
  1098  kind: EncryptionConfiguration
  1099  apiVersion: apiserver.config.k8s.io/v1
  1100  resources:
  1101    - resources:
  1102      - secrets
  1103      providers:
  1104      - kms:
  1105         apiVersion: v2
  1106         name: kms-provider
  1107         endpoint: unix:///@kms-provider.sock
  1108  `
  1109  	_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
  1110  
  1111  	test, err := newTransformTest(b, encryptionConfig, false, "", nil)
  1112  	if err != nil {
  1113  		b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
  1114  	}
  1115  	b.Cleanup(test.cleanUp)
  1116  
  1117  	client := kubernetes.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
  1118  
  1119  	restOptionsGetter := getRESTOptionsGetterForSecrets(b, test)
  1120  
  1121  	secretStorage, err := secretstore.NewREST(restOptionsGetter)
  1122  	if err != nil {
  1123  		b.Fatal(err)
  1124  	}
  1125  	b.Cleanup(secretStorage.Destroy)
  1126  
  1127  	const dataLen = 1_000
  1128  
  1129  	secrets := make([]*api.Secret, dataLen)
  1130  
  1131  	for i := 0; i < dataLen; i++ {
  1132  		secrets[i] = &api.Secret{
  1133  			ObjectMeta: metav1.ObjectMeta{
  1134  				Name:      fmt.Sprintf("test-secret-%d", i),
  1135  				Namespace: testNamespace,
  1136  			},
  1137  			Data: map[string][]byte{
  1138  				"lots_of_data": bytes.Repeat([]byte{1, 3, 3, 7}, i*dataLen/4),
  1139  			},
  1140  		}
  1141  	}
  1142  
  1143  	b.StartTimer()
  1144  	for i := 0; i < b.N; i++ {
  1145  		_, err := secretStorage.DeleteCollection(ctx, noValidation, &metav1.DeleteOptions{}, nil)
  1146  		if err != nil {
  1147  			b.Fatal(err)
  1148  		}
  1149  
  1150  		for i := 0; i < dataLen; i++ {
  1151  			out, err := secretStorage.Create(ctx, secrets[i], noValidation, &metav1.CreateOptions{})
  1152  			if err != nil {
  1153  				b.Fatal(err)
  1154  			}
  1155  
  1156  			benchSecret = out.(*api.Secret)
  1157  
  1158  			out, err = secretStorage.Get(ctx, benchSecret.Name, &metav1.GetOptions{})
  1159  			if err != nil {
  1160  				b.Fatal(err)
  1161  			}
  1162  
  1163  			benchSecret = out.(*api.Secret)
  1164  		}
  1165  	}
  1166  	b.StopTimer()
  1167  
  1168  	secretList, err := client.CoreV1().Secrets(testNamespace).List(ctx, metav1.ListOptions{})
  1169  	if err != nil {
  1170  		b.Fatal(err)
  1171  	}
  1172  
  1173  	if secretLen := len(secretList.Items); secretLen != dataLen {
  1174  		b.Errorf("unexpected secret len: want %d, got %d", dataLen, secretLen)
  1175  	}
  1176  }
  1177  
  1178  func getRESTOptionsGetterForSecrets(t testing.TB, test *transformTest) generic.RESTOptionsGetter {
  1179  	t.Helper()
  1180  
  1181  	s := test.kubeAPIServer.ServerOpts
  1182  
  1183  	etcdConfigCopy := *s.Etcd
  1184  	etcdConfigCopy.SkipHealthEndpoints = true                     // avoid running health check go routines
  1185  	etcdConfigCopy.EncryptionProviderConfigAutomaticReload = true // hack to use DynamicTransformers in t.Cleanup below
  1186  
  1187  	// mostly copied from BuildGenericConfig
  1188  
  1189  	genericConfig := genericapiserver.NewConfig(legacyscheme.Codecs)
  1190  
  1191  	genericConfig.MergedResourceConfig = controlplane.DefaultAPIResourceConfigSource()
  1192  
  1193  	if err := s.APIEnablement.ApplyTo(genericConfig, controlplane.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); err != nil {
  1194  		t.Fatal(err)
  1195  	}
  1196  
  1197  	storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
  1198  	storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
  1199  	storageFactory, err := storageFactoryConfig.Complete(&etcdConfigCopy).New()
  1200  	if err != nil {
  1201  		t.Fatal(err)
  1202  	}
  1203  	if err := etcdConfigCopy.ApplyWithStorageFactoryTo(storageFactory, genericConfig); err != nil {
  1204  		t.Fatal(err)
  1205  	}
  1206  
  1207  	transformers, ok := genericConfig.ResourceTransformers.(*encryptionconfig.DynamicTransformers)
  1208  	if !ok {
  1209  		t.Fatalf("incorrect type for ResourceTransformers: %T", genericConfig.ResourceTransformers)
  1210  	}
  1211  
  1212  	t.Cleanup(func() {
  1213  		// this is a hack to cause the existing transformers to shutdown
  1214  		transformers.Set(nil, nil, nil, 0)
  1215  		time.Sleep(10 * time.Second) // block this cleanup for longer than kmsCloseGracePeriod
  1216  	})
  1217  
  1218  	if genericConfig.RESTOptionsGetter == nil {
  1219  		t.Fatal("not REST options found")
  1220  	}
  1221  
  1222  	opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"})
  1223  	if err != nil {
  1224  		t.Fatal(err)
  1225  	}
  1226  
  1227  	if err := runtime.CheckCodec(opts.StorageConfig.Codec, &api.Secret{},
  1228  		schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"}); err != nil {
  1229  		t.Fatal(err)
  1230  	}
  1231  
  1232  	return genericConfig.RESTOptionsGetter
  1233  }
  1234  
  1235  func noValidation(_ context.Context, _ runtime.Object) error { return nil }
  1236  
  1237  var benchRESTSecret *corev1.Secret
  1238  
  1239  func BenchmarkKMSv2REST(b *testing.B) {
  1240  	b.StopTimer()
  1241  
  1242  	klog.SetOutput(io.Discard)
  1243  	klog.LogToStderr(false)
  1244  
  1245  	defer encryptionconfig.SetKDFForTests(true)()
  1246  
  1247  	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
  1248  	b.Cleanup(cancel)
  1249  
  1250  	encryptionConfig := `
  1251  kind: EncryptionConfiguration
  1252  apiVersion: apiserver.config.k8s.io/v1
  1253  resources:
  1254    - resources:
  1255      - secrets
  1256      providers:
  1257      - kms:
  1258         apiVersion: v2
  1259         name: kms-provider
  1260         endpoint: unix:///@kms-provider.sock
  1261  `
  1262  	_ = kmsv2mock.NewBase64Plugin(b, "@kms-provider.sock")
  1263  
  1264  	test, err := newTransformTest(b, encryptionConfig, false, "", nil)
  1265  	if err != nil {
  1266  		b.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
  1267  	}
  1268  	b.Cleanup(test.cleanUp)
  1269  
  1270  	client := kubernetes.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
  1271  
  1272  	const dataLen = 1_000
  1273  
  1274  	secretStorage := client.CoreV1().Secrets(testNamespace)
  1275  
  1276  	secrets := make([]*corev1.Secret, dataLen)
  1277  
  1278  	for i := 0; i < dataLen; i++ {
  1279  		secrets[i] = &corev1.Secret{
  1280  			ObjectMeta: metav1.ObjectMeta{
  1281  				Name:      fmt.Sprintf("test-secret-%d", i),
  1282  				Namespace: testNamespace,
  1283  			},
  1284  			Data: map[string][]byte{
  1285  				"lots_of_data": bytes.Repeat([]byte{1, 3, 3, 7}, i*dataLen/4),
  1286  			},
  1287  		}
  1288  	}
  1289  
  1290  	b.StartTimer()
  1291  	for i := 0; i < b.N; i++ {
  1292  		err := secretStorage.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})
  1293  		if err != nil {
  1294  			b.Fatal(err)
  1295  		}
  1296  
  1297  		for i := 0; i < dataLen; i++ {
  1298  			out, err := secretStorage.Create(ctx, secrets[i], metav1.CreateOptions{})
  1299  			if err != nil {
  1300  				b.Fatal(err)
  1301  			}
  1302  
  1303  			benchRESTSecret = out
  1304  
  1305  			out, err = secretStorage.Get(ctx, benchRESTSecret.Name, metav1.GetOptions{})
  1306  			if err != nil {
  1307  				b.Fatal(err)
  1308  			}
  1309  
  1310  			benchRESTSecret = out
  1311  		}
  1312  	}
  1313  	b.StopTimer()
  1314  
  1315  	secretList, err := client.CoreV1().Secrets(testNamespace).List(ctx, metav1.ListOptions{})
  1316  	if err != nil {
  1317  		b.Fatal(err)
  1318  	}
  1319  
  1320  	if secretLen := len(secretList.Items); secretLen != dataLen {
  1321  		b.Errorf("unexpected secret len: want %d, got %d", dataLen, secretLen)
  1322  	}
  1323  }
  1324  
  1325  func randomBool() bool { return utilrand.Int()%2 == 1 }
  1326  
  1327  // TestKMSv2ProviderLegacyData confirms that legacy data recorded from the earliest released commit can still be read.
  1328  func TestKMSv2ProviderLegacyData(t *testing.T) {
  1329  	t.Run("regular gcm", func(t *testing.T) {
  1330  		defer encryptionconfig.SetKDFForTests(false)()
  1331  		testKMSv2ProviderLegacyData(t)
  1332  	})
  1333  	t.Run("extended nonce gcm", func(t *testing.T) {
  1334  		defer encryptionconfig.SetKDFForTests(true)()
  1335  		testKMSv2ProviderLegacyData(t)
  1336  	})
  1337  }
  1338  
  1339  func testKMSv2ProviderLegacyData(t *testing.T) {
  1340  	encryptionConfig := `
  1341  kind: EncryptionConfiguration
  1342  apiVersion: apiserver.config.k8s.io/v1
  1343  resources:
  1344    - resources:
  1345      - secrets
  1346      providers:
  1347      - kms:
  1348         apiVersion: v2
  1349         name: kms-provider
  1350         endpoint: unix:///@kms-provider.sock
  1351  `
  1352  
  1353  	_ = kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
  1354  
  1355  	// the value.Context.AuthenticatedData during read is the etcd storage path of the associated resource
  1356  	// thus we need to manually construct the storage config so that we can have a static path
  1357  	const legacyDataEtcdPrefix = "43da1478-5e9c-4ef3-a92a-2b19d540c8a5"
  1358  
  1359  	storageConfig := storagebackend.NewDefaultConfig(path.Join(legacyDataEtcdPrefix, "registry"), nil)
  1360  	storageConfig.Transport.ServerList = []string{framework.GetEtcdURL()}
  1361  
  1362  	test, err := newTransformTest(t, encryptionConfig, false, "", storageConfig)
  1363  	if err != nil {
  1364  		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
  1365  	}
  1366  	defer test.cleanUp()
  1367  
  1368  	const legacyDataNamespace = "kms-v2-legacy-data"
  1369  
  1370  	if _, err := test.createNamespace(legacyDataNamespace); err != nil {
  1371  		t.Fatal(err)
  1372  	}
  1373  
  1374  	// commit: 1b4df30b3cdfeaba6024e81e559a6cd09a089d65
  1375  	// tag: v1.27.0
  1376  	const dekSourceAESGCMKeyName = "dek-source-aes-gcm-key"
  1377  	const dekSourceAESGCMKeyData = "k8s:enc:kms:v2:kms-provider:\n\x8a\x03\xb0ƙ\xdc\x01ʚ;\x00\x00\x00\x00\x8c\x80\v\xf0\x1dKo{0\v\xb5\xd5Ń\x1a\xb5\x0e\xcf\xd7Ve\xed邸\xdfE\xedMk\xcf!\x15\xc0\v\t\x82Wh \x9e\x8f\x1b\\9\b\xa4\x80\x02m\xf4P\x14z\xee\xf7\x8c\x1a\x84n5\xfdG\x83v#\x0e\xd4\f\x83YwH\xe1\x1c\xbf\x12\xc6\x1b\xba\x8br\b\x82z\xf8\xdb`\xa7]P\xb1\xe6!Lb\x8d\xb8I\x1aEL\xa0\xae+\xbe\x15R\x8e\x9b\x064\xf6P\xb6;\x9f\xa6\x8d\x96\xb2\x01\xa1\x8e\xe4a\xdf/\x90u\xde6\x9a\xc2ͻb\x88+\x16\x98=\xe9\x03\xdd\xd7HvC\n\xe5\x8cv\x05~\n\xabX_N\x9a\x84wp\xa8\x13\x0f\x82Y9x\xed\x89\x15\xb9\xe1ꦐ\xc6`R\n0\x04\xf2\xa6ѥ\x85\nk\xf4\xcf\xe4ul\x1c*[A\x12\xa0\xd9\xf2\xb5!\x82\xe4\x00\xa4L'&\xf5pln\"\xe0=\f[\xe1\xb0U97\x11|\xdaNk\xc3=\xc2\xf2\x85<7\x1e\x01\xb8\xa9\xf4\x89\xdb~\xe1\x8c\xe1\x1f\x05B@WʼS\n聛LY\x86$\xf6\x01ݝ\xcd\x1d\xe9\x02]\xf4i\xda\xfa\xc2\x0eUr\xc5Dʽdb\x13\xbe\xfe\x1c\xc5\xe1\x84\xcc\xdf&\x93j\x1eK\x04\xba\x06\x16\x85\x0e\x1f\xca\b\x90\x06\x11K\x9d[\rV\xe8E\xd5(\x91\fn\xd4\x10\x9cH\x1cܝX\x94ȁ5\x1b\x8c\xbbz\xf9Ho{\x1d\x112\x90F\xe7\xd8h\xa8\xa1\xf6\x8c\x8cvʲ1\xf9#\x82\xa3\xbe7ed\xd9\x14\xf3\x06\xff\xb7߫i\x12\x011\x1a,gafIvwT0ASoKdZ/D1L2SlH73LUMj5qa3hroljfS51wc=\"2\n\x1blocal-kek.kms.kubernetes.io\x12\x13encrypted-local-kek"
  1378  	dekSourceAESGCMKeyPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", dekSourceAESGCMKeyName, legacyDataNamespace)
  1379  	if _, err := test.writeRawRecordToETCD(dekSourceAESGCMKeyPath, []byte(dekSourceAESGCMKeyData)); err != nil {
  1380  		t.Fatal(err)
  1381  	}
  1382  
  1383  	expectedDEKSourceAESGCMKeyObject := &kmstypes.EncryptedObject{
  1384  		EncryptedData:          []byte("\xb0ƙ\xdc\x01ʚ;\x00\x00\x00\x00\x8c\x80\v\xf0\x1dKo{0\v\xb5\xd5Ń\x1a\xb5\x0e\xcf\xd7Ve\xed邸\xdfE\xedMk\xcf!\x15\xc0\v\t\x82Wh \x9e\x8f\x1b\\9\b\xa4\x80\x02m\xf4P\x14z\xee\xf7\x8c\x1a\x84n5\xfdG\x83v#\x0e\xd4\f\x83YwH\xe1\x1c\xbf\x12\xc6\x1b\xba\x8br\b\x82z\xf8\xdb`\xa7]P\xb1\xe6!Lb\x8d\xb8I\x1aEL\xa0\xae+\xbe\x15R\x8e\x9b\x064\xf6P\xb6;\x9f\xa6\x8d\x96\xb2\x01\xa1\x8e\xe4a\xdf/\x90u\xde6\x9a\xc2ͻb\x88+\x16\x98=\xe9\x03\xdd\xd7HvC\n\xe5\x8cv\x05~\n\xabX_N\x9a\x84wp\xa8\x13\x0f\x82Y9x\xed\x89\x15\xb9\xe1ꦐ\xc6`R\n0\x04\xf2\xa6ѥ\x85\nk\xf4\xcf\xe4ul\x1c*[A\x12\xa0\xd9\xf2\xb5!\x82\xe4\x00\xa4L'&\xf5pln\"\xe0=\f[\xe1\xb0U97\x11|\xdaNk\xc3=\xc2\xf2\x85<7\x1e\x01\xb8\xa9\xf4\x89\xdb~\xe1\x8c\xe1\x1f\x05B@WʼS\n聛LY\x86$\xf6\x01ݝ\xcd\x1d\xe9\x02]\xf4i\xda\xfa\xc2\x0eUr\xc5Dʽdb\x13\xbe\xfe\x1c\xc5\xe1\x84\xcc\xdf&\x93j\x1eK\x04\xba\x06\x16\x85\x0e\x1f\xca\b\x90\x06\x11K\x9d[\rV\xe8E\xd5(\x91\fn\xd4\x10\x9cH\x1cܝX\x94ȁ5\x1b\x8c\xbbz\xf9Ho{\x1d\x112\x90F\xe7\xd8h\xa8\xa1\xf6\x8c\x8cvʲ1\xf9#\x82\xa3\xbe7ed\xd9\x14\xf3\x06\xff\xb7߫i"),
  1385  		KeyID:                  "1",
  1386  		EncryptedDEKSource:     []byte("gafIvwT0ASoKdZ/D1L2SlH73LUMj5qa3hroljfS51wc="),
  1387  		Annotations:            map[string][]byte{"local-kek.kms.kubernetes.io": []byte("encrypted-local-kek")},
  1388  		EncryptedDEKSourceType: kmstypes.EncryptedDEKSourceType_AES_GCM_KEY,
  1389  	}
  1390  	legacyDEKSourceAESGCMKeyObject := &kmstypes.EncryptedObject{}
  1391  	if err := proto.Unmarshal([]byte(strings.TrimPrefix(dekSourceAESGCMKeyData, "k8s:enc:kms:v2:kms-provider:")), legacyDEKSourceAESGCMKeyObject); err != nil {
  1392  		t.Fatal(err)
  1393  	}
  1394  	if err := kmsv2.ValidateEncryptedObject(legacyDEKSourceAESGCMKeyObject); err != nil {
  1395  		t.Fatal(err)
  1396  	}
  1397  	if diff := cmp.Diff(expectedDEKSourceAESGCMKeyObject, legacyDEKSourceAESGCMKeyObject); len(diff) > 0 {
  1398  		t.Errorf("kms v2 legacy encrypted object diff (-want, +got):\n%s", diff)
  1399  	}
  1400  
  1401  	// commit: 855e7c48de7388eb330da0f8d9d2394ee818fb8d
  1402  	// tag: v1.28.0
  1403  	const dekSourceHKDFSHA256XNonceAESGCMSeedName = "dek-source-hkdf-sha256-xnonce-aes-gcm-seed"
  1404  	const dekSourceHKDFSHA256XNonceAESGCMSeedData = "k8s:enc:kms:v2:kms-provider:\n\xd1\x03\x12\x0f\x87\xae\xa2\xa2\xf5J\x11\x06о\x8a\x81\xb6\x15\xdf4H\xa3\xb7i\x93^)\xe5i\xe2\x7f\xfdo\xee\"\x170\xb7\n\xa0\v\xec\xe0\xfa\t#\tƖ\xf6ǧw\x01\xb9\xab\xb3\xf4\xdf\xec\x9eJ\x95$&Z=\x8awAc\xa5\xb2;\t\xd5\xf9\x05\xc1E)\x8b\xb8\x14\xc9\xda\x14{I\rV?\xe5:\xf0BM\x9b\xad\xaaHn>W/Q\xa3\xf5\xba\xe7˚\n\xe7\"\xa7\\p\x8c\xba2\xf2\xb0x<Ą\x88\x9a\xf1\xb5:d=z\xe3\xc3&\x03\x99m\x96\xe7\\\xe3\xa3\xd7i\xb2\f\x84g\xf94\xd2\f\xd6~\xed\xac\xf8\x1b\xc6(,[\xd1\xff\xba\x89ȇD\x02):wTM12\xb5\xfdl\xd2\xf2\x85\x120\xd3\xd9aak\xce\xddI֥\x0fk2\xf6I\xd0\xf9\xc2\xda\vŗ\xd7\u05fb\x83\xd6\xf7\x1a\xbf\x13iH\x8f\xe4\xb3#\xf3\xdf\xc8$y\rL(F\\Xt\x86\xbb\xe5K\x88=\x92\xe9\x04\xf70\x1e\t>\xac;\x9e\xe6\xf0+ۙ4\x1d\x1aV9Մ-g\xf3\xc7Z\x00\xf73\x0e\xe7\xa6\xcf\xc4\xfc\xe0\xb2\x1f\xa0\xbb\x1a\x81\xb3\xe4צ\x7f\xc6\b\x94͉\xad@\xac\x81\x015\x0f\xe8A\xe9B\xfb2\x81o\x9c?*\f\x8c\x15\xa8)\"\xe8\xff\x8d8\xd5!O\x17\xc5㍀\x83\xd3´\xca1;\xf7\xb0\xf4\x90x\xa6\x01\x95\x85\xc0\xaf\xf6\x82Qk\xab\xc1\x82<D\x93\xcf\r\xdb\xdf\x1c\x94\x17Q\xfaS\xe6\xcb\xd4Xƿ\x80\x1d\xc4\x1c\x9dP74\x82JK\vy\xe9)\xbchY\xbe\xcc|\xe4\x97\xdd<;3\x90J\a\xee#\xb2y\xe3\t`\xef\xef\x1f\"k\x8b\x96\xa0\x98\xd9\xffs\xde&\xb7\xa6\x0e\xf1\x7f2ͅb\xe3\xda5\"c\\K\xe21\xa2\xec`\x1b\xe5R\xe6j\b@\x187\xe1\xdb\x04\xf6bNO\x0e\x12\x011\x1a,/+WnKXQEM/AhXICYRNBeGk+WSuB+7OBuSYJTbP66Zyc=\"2\n\x1blocal-kek.kms.kubernetes.io\x12\x13encrypted-local-kek(\x01"
  1405  	dekSourceHKDFSHA256XNonceAESGCMSeedPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", dekSourceHKDFSHA256XNonceAESGCMSeedName, legacyDataNamespace)
  1406  	if _, err := test.writeRawRecordToETCD(dekSourceHKDFSHA256XNonceAESGCMSeedPath, []byte(dekSourceHKDFSHA256XNonceAESGCMSeedData)); err != nil {
  1407  		t.Fatal(err)
  1408  	}
  1409  
  1410  	expectedDEKSourceHKDFSHA256XNonceAESGCMSeedObject := &kmstypes.EncryptedObject{
  1411  		EncryptedData:          []byte("\x12\x0f\x87\xae\xa2\xa2\xf5J\x11\x06о\x8a\x81\xb6\x15\xdf4H\xa3\xb7i\x93^)\xe5i\xe2\x7f\xfdo\xee\"\x170\xb7\n\xa0\v\xec\xe0\xfa\t#\tƖ\xf6ǧw\x01\xb9\xab\xb3\xf4\xdf\xec\x9eJ\x95$&Z=\x8awAc\xa5\xb2;\t\xd5\xf9\x05\xc1E)\x8b\xb8\x14\xc9\xda\x14{I\rV?\xe5:\xf0BM\x9b\xad\xaaHn>W/Q\xa3\xf5\xba\xe7˚\n\xe7\"\xa7\\p\x8c\xba2\xf2\xb0x<Ą\x88\x9a\xf1\xb5:d=z\xe3\xc3&\x03\x99m\x96\xe7\\\xe3\xa3\xd7i\xb2\f\x84g\xf94\xd2\f\xd6~\xed\xac\xf8\x1b\xc6(,[\xd1\xff\xba\x89ȇD\x02):wTM12\xb5\xfdl\xd2\xf2\x85\x120\xd3\xd9aak\xce\xddI֥\x0fk2\xf6I\xd0\xf9\xc2\xda\vŗ\xd7\u05fb\x83\xd6\xf7\x1a\xbf\x13iH\x8f\xe4\xb3#\xf3\xdf\xc8$y\rL(F\\Xt\x86\xbb\xe5K\x88=\x92\xe9\x04\xf70\x1e\t>\xac;\x9e\xe6\xf0+ۙ4\x1d\x1aV9Մ-g\xf3\xc7Z\x00\xf73\x0e\xe7\xa6\xcf\xc4\xfc\xe0\xb2\x1f\xa0\xbb\x1a\x81\xb3\xe4צ\x7f\xc6\b\x94͉\xad@\xac\x81\x015\x0f\xe8A\xe9B\xfb2\x81o\x9c?*\f\x8c\x15\xa8)\"\xe8\xff\x8d8\xd5!O\x17\xc5㍀\x83\xd3´\xca1;\xf7\xb0\xf4\x90x\xa6\x01\x95\x85\xc0\xaf\xf6\x82Qk\xab\xc1\x82<D\x93\xcf\r\xdb\xdf\x1c\x94\x17Q\xfaS\xe6\xcb\xd4Xƿ\x80\x1d\xc4\x1c\x9dP74\x82JK\vy\xe9)\xbchY\xbe\xcc|\xe4\x97\xdd<;3\x90J\a\xee#\xb2y\xe3\t`\xef\xef\x1f\"k\x8b\x96\xa0\x98\xd9\xffs\xde&\xb7\xa6\x0e\xf1\x7f2ͅb\xe3\xda5\"c\\K\xe21\xa2\xec`\x1b\xe5R\xe6j\b@\x187\xe1\xdb\x04\xf6bNO\x0e"),
  1412  		KeyID:                  "1",
  1413  		EncryptedDEKSource:     []byte("/+WnKXQEM/AhXICYRNBeGk+WSuB+7OBuSYJTbP66Zyc="),
  1414  		Annotations:            map[string][]byte{"local-kek.kms.kubernetes.io": []byte("encrypted-local-kek")},
  1415  		EncryptedDEKSourceType: kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED,
  1416  	}
  1417  	legacyDEKSourceHKDFSHA256XNonceAESGCMSeedObject := &kmstypes.EncryptedObject{}
  1418  	if err := proto.Unmarshal([]byte(strings.TrimPrefix(dekSourceHKDFSHA256XNonceAESGCMSeedData, "k8s:enc:kms:v2:kms-provider:")), legacyDEKSourceHKDFSHA256XNonceAESGCMSeedObject); err != nil {
  1419  		t.Fatal(err)
  1420  	}
  1421  	if err := kmsv2.ValidateEncryptedObject(legacyDEKSourceHKDFSHA256XNonceAESGCMSeedObject); err != nil {
  1422  		t.Fatal(err)
  1423  	}
  1424  	if diff := cmp.Diff(expectedDEKSourceHKDFSHA256XNonceAESGCMSeedObject, legacyDEKSourceHKDFSHA256XNonceAESGCMSeedObject); len(diff) > 0 {
  1425  		t.Errorf("kms v2 legacy encrypted object diff (-want, +got):\n%s", diff)
  1426  	}
  1427  
  1428  	ctx := testContext(t)
  1429  
  1430  	legacySecrets, err := test.restClient.CoreV1().Secrets(legacyDataNamespace).List(ctx, metav1.ListOptions{})
  1431  	if err != nil {
  1432  		t.Fatal(err)
  1433  	}
  1434  
  1435  	dekSourceAESGCMKeyTime := metav1.NewTime(time.Date(2023, 9, 1, 11, 56, 49, 0, time.FixedZone("EDT", -4*60*60)))
  1436  	dekSourceHKDFSHA256XNonceAESGCMSeedTime := metav1.NewTime(time.Date(2023, 9, 1, 10, 23, 13, 0, time.FixedZone("EDT", -4*60*60)))
  1437  
  1438  	expectedLegacySecrets := &corev1.SecretList{
  1439  		Items: []corev1.Secret{
  1440  			{
  1441  				ObjectMeta: metav1.ObjectMeta{
  1442  					Name:              dekSourceAESGCMKeyName,
  1443  					Namespace:         legacyDataNamespace,
  1444  					UID:               "1f4a8f7b-01b4-49d1-b898-751eb56937f1",
  1445  					CreationTimestamp: dekSourceAESGCMKeyTime,
  1446  					ManagedFields: []metav1.ManagedFieldsEntry{
  1447  						{
  1448  							Manager:    "___TestKMSv2Provider_in_k8s_io_kubernetes_test_integration_controlplane_transformation.test",
  1449  							Operation:  "Update",
  1450  							APIVersion: "v1",
  1451  							Time:       &dekSourceAESGCMKeyTime,
  1452  							FieldsType: "FieldsV1",
  1453  							FieldsV1:   &metav1.FieldsV1{Raw: []byte(`{"f:data":{".":{},"f:api_key":{}},"f:type":{}}`)},
  1454  						},
  1455  					},
  1456  				},
  1457  				Data: map[string][]byte{
  1458  					secretKey: []byte(secretVal),
  1459  				},
  1460  				Type: corev1.SecretTypeOpaque,
  1461  			},
  1462  			{
  1463  				ObjectMeta: metav1.ObjectMeta{
  1464  					Name:              dekSourceHKDFSHA256XNonceAESGCMSeedName,
  1465  					Namespace:         legacyDataNamespace,
  1466  					UID:               "87c514b4-9c26-4041-ad0d-0d07dca557ed",
  1467  					CreationTimestamp: dekSourceHKDFSHA256XNonceAESGCMSeedTime,
  1468  					ManagedFields: []metav1.ManagedFieldsEntry{
  1469  						{
  1470  							Manager:    "___TestKMSv2Provider_extended_nonce_gcm_in_k8s_io_kubernetes_test_integration_controlplane_transformation.test",
  1471  							Operation:  "Update",
  1472  							APIVersion: "v1",
  1473  							Time:       &dekSourceHKDFSHA256XNonceAESGCMSeedTime,
  1474  							FieldsType: "FieldsV1",
  1475  							FieldsV1:   &metav1.FieldsV1{Raw: []byte(`{"f:data":{".":{},"f:api_key":{}},"f:type":{}}`)},
  1476  						},
  1477  					},
  1478  				},
  1479  				Data: map[string][]byte{
  1480  					secretKey: []byte(secretVal),
  1481  				},
  1482  				Type: corev1.SecretTypeOpaque,
  1483  			},
  1484  		},
  1485  	}
  1486  
  1487  	if diff := cmp.Diff(expectedLegacySecrets, legacySecrets,
  1488  		// resource version is set after decoding based on etcd state - it is not stored in the etcd value
  1489  		cmpopts.IgnoreFields(corev1.Secret{}, "ResourceVersion"),
  1490  		cmpopts.IgnoreFields(metav1.ListMeta{}, "ResourceVersion"),
  1491  	); len(diff) > 0 {
  1492  		t.Errorf("kms v2 legacy secret data diff (-want, +got):\n%s", diff)
  1493  	}
  1494  }
  1495  

View as plain text