storageversionmigrator_test.go

Documentation: k8s.io/kubernetes/test/integration/storageversionmigrator

     1  /*
     2  Copyright 2024 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package storageversionmigrator
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"testing"
    23  	"time"
    24  
    25  	etcd3watcher "k8s.io/apiserver/pkg/storage/etcd3"
    26  	"k8s.io/klog/v2/ktesting"
    27  
    28  	svmv1alpha1 "k8s.io/api/storagemigration/v1alpha1"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
    31  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    32  	clientgofeaturegate "k8s.io/client-go/features"
    33  	"k8s.io/component-base/featuregate"
    34  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    35  	"k8s.io/kubernetes/pkg/features"
    36  )
    37  
    38  // TestStorageVersionMigration is an integration test that verifies storage version migration works.
    39  // This test asserts following scenarios:
    40  // 1. Start API server with encryption at rest and hot reload of encryption config enabled
    41  // 2. Create a secret
    42  // 3. Update encryption config file to add a new key as write key
    43  // 4. Perform Storage Version Migration for secrets
    44  // 5. Verify that the secret is migrated to use the new key
    45  // 6. Verify that the secret is updated with a new resource version
    46  // 7. Perform another Storage Version Migration for secrets
    47  // 8. Verify that the resource version of the secret is not updated. i.e. it was a no-op update
    48  func TestStorageVersionMigration(t *testing.T) {
    49  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionMigrator, true)()
    50  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, featuregate.Feature(clientgofeaturegate.InformerResourceVersion), true)()
    51  
    52  	// this makes the test super responsive. It's set to a default of 1 minute.
    53  	encryptionconfigcontroller.EncryptionConfigFileChangePollDuration = time.Millisecond
    54  
    55  	_, ctx := ktesting.NewTestContext(t)
    56  	ctx, cancel := context.WithCancel(ctx)
    57  	defer cancel()
    58  
    59  	svmTest := svmSetup(ctx, t)
    60  
    61  	// ToDo: try to test with 1000 secrets
    62  	secret, err := svmTest.createSecret(ctx, t, secretName, defaultNamespace)
    63  	if err != nil {
    64  		t.Fatalf("Failed to create secret: %v", err)
    65  	}
    66  
    67  	metricBeforeUpdate := svmTest.getAutomaticReloadSuccessTotal(ctx, t)
    68  	svmTest.updateFile(t, svmTest.filePathForEncryptionConfig, encryptionConfigFileName, []byte(resources["updatedEncryptionConfig"]))
    69  	if !svmTest.isEncryptionConfigFileUpdated(ctx, t, metricBeforeUpdate) {
    70  		t.Fatalf("Failed to update encryption config file")
    71  	}
    72  
    73  	svm, err := svmTest.createSVMResource(
    74  		ctx,
    75  		t,
    76  		svmName,
    77  		svmv1alpha1.GroupVersionResource{
    78  			Group:    "",
    79  			Version:  "v1",
    80  			Resource: "secrets",
    81  		},
    82  	)
    83  	if err != nil {
    84  		t.Fatalf("Failed to create SVM resource: %v", err)
    85  	}
    86  	if !svmTest.waitForResourceMigration(ctx, t, svm.Name, secret.Name, 1) {
    87  		t.Fatalf("Failed to migrate resource %s/%s", secret.Namespace, secret.Name)
    88  	}
    89  
    90  	wantPrefix := "k8s:enc:aescbc:v1:key2"
    91  	etcdSecret, err := svmTest.getRawSecretFromETCD(t, secret.Name, secret.Namespace)
    92  	if err != nil {
    93  		t.Fatalf("Failed to get secret from etcd: %v", err)
    94  	}
    95  	// assert that secret is prefixed with the new key
    96  	if !bytes.HasPrefix(etcdSecret, []byte(wantPrefix)) {
    97  		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, etcdSecret)
    98  	}
    99  
   100  	secretAfterMigration, err := svmTest.client.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
   101  	if err != nil {
   102  		t.Fatalf("Failed to get secret: %v", err)
   103  	}
   104  	// assert that RV is different
   105  	// rv is expected to be different as the secret was re-written to etcd with the new key
   106  	if secret.ResourceVersion == secretAfterMigration.ResourceVersion {
   107  		t.Fatalf("Expected resource version to be different, but got the same, rv before: %s, rv after: %s", secret.ResourceVersion, secretAfterMigration.ResourceVersion)
   108  	}
   109  
   110  	secondSVM, err := svmTest.createSVMResource(
   111  		ctx,
   112  		t,
   113  		secondSVMName,
   114  		svmv1alpha1.GroupVersionResource{
   115  			Group:    "",
   116  			Version:  "v1",
   117  			Resource: "secrets",
   118  		},
   119  	)
   120  	if err != nil {
   121  		t.Fatalf("Failed to create SVM resource: %v", err)
   122  	}
   123  	if !svmTest.waitForResourceMigration(ctx, t, secondSVM.Name, secretAfterMigration.Name, 2) {
   124  		t.Fatalf("Failed to migrate resource %s/%s", secretAfterMigration.Namespace, secretAfterMigration.Name)
   125  	}
   126  
   127  	secretAfterSecondMigration, err := svmTest.client.CoreV1().Secrets(secretAfterMigration.Namespace).Get(ctx, secretAfterMigration.Name, metav1.GetOptions{})
   128  	if err != nil {
   129  		t.Fatalf("Failed to get secret: %v", err)
   130  	}
   131  	// assert that RV is same
   132  	if secretAfterMigration.ResourceVersion != secretAfterSecondMigration.ResourceVersion {
   133  		t.Fatalf("Expected resource version to be same, but got different, rv before: %s, rv after: %s", secretAfterMigration.ResourceVersion, secretAfterSecondMigration.ResourceVersion)
   134  	}
   135  }
   136  
   137  // TestStorageVersionMigrationWithCRD is an integration test that verifies storage version migration works with CRD.
   138  // This test asserts following scenarios:
   139  // 1. CRD is created with version v1 (serving and storage)
   140  // 2. Verify that CRs are written and stored as v1
   141  // 3. Update CRD to introduce v2 (for serving only), and a conversion webhook is added
   142  // 4. Verify that CRs are written to v2 but are stored as v1
   143  // 5. CRD storage version is changed from v1 to v2
   144  // 6. Verify that CR written as either v1 or v2 version are stored as v2
   145  // 7. Perform Storage Version Migration to migrate all v1 CRs to v2
   146  // 8. CRD is updated to no longer serve v1
   147  // 9. Shutdown conversion webhook
   148  // 10. Verify RV and Generations of CRs
   149  // 11. Verify the list of CRs at v2 works
   150  func TestStorageVersionMigrationWithCRD(t *testing.T) {
   151  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionMigrator, true)()
   152  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, featuregate.Feature(clientgofeaturegate.InformerResourceVersion), true)()
   153  	// decode errors are expected when using conversation webhooks
   154  	etcd3watcher.TestOnlySetFatalOnDecodeError(false)
   155  	defer etcd3watcher.TestOnlySetFatalOnDecodeError(true)
   156  
   157  	_, ctx := ktesting.NewTestContext(t)
   158  	ctx, cancel := context.WithCancel(ctx)
   159  	defer cancel()
   160  
   161  	crVersions := make(map[string]versions)
   162  
   163  	svmTest := svmSetup(ctx, t)
   164  	certCtx := svmTest.setupServerCert(t)
   165  
   166  	// create CRD with v1 serving and storage
   167  	crd := svmTest.createCRD(t, crdName, crdGroup, certCtx, v1CRDVersion)
   168  
   169  	// create CR
   170  	cr1 := svmTest.createCR(ctx, t, "cr1", "v1")
   171  	if ok := svmTest.isCRStoredAtVersion(t, "v1", cr1.GetName()); !ok {
   172  		t.Fatalf("CR not stored at version v1")
   173  	}
   174  	crVersions[cr1.GetName()] = versions{
   175  		generation:  cr1.GetGeneration(),
   176  		rv:          cr1.GetResourceVersion(),
   177  		isRVUpdated: true,
   178  	}
   179  
   180  	// add conversion webhook
   181  	shutdownServer := svmTest.createConversionWebhook(ctx, t, certCtx)
   182  
   183  	// add v2 for serving only
   184  	svmTest.updateCRD(ctx, t, crd.Name, v2CRDVersion)
   185  
   186  	// create another CR
   187  	cr2 := svmTest.createCR(ctx, t, "cr2", "v2")
   188  	if ok := svmTest.isCRStoredAtVersion(t, "v1", cr2.GetName()); !ok {
   189  		t.Fatalf("CR not stored at version v1")
   190  	}
   191  	crVersions[cr2.GetName()] = versions{
   192  		generation:  cr2.GetGeneration(),
   193  		rv:          cr2.GetResourceVersion(),
   194  		isRVUpdated: true,
   195  	}
   196  
   197  	// add v2 as storage version
   198  	svmTest.updateCRD(ctx, t, crd.Name, v2StorageCRDVersion)
   199  
   200  	// create CR with v1
   201  	cr3 := svmTest.createCR(ctx, t, "cr3", "v1")
   202  	if ok := svmTest.isCRStoredAtVersion(t, "v2", cr3.GetName()); !ok {
   203  		t.Fatalf("CR not stored at version v2")
   204  	}
   205  	crVersions[cr3.GetName()] = versions{
   206  		generation:  cr3.GetGeneration(),
   207  		rv:          cr3.GetResourceVersion(),
   208  		isRVUpdated: false,
   209  	}
   210  
   211  	// create CR with v2
   212  	cr4 := svmTest.createCR(ctx, t, "cr4", "v2")
   213  	if ok := svmTest.isCRStoredAtVersion(t, "v2", cr4.GetName()); !ok {
   214  		t.Fatalf("CR not stored at version v2")
   215  	}
   216  	crVersions[cr4.GetName()] = versions{
   217  		generation:  cr4.GetGeneration(),
   218  		rv:          cr4.GetResourceVersion(),
   219  		isRVUpdated: false,
   220  	}
   221  
   222  	// verify cr1 ans cr2 are still stored at v1
   223  	if ok := svmTest.isCRStoredAtVersion(t, "v1", cr1.GetName()); !ok {
   224  		t.Fatalf("CR not stored at version v1")
   225  	}
   226  	if ok := svmTest.isCRStoredAtVersion(t, "v1", cr2.GetName()); !ok {
   227  		t.Fatalf("CR not stored at version v1")
   228  	}
   229  
   230  	// migrate CRs from v1 to v2
   231  	svm, err := svmTest.createSVMResource(
   232  		ctx, t, "crdsvm",
   233  		svmv1alpha1.GroupVersionResource{
   234  			Group:    crd.Spec.Group,
   235  			Version:  "v1",
   236  			Resource: crd.Spec.Names.Plural,
   237  		})
   238  	if err != nil {
   239  		t.Fatalf("Failed to create SVM resource: %v", err)
   240  	}
   241  	if ok := svmTest.isCRDMigrated(ctx, t, svm.Name); !ok {
   242  		t.Fatalf("CRD not migrated")
   243  	}
   244  
   245  	// assert all the CRs are stored in the etcd at correct version
   246  	if ok := svmTest.isCRStoredAtVersion(t, "v2", cr1.GetName()); !ok {
   247  		t.Fatalf("CR not stored at version v2")
   248  	}
   249  	if ok := svmTest.isCRStoredAtVersion(t, "v2", cr2.GetName()); !ok {
   250  		t.Fatalf("CR not stored at version v2")
   251  	}
   252  	if ok := svmTest.isCRStoredAtVersion(t, "v2", cr3.GetName()); !ok {
   253  		t.Fatalf("CR not stored at version v2")
   254  	}
   255  	if ok := svmTest.isCRStoredAtVersion(t, "v2", cr4.GetName()); !ok {
   256  		t.Fatalf("CR not stored at version v2")
   257  	}
   258  
   259  	// update CRD to v1 not serving and storage followed by webhook shutdown
   260  	svmTest.updateCRD(ctx, t, crd.Name, v1NotServingCRDVersion)
   261  	shutdownServer()
   262  
   263  	// assert RV and Generations of CRs
   264  	svmTest.validateRVAndGeneration(ctx, t, crVersions)
   265  
   266  	// assert v2 CRs can be listed
   267  	if err := svmTest.listCR(ctx, t, "v2"); err != nil {
   268  		t.Fatalf("Failed to list CRs at version v2: %v", err)
   269  	}
   270  }
   271
View as plain text