csi_snapshot.go

Documentation: k8s.io/kubernetes/test/e2e/storage/csi_mock

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package csi_mock
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	"github.com/onsi/ginkgo/v2"
    25  	"google.golang.org/grpc/codes"
    26  	"google.golang.org/grpc/status"
    27  	v1 "k8s.io/api/core/v1"
    28  	storagev1 "k8s.io/api/storage/v1"
    29  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    32  	clientset "k8s.io/client-go/kubernetes"
    33  	"k8s.io/kubernetes/test/e2e/feature"
    34  	"k8s.io/kubernetes/test/e2e/framework"
    35  	e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
    36  	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
    37  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    38  	"k8s.io/kubernetes/test/e2e/storage/drivers"
    39  	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
    40  	"k8s.io/kubernetes/test/e2e/storage/utils"
    41  	admissionapi "k8s.io/pod-security-admission/api"
    42  )
    43  
    44  var _ = utils.SIGDescribe("CSI Mock volume snapshot", func() {
    45  	f := framework.NewDefaultFramework("csi-mock-volumes-snapshot")
    46  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    47  	m := newMockDriverSetup(f)
    48  
    49  	f.Context("CSI Volume Snapshots", feature.VolumeSnapshotDataSource, func() {
    50  		tests := []struct {
    51  			name               string
    52  			createSnapshotHook func(counter int64) error
    53  		}{
    54  			{
    55  				name: "volumesnapshotcontent and pvc in Bound state with deletion timestamp set should not get deleted while snapshot finalizer exists",
    56  				createSnapshotHook: func(counter int64) error {
    57  					if counter < 8 {
    58  						return status.Error(codes.DeadlineExceeded, "fake error")
    59  					}
    60  					return nil
    61  				},
    62  			},
    63  		}
    64  		for _, test := range tests {
    65  			test := test
    66  			ginkgo.It(test.name, func(ctx context.Context) {
    67  				var hooks *drivers.Hooks
    68  				if test.createSnapshotHook != nil {
    69  					hooks = createPreHook("CreateSnapshot", test.createSnapshotHook)
    70  				}
    71  				m.init(ctx, testParameters{
    72  					disableAttach:  true,
    73  					registerDriver: true,
    74  					enableSnapshot: true,
    75  					hooks:          hooks,
    76  				})
    77  				sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver)
    78  				if !ok {
    79  					e2eskipper.Skipf("mock driver %s does not support snapshots -- skipping", m.driver.GetDriverInfo().Name)
    80  
    81  				}
    82  				ctx, cancel := context.WithTimeout(ctx, csiPodRunningTimeout)
    83  				defer cancel()
    84  				ginkgo.DeferCleanup(m.cleanup)
    85  
    86  				sc := m.driver.GetDynamicProvisionStorageClass(ctx, m.config, "")
    87  				ginkgo.By("Creating storage class")
    88  				class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{})
    89  				framework.ExpectNoError(err, "Failed to create class: %v", err)
    90  				m.sc[class.Name] = class
    91  				claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
    92  					// Use static name so that the volumesnapshot can be created before the pvc.
    93  					Name:             "snapshot-test-pvc",
    94  					StorageClassName: &(class.Name),
    95  				}, f.Namespace.Name)
    96  
    97  				ginkgo.By("Creating snapshot")
    98  				// TODO: Test VolumeSnapshots with Retain policy
    99  				parameters := map[string]string{}
   100  				snapshotClass, snapshot := storageframework.CreateSnapshot(ctx, sDriver, m.config, storageframework.DynamicSnapshotDelete, claim.Name, claim.Namespace, f.Timeouts, parameters)
   101  				framework.ExpectNoError(err, "failed to create snapshot")
   102  				m.vsc[snapshotClass.GetName()] = snapshotClass
   103  				volumeSnapshotName := snapshot.GetName()
   104  
   105  				ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", claim.Namespace, claim.Name))
   106  				claim, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), claim, metav1.CreateOptions{})
   107  				framework.ExpectNoError(err, "Failed to create claim: %v", err)
   108  
   109  				ginkgo.By(fmt.Sprintf("Wait for finalizer to be added to claim %s/%s", claim.Namespace, claim.Name))
   110  				err = e2epv.WaitForPVCFinalizer(ctx, m.cs, claim.Name, claim.Namespace, pvcAsSourceProtectionFinalizer, 1*time.Millisecond, 1*time.Minute)
   111  				framework.ExpectNoError(err)
   112  
   113  				ginkgo.By("Wait for PVC to be Bound")
   114  				_, err = e2epv.WaitForPVClaimBoundPhase(ctx, m.cs, []*v1.PersistentVolumeClaim{claim}, 1*time.Minute)
   115  				framework.ExpectNoError(err, "Failed to create claim: %v", err)
   116  
   117  				ginkgo.By(fmt.Sprintf("Delete PVC %s", claim.Name))
   118  				err = e2epv.DeletePersistentVolumeClaim(ctx, m.cs, claim.Name, claim.Namespace)
   119  				framework.ExpectNoError(err, "failed to delete pvc")
   120  
   121  				ginkgo.By("Get PVC from API server and verify deletion timestamp is set")
   122  				claim, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(context.TODO(), claim.Name, metav1.GetOptions{})
   123  				if err != nil {
   124  					if !apierrors.IsNotFound(err) {
   125  						framework.ExpectNoError(err, "Failed to get claim: %v", err)
   126  					}
   127  					framework.Logf("PVC not found. Continuing to test VolumeSnapshotContent finalizer")
   128  				} else if claim.DeletionTimestamp == nil {
   129  					framework.Failf("Expected deletion timestamp to be set on PVC: %v", claim)
   130  				}
   131  
   132  				ginkgo.By(fmt.Sprintf("Get VolumeSnapshotContent bound to VolumeSnapshot %s", snapshot.GetName()))
   133  				snapshotContent := utils.GetSnapshotContentFromSnapshot(ctx, m.config.Framework.DynamicClient, snapshot, f.Timeouts.SnapshotCreate)
   134  				volumeSnapshotContentName := snapshotContent.GetName()
   135  
   136  				ginkgo.By(fmt.Sprintf("Verify VolumeSnapshotContent %s contains finalizer %s", snapshot.GetName(), volumeSnapshotContentFinalizer))
   137  				err = utils.WaitForGVRFinalizer(ctx, m.config.Framework.DynamicClient, utils.SnapshotContentGVR, volumeSnapshotContentName, "", volumeSnapshotContentFinalizer, 1*time.Millisecond, 1*time.Minute)
   138  				framework.ExpectNoError(err)
   139  
   140  				ginkgo.By(fmt.Sprintf("Delete VolumeSnapshotContent %s", snapshotContent.GetName()))
   141  				err = m.config.Framework.DynamicClient.Resource(utils.SnapshotContentGVR).Delete(ctx, snapshotContent.GetName(), metav1.DeleteOptions{})
   142  				framework.ExpectNoError(err, "Failed to delete snapshotcontent: %v", err)
   143  
   144  				ginkgo.By("Get VolumeSnapshotContent from API server and verify deletion timestamp is set")
   145  				snapshotContent, err = m.config.Framework.DynamicClient.Resource(utils.SnapshotContentGVR).Get(context.TODO(), snapshotContent.GetName(), metav1.GetOptions{})
   146  				framework.ExpectNoError(err)
   147  
   148  				if snapshotContent.GetDeletionTimestamp() == nil {
   149  					framework.Failf("Expected deletion timestamp to be set on snapshotcontent")
   150  				}
   151  
   152  				// If the claim is non existent, the Get() call on the API server returns
   153  				// an non-nil claim object with all fields unset.
   154  				// Refer https://github.com/kubernetes/kubernetes/pull/99167#issuecomment-781670012
   155  				if claim != nil && claim.Spec.VolumeName != "" {
   156  					ginkgo.By(fmt.Sprintf("Wait for PV %s to be deleted", claim.Spec.VolumeName))
   157  					err = e2epv.WaitForPersistentVolumeDeleted(ctx, m.cs, claim.Spec.VolumeName, framework.Poll, 3*time.Minute)
   158  					framework.ExpectNoError(err, fmt.Sprintf("failed to delete PV %s", claim.Spec.VolumeName))
   159  				}
   160  
   161  				ginkgo.By(fmt.Sprintf("Verify VolumeSnapshot %s contains finalizer %s", snapshot.GetName(), volumeSnapshotBoundFinalizer))
   162  				err = utils.WaitForGVRFinalizer(ctx, m.config.Framework.DynamicClient, utils.SnapshotGVR, volumeSnapshotName, f.Namespace.Name, volumeSnapshotBoundFinalizer, 1*time.Millisecond, 1*time.Minute)
   163  				framework.ExpectNoError(err)
   164  
   165  				ginkgo.By("Delete VolumeSnapshot")
   166  				err = utils.DeleteAndWaitSnapshot(ctx, m.config.Framework.DynamicClient, f.Namespace.Name, volumeSnapshotName, framework.Poll, framework.SnapshotDeleteTimeout)
   167  				framework.ExpectNoError(err, fmt.Sprintf("failed to delete VolumeSnapshot %s", volumeSnapshotName))
   168  
   169  				ginkgo.By(fmt.Sprintf("Wait for VolumeSnapshotContent %s to be deleted", volumeSnapshotContentName))
   170  				err = utils.WaitForGVRDeletion(ctx, m.config.Framework.DynamicClient, utils.SnapshotContentGVR, volumeSnapshotContentName, framework.Poll, framework.SnapshotDeleteTimeout)
   171  				framework.ExpectNoError(err, fmt.Sprintf("failed to delete VolumeSnapshotContent %s", volumeSnapshotContentName))
   172  			})
   173  		}
   174  	})
   175  
   176  	f.Context("CSI Volume Snapshots secrets", feature.VolumeSnapshotDataSource, func() {
   177  
   178  		var (
   179  			// CSISnapshotterSecretName is the name of the secret to be created
   180  			CSISnapshotterSecretName string = "snapshot-secret"
   181  
   182  			// CSISnapshotterSecretNameAnnotation is the annotation key for the CSI snapshotter secret name in VolumeSnapshotClass.parameters
   183  			CSISnapshotterSecretNameAnnotation string = "csi.storage.k8s.io/snapshotter-secret-name"
   184  
   185  			// CSISnapshotterSecretNamespaceAnnotation is the annotation key for the CSI snapshotter secret namespace in VolumeSnapshotClass.parameters
   186  			CSISnapshotterSecretNamespaceAnnotation string = "csi.storage.k8s.io/snapshotter-secret-namespace"
   187  
   188  			// anotations holds the annotations object
   189  			annotations interface{}
   190  		)
   191  
   192  		tests := []struct {
   193  			name               string
   194  			createSnapshotHook func(counter int64) error
   195  		}{
   196  			{
   197  				// volume snapshot should be created using secrets successfully even if there is a failure in the first few attempts,
   198  				name: "volume snapshot create/delete with secrets",
   199  				// Fail the first 8 calls to create snapshot and succeed the  9th call.
   200  				createSnapshotHook: func(counter int64) error {
   201  					if counter < 8 {
   202  						return status.Error(codes.DeadlineExceeded, "fake error")
   203  					}
   204  					return nil
   205  				},
   206  			},
   207  		}
   208  		for _, test := range tests {
   209  			test := test
   210  			ginkgo.It(test.name, func(ctx context.Context) {
   211  				hooks := createPreHook("CreateSnapshot", test.createSnapshotHook)
   212  				m.init(ctx, testParameters{
   213  					disableAttach:  true,
   214  					registerDriver: true,
   215  					enableSnapshot: true,
   216  					hooks:          hooks,
   217  				})
   218  
   219  				sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver)
   220  				if !ok {
   221  					e2eskipper.Skipf("mock driver does not support snapshots -- skipping")
   222  				}
   223  				ginkgo.DeferCleanup(m.cleanup)
   224  
   225  				var sc *storagev1.StorageClass
   226  				if dDriver, ok := m.driver.(storageframework.DynamicPVTestDriver); ok {
   227  					sc = dDriver.GetDynamicProvisionStorageClass(ctx, m.config, "")
   228  				}
   229  				ginkgo.By("Creating storage class")
   230  				class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{})
   231  				framework.ExpectNoError(err, "Failed to create storage class: %v", err)
   232  				m.sc[class.Name] = class
   233  				pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   234  					Name:             "snapshot-test-pvc",
   235  					StorageClassName: &(class.Name),
   236  				}, f.Namespace.Name)
   237  
   238  				ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", pvc.Namespace, pvc.Name))
   239  				pvc, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
   240  				framework.ExpectNoError(err, "Failed to create claim: %v", err)
   241  
   242  				ginkgo.By("Wait for PVC to be Bound")
   243  				_, err = e2epv.WaitForPVClaimBoundPhase(ctx, m.cs, []*v1.PersistentVolumeClaim{pvc}, 1*time.Minute)
   244  				framework.ExpectNoError(err, "Failed to create claim: %v", err)
   245  
   246  				m.pvcs = append(m.pvcs, pvc)
   247  
   248  				ginkgo.By("Creating Secret")
   249  				secret := &v1.Secret{
   250  					ObjectMeta: metav1.ObjectMeta{
   251  						Namespace: f.Namespace.Name,
   252  						Name:      CSISnapshotterSecretName,
   253  					},
   254  					Data: map[string][]byte{
   255  						"secret-data": []byte("secret-value-1"),
   256  					},
   257  				}
   258  
   259  				if secret, err := m.cs.CoreV1().Secrets(f.Namespace.Name).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil {
   260  					framework.Failf("unable to create test secret %s: %v", secret.Name, err)
   261  				}
   262  
   263  				ginkgo.By("Creating snapshot with secrets")
   264  				parameters := map[string]string{
   265  					CSISnapshotterSecretNameAnnotation:      CSISnapshotterSecretName,
   266  					CSISnapshotterSecretNamespaceAnnotation: f.Namespace.Name,
   267  				}
   268  
   269  				_, snapshot := storageframework.CreateSnapshot(ctx, sDriver, m.config, storageframework.DynamicSnapshotDelete, pvc.Name, pvc.Namespace, f.Timeouts, parameters)
   270  				framework.ExpectNoError(err, "failed to create snapshot")
   271  				snapshotcontent := utils.GetSnapshotContentFromSnapshot(ctx, m.config.Framework.DynamicClient, snapshot, f.Timeouts.SnapshotCreate)
   272  				if annotations, ok = snapshotcontent.Object["metadata"].(map[string]interface{})["annotations"]; !ok {
   273  					framework.Failf("Unable to get volume snapshot content annotations")
   274  				}
   275  
   276  				// checks if delete snapshot secrets annotation is applied to the VolumeSnapshotContent.
   277  				checkDeleteSnapshotSecrets(m.cs, annotations)
   278  
   279  				// delete the snapshot and check if the snapshot is deleted.
   280  				deleteSnapshot(m.cs, m.config, snapshot)
   281  			})
   282  		}
   283  	})
   284  
   285  	f.Context("CSI Snapshot Controller metrics", feature.VolumeSnapshotDataSource, func() {
   286  		tests := []struct {
   287  			name    string
   288  			pattern storageframework.TestPattern
   289  		}{
   290  			{
   291  				name:    "snapshot controller should emit dynamic CreateSnapshot, CreateSnapshotAndReady, and DeleteSnapshot metrics",
   292  				pattern: storageframework.DynamicSnapshotDelete,
   293  			},
   294  			{
   295  				name:    "snapshot controller should emit pre-provisioned CreateSnapshot, CreateSnapshotAndReady, and DeleteSnapshot metrics",
   296  				pattern: storageframework.PreprovisionedSnapshotDelete,
   297  			},
   298  		}
   299  		for _, test := range tests {
   300  			test := test
   301  			ginkgo.It(test.name, func(ctx context.Context) {
   302  				m.init(ctx, testParameters{
   303  					disableAttach:  true,
   304  					registerDriver: true,
   305  					enableSnapshot: true,
   306  				})
   307  
   308  				sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver)
   309  				if !ok {
   310  					e2eskipper.Skipf("mock driver does not support snapshots -- skipping")
   311  				}
   312  				ginkgo.DeferCleanup(m.cleanup)
   313  
   314  				metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, m.config.Framework.ClientSet, nil, f.ClientConfig(), false, false, false, false, false, true)
   315  				if err != nil {
   316  					framework.Failf("Error creating metrics grabber : %v", err)
   317  				}
   318  
   319  				// Grab initial metrics - if this fails, snapshot controller metrics are not setup. Skip in this case.
   320  				_, err = metricsGrabber.GrabFromSnapshotController(ctx, framework.TestContext.SnapshotControllerPodName, framework.TestContext.SnapshotControllerHTTPPort)
   321  				if err != nil {
   322  					e2eskipper.Skipf("Snapshot controller metrics not found -- skipping")
   323  				}
   324  
   325  				ginkgo.By("getting all initial metric values")
   326  				metricsTestConfig := newSnapshotMetricsTestConfig("snapshot_controller_operation_total_seconds_count",
   327  					"count",
   328  					m.config.GetUniqueDriverName(),
   329  					"CreateSnapshot",
   330  					"success",
   331  					"",
   332  					test.pattern)
   333  				createSnapshotMetrics := newSnapshotControllerMetrics(metricsTestConfig, metricsGrabber)
   334  				originalCreateSnapshotCount, _ := createSnapshotMetrics.getSnapshotControllerMetricValue(ctx)
   335  				metricsTestConfig.operationName = "CreateSnapshotAndReady"
   336  				createSnapshotAndReadyMetrics := newSnapshotControllerMetrics(metricsTestConfig, metricsGrabber)
   337  				originalCreateSnapshotAndReadyCount, _ := createSnapshotAndReadyMetrics.getSnapshotControllerMetricValue(ctx)
   338  
   339  				metricsTestConfig.operationName = "DeleteSnapshot"
   340  				deleteSnapshotMetrics := newSnapshotControllerMetrics(metricsTestConfig, metricsGrabber)
   341  				originalDeleteSnapshotCount, _ := deleteSnapshotMetrics.getSnapshotControllerMetricValue(ctx)
   342  
   343  				ginkgo.By("Creating storage class")
   344  				var sc *storagev1.StorageClass
   345  				if dDriver, ok := m.driver.(storageframework.DynamicPVTestDriver); ok {
   346  					sc = dDriver.GetDynamicProvisionStorageClass(ctx, m.config, "")
   347  				}
   348  				class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{})
   349  				framework.ExpectNoError(err, "Failed to create storage class: %v", err)
   350  				m.sc[class.Name] = class
   351  				pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   352  					Name:             "snapshot-test-pvc",
   353  					StorageClassName: &(class.Name),
   354  				}, f.Namespace.Name)
   355  
   356  				ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", pvc.Namespace, pvc.Name))
   357  				pvc, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
   358  				framework.ExpectNoError(err, "Failed to create claim: %v", err)
   359  
   360  				ginkgo.By("Wait for PVC to be Bound")
   361  				_, err = e2epv.WaitForPVClaimBoundPhase(ctx, m.cs, []*v1.PersistentVolumeClaim{pvc}, 1*time.Minute)
   362  				framework.ExpectNoError(err, "Failed to create claim: %v", err)
   363  
   364  				ginkgo.By("Creating snapshot")
   365  				parameters := map[string]string{}
   366  				sr := storageframework.CreateSnapshotResource(ctx, sDriver, m.config, test.pattern, pvc.Name, pvc.Namespace, f.Timeouts, parameters)
   367  				framework.ExpectNoError(err, "failed to create snapshot")
   368  
   369  				ginkgo.By("Checking for CreateSnapshot metrics")
   370  				createSnapshotMetrics.waitForSnapshotControllerMetric(ctx, originalCreateSnapshotCount+1.0, f.Timeouts.SnapshotControllerMetrics)
   371  
   372  				ginkgo.By("Checking for CreateSnapshotAndReady metrics")
   373  				err = utils.WaitForSnapshotReady(ctx, m.config.Framework.DynamicClient, pvc.Namespace, sr.Vs.GetName(), framework.Poll, f.Timeouts.SnapshotCreate)
   374  				framework.ExpectNoError(err, "failed to wait for snapshot ready")
   375  				createSnapshotAndReadyMetrics.waitForSnapshotControllerMetric(ctx, originalCreateSnapshotAndReadyCount+1.0, f.Timeouts.SnapshotControllerMetrics)
   376  
   377  				// delete the snapshot and check if the snapshot is deleted
   378  				deleteSnapshot(m.cs, m.config, sr.Vs)
   379  
   380  				ginkgo.By("check for delete metrics")
   381  				metricsTestConfig.operationName = "DeleteSnapshot"
   382  				deleteSnapshotMetrics.waitForSnapshotControllerMetric(ctx, originalDeleteSnapshotCount+1.0, f.Timeouts.SnapshotControllerMetrics)
   383  			})
   384  		}
   385  	})
   386  })
   387  
   388  // checkDeleteSnapshotSecrets checks if delete snapshot secrets annotation is applied to the VolumeSnapshotContent.
   389  func checkDeleteSnapshotSecrets(cs clientset.Interface, annotations interface{}) error {
   390  	ginkgo.By("checking if delete snapshot secrets annotation is applied to the VolumeSnapshotContent")
   391  
   392  	var (
   393  		annDeletionSecretName      string
   394  		annDeletionSecretNamespace string
   395  		ok                         bool
   396  		err                        error
   397  
   398  		// CSISnapshotterDeleteSecretNameAnnotation is the annotation key for the CSI snapshotter delete secret name in VolumeSnapshotClass.parameters
   399  		CSISnapshotterDeleteSecretNameAnnotation string = "snapshot.storage.kubernetes.io/deletion-secret-name"
   400  
   401  		// CSISnapshotterDeleteSecretNamespaceAnnotation is the annotation key for the CSI snapshotter delete secret namespace in VolumeSnapshotClass.parameters
   402  		CSISnapshotterDeleteSecretNamespaceAnnotation string = "snapshot.storage.kubernetes.io/deletion-secret-namespace"
   403  	)
   404  
   405  	annotationsObj, ok := annotations.(map[string]interface{})
   406  	if !ok {
   407  		framework.Failf("failed to get annotations from annotations object")
   408  	}
   409  
   410  	if annDeletionSecretName, ok = annotationsObj[CSISnapshotterDeleteSecretNameAnnotation].(string); !ok {
   411  		framework.Failf("unable to get secret annotation name")
   412  	}
   413  	if annDeletionSecretNamespace, ok = annotationsObj[CSISnapshotterDeleteSecretNamespaceAnnotation].(string); !ok {
   414  		framework.Failf("unable to get secret annotation namespace")
   415  	}
   416  
   417  	// verify if secrets exists
   418  	if _, err = cs.CoreV1().Secrets(annDeletionSecretNamespace).Get(context.TODO(), annDeletionSecretName, metav1.GetOptions{}); err != nil {
   419  		framework.Failf("unable to get test secret %s: %v", annDeletionSecretName, err)
   420  	}
   421  
   422  	return err
   423  }
   424  
   425  func deleteSnapshot(cs clientset.Interface, config *storageframework.PerTestConfig, snapshot *unstructured.Unstructured) {
   426  	// delete the given snapshot
   427  	dc := config.Framework.DynamicClient
   428  	err := dc.Resource(utils.SnapshotGVR).Namespace(snapshot.GetNamespace()).Delete(context.TODO(), snapshot.GetName(), metav1.DeleteOptions{})
   429  	framework.ExpectNoError(err)
   430  
   431  	// check if the snapshot is deleted
   432  	_, err = dc.Resource(utils.SnapshotGVR).Get(context.TODO(), snapshot.GetName(), metav1.GetOptions{})
   433  	framework.ExpectError(err)
   434  }
   435  
   436  type snapshotMetricsTestConfig struct {
   437  	// expected values
   438  	metricName      string
   439  	metricType      string
   440  	driverName      string
   441  	operationName   string
   442  	operationStatus string
   443  	snapshotType    string
   444  	le              string
   445  }
   446  
   447  type snapshotControllerMetrics struct {
   448  	// configuration for metric
   449  	cfg            snapshotMetricsTestConfig
   450  	metricsGrabber *e2emetrics.Grabber
   451  
   452  	// results
   453  	countMetrics  map[string]float64
   454  	sumMetrics    map[string]float64
   455  	bucketMetrics map[string]float64
   456  }
   457  
   458  func newSnapshotMetricsTestConfig(metricName, metricType, driverName, operationName, operationStatus, le string, pattern storageframework.TestPattern) snapshotMetricsTestConfig {
   459  	var snapshotType string
   460  	switch pattern.SnapshotType {
   461  	case storageframework.DynamicCreatedSnapshot:
   462  		snapshotType = "dynamic"
   463  
   464  	case storageframework.PreprovisionedCreatedSnapshot:
   465  		snapshotType = "pre-provisioned"
   466  
   467  	default:
   468  		framework.Failf("invalid snapshotType: %v", pattern.SnapshotType)
   469  	}
   470  
   471  	return snapshotMetricsTestConfig{
   472  		metricName:      metricName,
   473  		metricType:      metricType,
   474  		driverName:      driverName,
   475  		operationName:   operationName,
   476  		operationStatus: operationStatus,
   477  		snapshotType:    snapshotType,
   478  		le:              le,
   479  	}
   480  }
   481  
   482  func newSnapshotControllerMetrics(cfg snapshotMetricsTestConfig, metricsGrabber *e2emetrics.Grabber) *snapshotControllerMetrics {
   483  	return &snapshotControllerMetrics{
   484  		cfg:            cfg,
   485  		metricsGrabber: metricsGrabber,
   486  
   487  		countMetrics:  make(map[string]float64),
   488  		sumMetrics:    make(map[string]float64),
   489  		bucketMetrics: make(map[string]float64),
   490  	}
   491  }
   492  
   493  func (scm *snapshotControllerMetrics) waitForSnapshotControllerMetric(ctx context.Context, expectedValue float64, timeout time.Duration) {
   494  	metricKey := scm.getMetricKey()
   495  	if successful := utils.WaitUntil(10*time.Second, timeout, func() bool {
   496  		// get metric value
   497  		actualValue, err := scm.getSnapshotControllerMetricValue(ctx)
   498  		if err != nil {
   499  			return false
   500  		}
   501  
   502  		// Another operation could have finished from a previous test,
   503  		// so we check if we have at least the expected value.
   504  		if actualValue < expectedValue {
   505  			return false
   506  		}
   507  
   508  		return true
   509  	}); successful {
   510  		return
   511  	}
   512  
   513  	scm.showMetricsFailure(metricKey)
   514  	framework.Failf("Unable to get valid snapshot controller metrics after %v", timeout)
   515  }
   516  
   517  func (scm *snapshotControllerMetrics) getSnapshotControllerMetricValue(ctx context.Context) (float64, error) {
   518  	metricKey := scm.getMetricKey()
   519  
   520  	// grab and parse into readable format
   521  	err := scm.grabSnapshotControllerMetrics(ctx)
   522  	if err != nil {
   523  		return 0, err
   524  	}
   525  
   526  	metrics := scm.getMetricsTable()
   527  	actual, ok := metrics[metricKey]
   528  	if !ok {
   529  		return 0, fmt.Errorf("did not find metric for key %s", metricKey)
   530  	}
   531  
   532  	return actual, nil
   533  }
   534  
   535  func (scm *snapshotControllerMetrics) getMetricsTable() map[string]float64 {
   536  	var metrics map[string]float64
   537  	switch scm.cfg.metricType {
   538  	case "count":
   539  		metrics = scm.countMetrics
   540  
   541  	case "sum":
   542  		metrics = scm.sumMetrics
   543  
   544  	case "bucket":
   545  		metrics = scm.bucketMetrics
   546  	}
   547  
   548  	return metrics
   549  }
   550  
   551  func (scm *snapshotControllerMetrics) showMetricsFailure(metricKey string) {
   552  	framework.Logf("failed to find metric key %s inside of the following metrics:", metricKey)
   553  
   554  	metrics := scm.getMetricsTable()
   555  	for k, v := range metrics {
   556  		framework.Logf("%s: %v", k, v)
   557  	}
   558  }
   559  
   560  func (scm *snapshotControllerMetrics) grabSnapshotControllerMetrics(ctx context.Context) error {
   561  	// pull all metrics
   562  	metrics, err := scm.metricsGrabber.GrabFromSnapshotController(ctx, framework.TestContext.SnapshotControllerPodName, framework.TestContext.SnapshotControllerHTTPPort)
   563  	if err != nil {
   564  		return err
   565  	}
   566  
   567  	for method, samples := range metrics {
   568  
   569  		for _, sample := range samples {
   570  			operationName := string(sample.Metric["operation_name"])
   571  			driverName := string(sample.Metric["driver_name"])
   572  			operationStatus := string(sample.Metric["operation_status"])
   573  			snapshotType := string(sample.Metric["snapshot_type"])
   574  			le := string(sample.Metric["le"])
   575  			key := snapshotMetricKey(scm.cfg.metricName, driverName, operationName, operationStatus, snapshotType, le)
   576  
   577  			switch method {
   578  			case "snapshot_controller_operation_total_seconds_count":
   579  				for _, sample := range samples {
   580  					scm.countMetrics[key] = float64(sample.Value)
   581  				}
   582  
   583  			case "snapshot_controller_operation_total_seconds_sum":
   584  				for _, sample := range samples {
   585  					scm.sumMetrics[key] = float64(sample.Value)
   586  				}
   587  
   588  			case "snapshot_controller_operation_total_seconds_bucket":
   589  				for _, sample := range samples {
   590  					scm.bucketMetrics[key] = float64(sample.Value)
   591  				}
   592  			}
   593  		}
   594  	}
   595  
   596  	return nil
   597  }
   598  
   599  func (scm *snapshotControllerMetrics) getMetricKey() string {
   600  	return snapshotMetricKey(scm.cfg.metricName, scm.cfg.driverName, scm.cfg.operationName, scm.cfg.operationStatus, scm.cfg.snapshotType, scm.cfg.le)
   601  }
   602  
   603  func snapshotMetricKey(metricName, driverName, operationName, operationStatus, snapshotType, le string) string {
   604  	key := driverName
   605  
   606  	// build key for shorthand metrics storage
   607  	for _, s := range []string{metricName, operationName, operationStatus, snapshotType, le} {
   608  		if s != "" {
   609  			key = fmt.Sprintf("%s_%s", key, s)
   610  		}
   611  	}
   612  
   613  	return key
   614  }
   615
View as plain text