snapshot_resource.go

Documentation: k8s.io/kubernetes/test/e2e/storage/framework

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package framework
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  
    23  	"github.com/onsi/ginkgo/v2"
    24  
    25  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    28  	"k8s.io/apimachinery/pkg/types"
    29  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    30  	"k8s.io/apimachinery/pkg/util/uuid"
    31  	"k8s.io/kubernetes/test/e2e/framework"
    32  	"k8s.io/kubernetes/test/e2e/storage/utils"
    33  )
    34  
    35  // SnapshotResource represents a snapshot class, a snapshot and its bound snapshot contents for a specific test case
    36  type SnapshotResource struct {
    37  	Config  *PerTestConfig
    38  	Pattern TestPattern
    39  
    40  	Vs        *unstructured.Unstructured
    41  	Vscontent *unstructured.Unstructured
    42  	Vsclass   *unstructured.Unstructured
    43  }
    44  
    45  // CreateSnapshot creates a VolumeSnapshotClass with given SnapshotDeletionPolicy and a VolumeSnapshot
    46  // from the VolumeSnapshotClass using a dynamic client.
    47  // Returns the unstructured VolumeSnapshotClass and VolumeSnapshot objects.
    48  func CreateSnapshot(ctx context.Context, sDriver SnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured) {
    49  	defer ginkgo.GinkgoRecover()
    50  	var err error
    51  	if pattern.SnapshotType != DynamicCreatedSnapshot && pattern.SnapshotType != PreprovisionedCreatedSnapshot {
    52  		err = fmt.Errorf("SnapshotType must be set to either DynamicCreatedSnapshot or PreprovisionedCreatedSnapshot")
    53  		framework.ExpectNoError(err)
    54  	}
    55  	dc := config.Framework.DynamicClient
    56  
    57  	ginkgo.By("creating a SnapshotClass")
    58  	sclass := sDriver.GetSnapshotClass(ctx, config, parameters)
    59  	if sclass == nil {
    60  		framework.Failf("Failed to get snapshot class based on test config")
    61  	}
    62  	sclass.Object["deletionPolicy"] = pattern.SnapshotDeletionPolicy.String()
    63  
    64  	sclass, err = dc.Resource(utils.SnapshotClassGVR).Create(ctx, sclass, metav1.CreateOptions{})
    65  	framework.ExpectNoError(err)
    66  
    67  	sclass, err = dc.Resource(utils.SnapshotClassGVR).Get(ctx, sclass.GetName(), metav1.GetOptions{})
    68  	framework.ExpectNoError(err)
    69  
    70  	ginkgo.By("creating a dynamic VolumeSnapshot")
    71  	// prepare a dynamically provisioned volume snapshot with certain data
    72  	snapshot := getSnapshot(pvcName, pvcNamespace, sclass.GetName())
    73  
    74  	snapshot, err = dc.Resource(utils.SnapshotGVR).Namespace(snapshot.GetNamespace()).Create(ctx, snapshot, metav1.CreateOptions{})
    75  	framework.ExpectNoError(err)
    76  
    77  	return sclass, snapshot
    78  }
    79  
    80  // CreateSnapshotResource creates a snapshot resource for the current test. It knows how to deal with
    81  // different test pattern snapshot provisioning and deletion policy
    82  func CreateSnapshotResource(ctx context.Context, sDriver SnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *SnapshotResource {
    83  	var err error
    84  	r := SnapshotResource{
    85  		Config:  config,
    86  		Pattern: pattern,
    87  	}
    88  	r.Vsclass, r.Vs = CreateSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters)
    89  
    90  	dc := r.Config.Framework.DynamicClient
    91  
    92  	r.Vscontent = utils.GetSnapshotContentFromSnapshot(ctx, dc, r.Vs, timeouts.SnapshotCreate)
    93  
    94  	if pattern.SnapshotType == PreprovisionedCreatedSnapshot {
    95  		// prepare a pre-provisioned VolumeSnapshotContent with certain data
    96  		// Because this could be run with an external CSI driver, we have no way
    97  		// to pre-provision the snapshot as we normally would using their API.
    98  		// We instead dynamically take a snapshot (above step), delete the old snapshot,
    99  		// and create another snapshot using the first snapshot's snapshot handle.
   100  
   101  		ginkgo.By("updating the snapshot content deletion policy to retain")
   102  		r.Vscontent.Object["spec"].(map[string]interface{})["deletionPolicy"] = "Retain"
   103  
   104  		r.Vscontent, err = dc.Resource(utils.SnapshotContentGVR).Update(ctx, r.Vscontent, metav1.UpdateOptions{})
   105  		framework.ExpectNoError(err)
   106  
   107  		ginkgo.By("recording properties of the preprovisioned snapshot")
   108  		snapshotHandle := r.Vscontent.Object["status"].(map[string]interface{})["snapshotHandle"].(string)
   109  		framework.Logf("Recording snapshot content handle: %s", snapshotHandle)
   110  		snapshotContentAnnotations := r.Vscontent.GetAnnotations()
   111  		framework.Logf("Recording snapshot content annotations: %v", snapshotContentAnnotations)
   112  		csiDriverName := r.Vsclass.Object["driver"].(string)
   113  		framework.Logf("Recording snapshot driver: %s", csiDriverName)
   114  
   115  		// If the deletion policy is retain on vscontent:
   116  		// when vs is deleted vscontent will not be deleted
   117  		// when the vscontent is manually deleted then the underlying snapshot resource will not be deleted.
   118  		// We exploit this to create a snapshot resource from which we can create a preprovisioned snapshot
   119  		ginkgo.By("deleting the snapshot and snapshot content")
   120  		err = dc.Resource(utils.SnapshotGVR).Namespace(r.Vs.GetNamespace()).Delete(ctx, r.Vs.GetName(), metav1.DeleteOptions{})
   121  		if apierrors.IsNotFound(err) {
   122  			err = nil
   123  		}
   124  		framework.ExpectNoError(err)
   125  
   126  		ginkgo.By("checking the Snapshot has been deleted")
   127  		err = utils.WaitForNamespacedGVRDeletion(ctx, dc, utils.SnapshotGVR, r.Vs.GetName(), r.Vs.GetNamespace(), framework.Poll, timeouts.SnapshotDelete)
   128  		framework.ExpectNoError(err)
   129  
   130  		err = dc.Resource(utils.SnapshotContentGVR).Delete(ctx, r.Vscontent.GetName(), metav1.DeleteOptions{})
   131  		if apierrors.IsNotFound(err) {
   132  			err = nil
   133  		}
   134  		framework.ExpectNoError(err)
   135  
   136  		ginkgo.By("checking the Snapshot content has been deleted")
   137  		err = utils.WaitForGVRDeletion(ctx, dc, utils.SnapshotContentGVR, r.Vscontent.GetName(), framework.Poll, timeouts.SnapshotDelete)
   138  		framework.ExpectNoError(err)
   139  
   140  		ginkgo.By("creating a snapshot content with the snapshot handle")
   141  		uuid := uuid.NewUUID()
   142  
   143  		snapName := getPreProvisionedSnapshotName(uuid)
   144  		snapcontentName := getPreProvisionedSnapshotContentName(uuid)
   145  
   146  		r.Vscontent = getPreProvisionedSnapshotContent(snapcontentName, snapshotContentAnnotations, snapName, pvcNamespace, snapshotHandle, pattern.SnapshotDeletionPolicy.String(), csiDriverName)
   147  		r.Vscontent, err = dc.Resource(utils.SnapshotContentGVR).Create(ctx, r.Vscontent, metav1.CreateOptions{})
   148  		framework.ExpectNoError(err)
   149  
   150  		ginkgo.By("creating a snapshot with that snapshot content")
   151  		r.Vs = getPreProvisionedSnapshot(snapName, pvcNamespace, snapcontentName)
   152  		r.Vs, err = dc.Resource(utils.SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(ctx, r.Vs, metav1.CreateOptions{})
   153  		framework.ExpectNoError(err)
   154  
   155  		err = utils.WaitForSnapshotReady(ctx, dc, r.Vs.GetNamespace(), r.Vs.GetName(), framework.Poll, timeouts.SnapshotCreate)
   156  		framework.ExpectNoError(err)
   157  
   158  		ginkgo.By("getting the snapshot and snapshot content")
   159  		r.Vs, err = dc.Resource(utils.SnapshotGVR).Namespace(r.Vs.GetNamespace()).Get(ctx, r.Vs.GetName(), metav1.GetOptions{})
   160  		framework.ExpectNoError(err)
   161  
   162  		r.Vscontent, err = dc.Resource(utils.SnapshotContentGVR).Get(ctx, r.Vscontent.GetName(), metav1.GetOptions{})
   163  		framework.ExpectNoError(err)
   164  	}
   165  	return &r
   166  }
   167  
   168  // CleanupResource cleans up the snapshot resource and ignores not found errors
   169  func (sr *SnapshotResource) CleanupResource(ctx context.Context, timeouts *framework.TimeoutContext) error {
   170  	var err error
   171  	var cleanupErrs []error
   172  
   173  	dc := sr.Config.Framework.DynamicClient
   174  
   175  	if sr.Vs != nil {
   176  		framework.Logf("deleting snapshot %q/%q", sr.Vs.GetNamespace(), sr.Vs.GetName())
   177  
   178  		sr.Vs, err = dc.Resource(utils.SnapshotGVR).Namespace(sr.Vs.GetNamespace()).Get(ctx, sr.Vs.GetName(), metav1.GetOptions{})
   179  		switch {
   180  		case err == nil:
   181  			snapshotStatus := sr.Vs.Object["status"].(map[string]interface{})
   182  			snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string)
   183  			framework.Logf("received snapshotStatus %v", snapshotStatus)
   184  			framework.Logf("snapshotContentName %s", snapshotContentName)
   185  
   186  			boundVsContent, err := dc.Resource(utils.SnapshotContentGVR).Get(ctx, snapshotContentName, metav1.GetOptions{})
   187  			switch {
   188  			case err == nil:
   189  				if boundVsContent.Object["spec"].(map[string]interface{})["deletionPolicy"] != "Delete" {
   190  					// The purpose of this block is to prevent physical snapshotContent leaks.
   191  					// We must update the SnapshotContent to have Delete Deletion policy,
   192  					// or else the physical snapshot content will be leaked.
   193  					boundVsContent.Object["spec"].(map[string]interface{})["deletionPolicy"] = "Delete"
   194  					boundVsContent, err = dc.Resource(utils.SnapshotContentGVR).Update(ctx, boundVsContent, metav1.UpdateOptions{})
   195  					framework.ExpectNoError(err)
   196  				}
   197  				err = dc.Resource(utils.SnapshotGVR).Namespace(sr.Vs.GetNamespace()).Delete(ctx, sr.Vs.GetName(), metav1.DeleteOptions{})
   198  				if apierrors.IsNotFound(err) {
   199  					err = nil
   200  				}
   201  				framework.ExpectNoError(err)
   202  
   203  				err = utils.WaitForGVRDeletion(ctx, dc, utils.SnapshotContentGVR, boundVsContent.GetName(), framework.Poll, timeouts.SnapshotDelete)
   204  				framework.ExpectNoError(err)
   205  
   206  			case apierrors.IsNotFound(err):
   207  				// the volume snapshot is not bound to snapshot content yet
   208  				err = dc.Resource(utils.SnapshotGVR).Namespace(sr.Vs.GetNamespace()).Delete(ctx, sr.Vs.GetName(), metav1.DeleteOptions{})
   209  				if apierrors.IsNotFound(err) {
   210  					err = nil
   211  				}
   212  				framework.ExpectNoError(err)
   213  
   214  				err = utils.WaitForNamespacedGVRDeletion(ctx, dc, utils.SnapshotGVR, sr.Vs.GetName(), sr.Vs.GetNamespace(), framework.Poll, timeouts.SnapshotDelete)
   215  				framework.ExpectNoError(err)
   216  			default:
   217  				cleanupErrs = append(cleanupErrs, err)
   218  			}
   219  		case apierrors.IsNotFound(err):
   220  			// Hope that the underlying snapshot content and resource is gone already
   221  		default:
   222  			cleanupErrs = append(cleanupErrs, err)
   223  		}
   224  	}
   225  	if sr.Vscontent != nil {
   226  		framework.Logf("deleting snapshot content %q", sr.Vscontent.GetName())
   227  
   228  		sr.Vscontent, err = dc.Resource(utils.SnapshotContentGVR).Get(ctx, sr.Vscontent.GetName(), metav1.GetOptions{})
   229  		switch {
   230  		case err == nil:
   231  			if sr.Vscontent.Object["spec"].(map[string]interface{})["deletionPolicy"] != "Delete" {
   232  				// The purpose of this block is to prevent physical snapshotContent leaks.
   233  				// We must update the SnapshotContent to have Delete Deletion policy,
   234  				// or else the physical snapshot content will be leaked.
   235  				sr.Vscontent.Object["spec"].(map[string]interface{})["deletionPolicy"] = "Delete"
   236  				sr.Vscontent, err = dc.Resource(utils.SnapshotContentGVR).Update(ctx, sr.Vscontent, metav1.UpdateOptions{})
   237  				framework.ExpectNoError(err)
   238  			}
   239  			err = dc.Resource(utils.SnapshotContentGVR).Delete(ctx, sr.Vscontent.GetName(), metav1.DeleteOptions{})
   240  			if apierrors.IsNotFound(err) {
   241  				err = nil
   242  			}
   243  			framework.ExpectNoError(err)
   244  
   245  			err = utils.WaitForGVRDeletion(ctx, dc, utils.SnapshotContentGVR, sr.Vscontent.GetName(), framework.Poll, timeouts.SnapshotDelete)
   246  			framework.ExpectNoError(err)
   247  		case apierrors.IsNotFound(err):
   248  			// Hope the underlying physical snapshot resource has been deleted already
   249  		default:
   250  			cleanupErrs = append(cleanupErrs, err)
   251  		}
   252  	}
   253  	if sr.Vsclass != nil {
   254  		framework.Logf("deleting snapshot class %q", sr.Vsclass.GetName())
   255  		// typically this snapshot class has already been deleted
   256  		err = dc.Resource(utils.SnapshotClassGVR).Delete(ctx, sr.Vsclass.GetName(), metav1.DeleteOptions{})
   257  		if err != nil && !apierrors.IsNotFound(err) {
   258  			framework.Failf("Error deleting snapshot class %q. Error: %v", sr.Vsclass.GetName(), err)
   259  		}
   260  		err = utils.WaitForGVRDeletion(ctx, dc, utils.SnapshotClassGVR, sr.Vsclass.GetName(), framework.Poll, timeouts.SnapshotDelete)
   261  		framework.ExpectNoError(err)
   262  	}
   263  	return utilerrors.NewAggregate(cleanupErrs)
   264  }
   265  
   266  func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.Unstructured {
   267  	snapshot := &unstructured.Unstructured{
   268  		Object: map[string]interface{}{
   269  			"kind":       "VolumeSnapshot",
   270  			"apiVersion": utils.SnapshotAPIVersion,
   271  			"metadata": map[string]interface{}{
   272  				"generateName": "snapshot-",
   273  				"namespace":    ns,
   274  			},
   275  			"spec": map[string]interface{}{
   276  				"volumeSnapshotClassName": snapshotClassName,
   277  				"source": map[string]interface{}{
   278  					"persistentVolumeClaimName": claimName,
   279  				},
   280  			},
   281  		},
   282  	}
   283  
   284  	return snapshot
   285  }
   286  func getPreProvisionedSnapshot(snapName, ns, snapshotContentName string) *unstructured.Unstructured {
   287  	snapshot := &unstructured.Unstructured{
   288  		Object: map[string]interface{}{
   289  			"kind":       "VolumeSnapshot",
   290  			"apiVersion": utils.SnapshotAPIVersion,
   291  			"metadata": map[string]interface{}{
   292  				"name":      snapName,
   293  				"namespace": ns,
   294  			},
   295  			"spec": map[string]interface{}{
   296  				"source": map[string]interface{}{
   297  					"volumeSnapshotContentName": snapshotContentName,
   298  				},
   299  			},
   300  		},
   301  	}
   302  
   303  	return snapshot
   304  }
   305  func getPreProvisionedSnapshotContent(snapcontentName string, snapshotContentAnnotations map[string]string, snapshotName, snapshotNamespace, snapshotHandle, deletionPolicy, csiDriverName string) *unstructured.Unstructured {
   306  	snapshotContent := &unstructured.Unstructured{
   307  		Object: map[string]interface{}{
   308  			"kind":       "VolumeSnapshotContent",
   309  			"apiVersion": utils.SnapshotAPIVersion,
   310  			"metadata": map[string]interface{}{
   311  				"name":        snapcontentName,
   312  				"annotations": snapshotContentAnnotations,
   313  			},
   314  			"spec": map[string]interface{}{
   315  				"source": map[string]interface{}{
   316  					"snapshotHandle": snapshotHandle,
   317  				},
   318  				"volumeSnapshotRef": map[string]interface{}{
   319  					"name":      snapshotName,
   320  					"namespace": snapshotNamespace,
   321  				},
   322  				"driver":         csiDriverName,
   323  				"deletionPolicy": deletionPolicy,
   324  			},
   325  		},
   326  	}
   327  
   328  	return snapshotContent
   329  }
   330  
   331  func getPreProvisionedSnapshotContentName(uuid types.UID) string {
   332  	return fmt.Sprintf("pre-provisioned-snapcontent-%s", string(uuid))
   333  }
   334  
   335  func getPreProvisionedSnapshotName(uuid types.UID) string {
   336  	return fmt.Sprintf("pre-provisioned-snapshot-%s", string(uuid))
   337  }
   338
View as plain text