Source file src/k8s.io/kubernetes/test/integration/volume/persistent_volumes_test.go

Documentation: k8s.io/kubernetes/test/integration/volume

     1  /*
     2  Copyright 2014 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package volume
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"math/rand"
    23  	"os"
    24  	"strconv"
    25  	"testing"
    26  	"time"
    27  
    28  	v1 "k8s.io/api/core/v1"
    29  	storage "k8s.io/api/storage/v1"
    30  	"k8s.io/apimachinery/pkg/api/resource"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/watch"
    33  	"k8s.io/client-go/informers"
    34  	clientset "k8s.io/client-go/kubernetes"
    35  	restclient "k8s.io/client-go/rest"
    36  	ref "k8s.io/client-go/tools/reference"
    37  	fakecloud "k8s.io/cloud-provider/fake"
    38  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    39  	"k8s.io/kubernetes/pkg/api/legacyscheme"
    40  	persistentvolumecontroller "k8s.io/kubernetes/pkg/controller/volume/persistentvolume"
    41  	"k8s.io/kubernetes/pkg/volume"
    42  	volumetest "k8s.io/kubernetes/pkg/volume/testing"
    43  	"k8s.io/kubernetes/pkg/volume/util"
    44  	"k8s.io/kubernetes/test/integration/framework"
    45  	"k8s.io/kubernetes/test/utils/ktesting"
    46  
    47  	"k8s.io/klog/v2"
    48  )
    49  
    50  // Several tests in this file are configurable by environment variables:
    51  // KUBE_INTEGRATION_PV_OBJECTS - nr. of PVs/PVCs to be created
    52  //
    53  //	(100 by default)
    54  //
    55  // KUBE_INTEGRATION_PV_SYNC_PERIOD - volume controller sync period
    56  //
    57  //	(1s by default)
    58  //
    59  // KUBE_INTEGRATION_PV_END_SLEEP - for how long should
    60  //
    61  //	TestPersistentVolumeMultiPVsPVCs sleep when it's finished (0s by
    62  //	default). This is useful to test how long does it take for periodic sync
    63  //	to process bound PVs/PVCs.
    64  const defaultObjectCount = 100
    65  const defaultSyncPeriod = 1 * time.Second
    66  
    67  const provisionerPluginName = "kubernetes.io/mock-provisioner"
    68  
    69  func getObjectCount() int {
    70  	objectCount := defaultObjectCount
    71  	if s := os.Getenv("KUBE_INTEGRATION_PV_OBJECTS"); s != "" {
    72  		var err error
    73  		objectCount, err = strconv.Atoi(s)
    74  		if err != nil {
    75  			klog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_OBJECTS: %v", err)
    76  		}
    77  	}
    78  	klog.V(2).Infof("using KUBE_INTEGRATION_PV_OBJECTS=%d", objectCount)
    79  	return objectCount
    80  }
    81  
    82  func getSyncPeriod(syncPeriod time.Duration) time.Duration {
    83  	period := syncPeriod
    84  	if s := os.Getenv("KUBE_INTEGRATION_PV_SYNC_PERIOD"); s != "" {
    85  		var err error
    86  		period, err = time.ParseDuration(s)
    87  		if err != nil {
    88  			klog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_SYNC_PERIOD: %v", err)
    89  		}
    90  	}
    91  	klog.V(2).Infof("using KUBE_INTEGRATION_PV_SYNC_PERIOD=%v", period)
    92  	return period
    93  }
    94  
    95  func testSleep() {
    96  	var period time.Duration
    97  	if s := os.Getenv("KUBE_INTEGRATION_PV_END_SLEEP"); s != "" {
    98  		var err error
    99  		period, err = time.ParseDuration(s)
   100  		if err != nil {
   101  			klog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_END_SLEEP: %v", err)
   102  		}
   103  	}
   104  	klog.V(2).Infof("using KUBE_INTEGRATION_PV_END_SLEEP=%v", period)
   105  	if period != 0 {
   106  		time.Sleep(period)
   107  		klog.V(2).Infof("sleep finished")
   108  	}
   109  }
   110  
   111  func TestPersistentVolumeRecycler(t *testing.T) {
   112  	klog.V(2).Infof("TestPersistentVolumeRecycler started")
   113  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   114  	defer s.TearDownFn()
   115  	namespaceName := "pv-recycler"
   116  
   117  	tCtx := ktesting.Init(t)
   118  	defer tCtx.Cancel("test has completed")
   119  
   120  	testClient, ctrl, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
   121  	defer watchPV.Stop()
   122  	defer watchPVC.Stop()
   123  
   124  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   125  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   126  
   127  	// NOTE: This test cannot run in parallel, because it is creating and deleting
   128  	// non-namespaced objects (PersistenceVolumes).
   129  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   130  
   131  	informers.Start(tCtx.Done())
   132  	go ctrl.Run(tCtx)
   133  
   134  	// This PV will be claimed, released, and recycled.
   135  	pv := createPV("fake-pv-recycler", "/tmp/foo", "10G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRecycle)
   136  	pvc := createPVC("fake-pvc-recycler", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "")
   137  
   138  	_, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
   139  	if err != nil {
   140  		t.Errorf("Failed to create PersistentVolume: %v", err)
   141  	}
   142  	klog.V(2).Infof("TestPersistentVolumeRecycler pvc created")
   143  
   144  	_, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
   145  	if err != nil {
   146  		t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
   147  	}
   148  	klog.V(2).Infof("TestPersistentVolumeRecycler pvc created")
   149  
   150  	// wait until the controller pairs the volume and claim
   151  	waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeBound)
   152  	klog.V(2).Infof("TestPersistentVolumeRecycler pv bound")
   153  	waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound)
   154  	klog.V(2).Infof("TestPersistentVolumeRecycler pvc bound")
   155  
   156  	// deleting a claim releases the volume, after which it can be recycled
   157  	if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}); err != nil {
   158  		t.Errorf("error deleting claim %s", pvc.Name)
   159  	}
   160  	klog.V(2).Infof("TestPersistentVolumeRecycler pvc deleted")
   161  
   162  	waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeReleased)
   163  	klog.V(2).Infof("TestPersistentVolumeRecycler pv released")
   164  	waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeAvailable)
   165  	klog.V(2).Infof("TestPersistentVolumeRecycler pv available")
   166  }
   167  
   168  func TestPersistentVolumeDeleter(t *testing.T) {
   169  	klog.V(2).Infof("TestPersistentVolumeDeleter started")
   170  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   171  	defer s.TearDownFn()
   172  	namespaceName := "pv-deleter"
   173  
   174  	tCtx := ktesting.Init(t)
   175  	defer tCtx.Cancel("test has completed")
   176  	testClient, ctrl, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
   177  	defer watchPV.Stop()
   178  	defer watchPVC.Stop()
   179  
   180  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   181  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   182  
   183  	// NOTE: This test cannot run in parallel, because it is creating and deleting
   184  	// non-namespaced objects (PersistenceVolumes).
   185  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   186  
   187  	informers.Start(tCtx.Done())
   188  	go ctrl.Run(tCtx)
   189  
   190  	// This PV will be claimed, released, and deleted.
   191  	pv := createPV("fake-pv-deleter", "/tmp/foo", "10G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimDelete)
   192  	pvc := createPVC("fake-pvc-deleter", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "")
   193  
   194  	_, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
   195  	if err != nil {
   196  		t.Errorf("Failed to create PersistentVolume: %v", err)
   197  	}
   198  	klog.V(2).Infof("TestPersistentVolumeDeleter pv created")
   199  	_, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
   200  	if err != nil {
   201  		t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
   202  	}
   203  	klog.V(2).Infof("TestPersistentVolumeDeleter pvc created")
   204  	waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeBound)
   205  	klog.V(2).Infof("TestPersistentVolumeDeleter pv bound")
   206  	waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound)
   207  	klog.V(2).Infof("TestPersistentVolumeDeleter pvc bound")
   208  
   209  	// deleting a claim releases the volume, after which it can be recycled
   210  	if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}); err != nil {
   211  		t.Errorf("error deleting claim %s", pvc.Name)
   212  	}
   213  	klog.V(2).Infof("TestPersistentVolumeDeleter pvc deleted")
   214  
   215  	waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeReleased)
   216  	klog.V(2).Infof("TestPersistentVolumeDeleter pv released")
   217  
   218  	for {
   219  		event := <-watchPV.ResultChan()
   220  		if event.Type == watch.Deleted {
   221  			break
   222  		}
   223  	}
   224  	klog.V(2).Infof("TestPersistentVolumeDeleter pv deleted")
   225  }
   226  
   227  func TestPersistentVolumeBindRace(t *testing.T) {
   228  	// Test a race binding many claims to a PV that is pre-bound to a specific
   229  	// PVC. Only this specific PVC should get bound.
   230  	klog.V(2).Infof("TestPersistentVolumeBindRace started")
   231  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   232  	defer s.TearDownFn()
   233  	namespaceName := "pv-bind-race"
   234  
   235  	tCtx := ktesting.Init(t)
   236  	defer tCtx.Cancel("test has completed")
   237  	testClient, ctrl, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
   238  	defer watchPV.Stop()
   239  	defer watchPVC.Stop()
   240  
   241  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   242  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   243  
   244  	// NOTE: This test cannot run in parallel, because it is creating and deleting
   245  	// non-namespaced objects (PersistenceVolumes).
   246  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   247  
   248  	informers.Start(tCtx.Done())
   249  	go ctrl.Run(tCtx)
   250  
   251  	pv := createPV("fake-pv-race", "/tmp/foo", "10G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain)
   252  	pvc := createPVC("fake-pvc-race", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "")
   253  	counter := 0
   254  	maxClaims := 100
   255  	claims := []*v1.PersistentVolumeClaim{}
   256  	for counter <= maxClaims {
   257  		counter++
   258  		newPvc := pvc.DeepCopy()
   259  		newPvc.ObjectMeta = metav1.ObjectMeta{Name: fmt.Sprintf("fake-pvc-race-%d", counter)}
   260  		claim, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), newPvc, metav1.CreateOptions{})
   261  		if err != nil {
   262  			t.Fatalf("Error creating newPvc: %v", err)
   263  		}
   264  		claims = append(claims, claim)
   265  	}
   266  	klog.V(2).Infof("TestPersistentVolumeBindRace claims created")
   267  
   268  	// putting a bind manually on a pv should only match the claim it is bound to
   269  	claim := claims[rand.Intn(maxClaims-1)]
   270  	claimRef, err := ref.GetReference(legacyscheme.Scheme, claim)
   271  	if err != nil {
   272  		t.Fatalf("Unexpected error getting claimRef: %v", err)
   273  	}
   274  	pv.Spec.ClaimRef = claimRef
   275  	pv.Spec.ClaimRef.UID = ""
   276  
   277  	pv, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
   278  	if err != nil {
   279  		t.Fatalf("Unexpected error creating pv: %v", err)
   280  	}
   281  	klog.V(2).Infof("TestPersistentVolumeBindRace pv created, pre-bound to %s", claim.Name)
   282  
   283  	waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeBound)
   284  	klog.V(2).Infof("TestPersistentVolumeBindRace pv bound")
   285  	waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound)
   286  	klog.V(2).Infof("TestPersistentVolumeBindRace pvc bound")
   287  
   288  	pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), pv.Name, metav1.GetOptions{})
   289  	if err != nil {
   290  		t.Fatalf("Unexpected error getting pv: %v", err)
   291  	}
   292  	if pv.Spec.ClaimRef == nil {
   293  		t.Fatalf("Unexpected nil claimRef")
   294  	}
   295  	if pv.Spec.ClaimRef.Namespace != claimRef.Namespace || pv.Spec.ClaimRef.Name != claimRef.Name {
   296  		t.Fatalf("Bind mismatch! Expected %s/%s but got %s/%s", claimRef.Namespace, claimRef.Name, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name)
   297  	}
   298  }
   299  
   300  // TestPersistentVolumeClaimLabelSelector test binding using label selectors
   301  func TestPersistentVolumeClaimLabelSelector(t *testing.T) {
   302  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   303  	defer s.TearDownFn()
   304  	namespaceName := "pvc-label-selector"
   305  
   306  	tCtx := ktesting.Init(t)
   307  	defer tCtx.Cancel("test has completed")
   308  	testClient, controller, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
   309  	defer watchPV.Stop()
   310  	defer watchPVC.Stop()
   311  
   312  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   313  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   314  
   315  	// NOTE: This test cannot run in parallel, because it is creating and deleting
   316  	// non-namespaced objects (PersistenceVolumes).
   317  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   318  
   319  	informers.Start(tCtx.Done())
   320  	go controller.Run(tCtx)
   321  
   322  	var (
   323  		err     error
   324  		modes   = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}
   325  		reclaim = v1.PersistentVolumeReclaimRetain
   326  
   327  		pvTrue  = createPV("pv-true", "/tmp/foo-label", "1G", modes, reclaim)
   328  		pvFalse = createPV("pv-false", "/tmp/foo-label", "1G", modes, reclaim)
   329  		pvc     = createPVC("pvc-ls-1", ns.Name, "1G", modes, "")
   330  	)
   331  
   332  	pvTrue.ObjectMeta.SetLabels(map[string]string{"foo": "true"})
   333  	pvFalse.ObjectMeta.SetLabels(map[string]string{"foo": "false"})
   334  
   335  	_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvTrue, metav1.CreateOptions{})
   336  	if err != nil {
   337  		t.Fatalf("Failed to create PersistentVolume: %v", err)
   338  	}
   339  	_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvFalse, metav1.CreateOptions{})
   340  	if err != nil {
   341  		t.Fatalf("Failed to create PersistentVolume: %v", err)
   342  	}
   343  	t.Log("volumes created")
   344  
   345  	pvc.Spec.Selector = &metav1.LabelSelector{
   346  		MatchLabels: map[string]string{
   347  			"foo": "true",
   348  		},
   349  	}
   350  
   351  	_, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
   352  	if err != nil {
   353  		t.Fatalf("Failed to create PersistentVolumeClaim: %v", err)
   354  	}
   355  	t.Log("claim created")
   356  
   357  	waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound)
   358  	t.Log("volume bound")
   359  	waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound)
   360  	t.Log("claim bound")
   361  
   362  	pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-false", metav1.GetOptions{})
   363  	if err != nil {
   364  		t.Fatalf("Unexpected error getting pv: %v", err)
   365  	}
   366  	if pv.Spec.ClaimRef != nil {
   367  		t.Fatalf("False PV shouldn't be bound")
   368  	}
   369  	pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-true", metav1.GetOptions{})
   370  	if err != nil {
   371  		t.Fatalf("Unexpected error getting pv: %v", err)
   372  	}
   373  	if pv.Spec.ClaimRef == nil {
   374  		t.Fatalf("True PV should be bound")
   375  	}
   376  	if pv.Spec.ClaimRef.Namespace != pvc.Namespace || pv.Spec.ClaimRef.Name != pvc.Name {
   377  		t.Fatalf("Bind mismatch! Expected %s/%s but got %s/%s", pvc.Namespace, pvc.Name, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name)
   378  	}
   379  }
   380  
   381  // TestPersistentVolumeClaimLabelSelectorMatchExpressions test binding using
   382  // MatchExpressions label selectors
   383  func TestPersistentVolumeClaimLabelSelectorMatchExpressions(t *testing.T) {
   384  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   385  	defer s.TearDownFn()
   386  	namespaceName := "pvc-match-expressions"
   387  
   388  	tCtx := ktesting.Init(t)
   389  	defer tCtx.Cancel("test has completed")
   390  	testClient, controller, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
   391  	defer watchPV.Stop()
   392  	defer watchPVC.Stop()
   393  
   394  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   395  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   396  
   397  	// NOTE: This test cannot run in parallel, because it is creating and deleting
   398  	// non-namespaced objects (PersistenceVolumes).
   399  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   400  
   401  	informers.Start(tCtx.Done())
   402  	go controller.Run(tCtx)
   403  
   404  	var (
   405  		err     error
   406  		modes   = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}
   407  		reclaim = v1.PersistentVolumeReclaimRetain
   408  
   409  		pvTrue  = createPV("pv-true", "/tmp/foo-label", "1G", modes, reclaim)
   410  		pvFalse = createPV("pv-false", "/tmp/foo-label", "1G", modes, reclaim)
   411  		pvc     = createPVC("pvc-ls-1", ns.Name, "1G", modes, "")
   412  	)
   413  
   414  	pvTrue.ObjectMeta.SetLabels(map[string]string{"foo": "valA", "bar": ""})
   415  	pvFalse.ObjectMeta.SetLabels(map[string]string{"foo": "valB", "baz": ""})
   416  
   417  	_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvTrue, metav1.CreateOptions{})
   418  	if err != nil {
   419  		t.Fatalf("Failed to create PersistentVolume: %v", err)
   420  	}
   421  	_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvFalse, metav1.CreateOptions{})
   422  	if err != nil {
   423  		t.Fatalf("Failed to create PersistentVolume: %v", err)
   424  	}
   425  	t.Log("volumes created")
   426  
   427  	pvc.Spec.Selector = &metav1.LabelSelector{
   428  		MatchExpressions: []metav1.LabelSelectorRequirement{
   429  			{
   430  				Key:      "foo",
   431  				Operator: metav1.LabelSelectorOpIn,
   432  				Values:   []string{"valA"},
   433  			},
   434  			{
   435  				Key:      "foo",
   436  				Operator: metav1.LabelSelectorOpNotIn,
   437  				Values:   []string{"valB"},
   438  			},
   439  			{
   440  				Key:      "bar",
   441  				Operator: metav1.LabelSelectorOpExists,
   442  				Values:   []string{},
   443  			},
   444  			{
   445  				Key:      "baz",
   446  				Operator: metav1.LabelSelectorOpDoesNotExist,
   447  				Values:   []string{},
   448  			},
   449  		},
   450  	}
   451  
   452  	_, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
   453  	if err != nil {
   454  		t.Fatalf("Failed to create PersistentVolumeClaim: %v", err)
   455  	}
   456  	t.Log("claim created")
   457  
   458  	waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound)
   459  	t.Log("volume bound")
   460  	waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound)
   461  	t.Log("claim bound")
   462  
   463  	pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-false", metav1.GetOptions{})
   464  	if err != nil {
   465  		t.Fatalf("Unexpected error getting pv: %v", err)
   466  	}
   467  	if pv.Spec.ClaimRef != nil {
   468  		t.Fatalf("False PV shouldn't be bound")
   469  	}
   470  	pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-true", metav1.GetOptions{})
   471  	if err != nil {
   472  		t.Fatalf("Unexpected error getting pv: %v", err)
   473  	}
   474  	if pv.Spec.ClaimRef == nil {
   475  		t.Fatalf("True PV should be bound")
   476  	}
   477  	if pv.Spec.ClaimRef.Namespace != pvc.Namespace || pv.Spec.ClaimRef.Name != pvc.Name {
   478  		t.Fatalf("Bind mismatch! Expected %s/%s but got %s/%s", pvc.Namespace, pvc.Name, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name)
   479  	}
   480  }
   481  
   482  // TestPersistentVolumeMultiPVs tests binding of one PVC to 100 PVs with
   483  // different size.
   484  func TestPersistentVolumeMultiPVs(t *testing.T) {
   485  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   486  	defer s.TearDownFn()
   487  	namespaceName := "multi-pvs"
   488  
   489  	tCtx := ktesting.Init(t)
   490  	defer tCtx.Cancel("test has completed")
   491  	testClient, controller, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
   492  	defer watchPV.Stop()
   493  	defer watchPVC.Stop()
   494  
   495  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   496  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   497  
   498  	// NOTE: This test cannot run in parallel, because it is creating and deleting
   499  	// non-namespaced objects (PersistenceVolumes).
   500  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   501  
   502  	informers.Start(tCtx.Done())
   503  	go controller.Run(tCtx)
   504  
   505  	maxPVs := getObjectCount()
   506  	pvs := make([]*v1.PersistentVolume, maxPVs)
   507  	for i := 0; i < maxPVs; i++ {
   508  		// This PV will be claimed, released, and deleted
   509  		pvs[i] = createPV("pv-"+strconv.Itoa(i), "/tmp/foo"+strconv.Itoa(i), strconv.Itoa(i+1)+"G",
   510  			[]v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain)
   511  	}
   512  
   513  	pvc := createPVC("pvc-2", ns.Name, strconv.Itoa(maxPVs/2)+"G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "")
   514  
   515  	for i := 0; i < maxPVs; i++ {
   516  		_, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvs[i], metav1.CreateOptions{})
   517  		if err != nil {
   518  			t.Errorf("Failed to create PersistentVolume %d: %v", i, err)
   519  		}
   520  		waitForPersistentVolumePhase(testClient, pvs[i].Name, watchPV, v1.VolumeAvailable)
   521  	}
   522  	t.Log("volumes created")
   523  
   524  	_, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
   525  	if err != nil {
   526  		t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
   527  	}
   528  	t.Log("claim created")
   529  
   530  	// wait until the binder pairs the claim with a volume
   531  	waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound)
   532  	t.Log("volume bound")
   533  	waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound)
   534  	t.Log("claim bound")
   535  
   536  	// only one PV is bound
   537  	bound := 0
   538  	for i := 0; i < maxPVs; i++ {
   539  		pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), pvs[i].Name, metav1.GetOptions{})
   540  		if err != nil {
   541  			t.Fatalf("Unexpected error getting pv: %v", err)
   542  		}
   543  		if pv.Spec.ClaimRef == nil {
   544  			continue
   545  		}
   546  		// found a bounded PV
   547  		p := pv.Spec.Capacity[v1.ResourceStorage]
   548  		pvCap := p.Value()
   549  		expectedCap := resource.MustParse(strconv.Itoa(maxPVs/2) + "G")
   550  		expectedCapVal := expectedCap.Value()
   551  		if pv.Spec.ClaimRef.Name != pvc.Name || pvCap != expectedCapVal {
   552  			t.Fatalf("Bind mismatch! Expected %s capacity %d but got %s capacity %d", pvc.Name, expectedCapVal, pv.Spec.ClaimRef.Name, pvCap)
   553  		}
   554  		t.Logf("claim bounded to %s capacity %v", pv.Name, pv.Spec.Capacity[v1.ResourceStorage])
   555  		bound++
   556  	}
   557  	t.Log("volumes checked")
   558  
   559  	if bound != 1 {
   560  		t.Fatalf("Only 1 PV should be bound but got %d", bound)
   561  	}
   562  
   563  	// deleting a claim releases the volume
   564  	if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}); err != nil {
   565  		t.Errorf("error deleting claim %s", pvc.Name)
   566  	}
   567  	t.Log("claim deleted")
   568  
   569  	waitForAnyPersistentVolumePhase(watchPV, v1.VolumeReleased)
   570  	t.Log("volumes released")
   571  }
   572  
   573  // TestPersistentVolumeMultiPVsPVCs tests binding of 100 PVC to 100 PVs.
   574  // This test is configurable by KUBE_INTEGRATION_PV_* variables.
   575  func TestPersistentVolumeMultiPVsPVCs(t *testing.T) {
   576  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   577  	defer s.TearDownFn()
   578  	namespaceName := "multi-pvs-pvcs"
   579  
   580  	tCtx := ktesting.Init(t)
   581  	defer tCtx.Cancel("test has completed")
   582  	testClient, binder, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
   583  	defer watchPV.Stop()
   584  	defer watchPVC.Stop()
   585  
   586  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   587  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   588  
   589  	// NOTE: This test cannot run in parallel, because it is creating and deleting
   590  	// non-namespaced objects (PersistenceVolumes).
   591  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   592  
   593  	informers.Start(tCtx.Done())
   594  	go binder.Run(tCtx)
   595  
   596  	objCount := getObjectCount()
   597  	pvs := make([]*v1.PersistentVolume, objCount)
   598  	pvcs := make([]*v1.PersistentVolumeClaim, objCount)
   599  	for i := 0; i < objCount; i++ {
   600  		// This PV will be claimed, released, and deleted
   601  		pvs[i] = createPV("pv-"+strconv.Itoa(i), "/tmp/foo"+strconv.Itoa(i), "1G",
   602  			[]v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain)
   603  		pvcs[i] = createPVC("pvc-"+strconv.Itoa(i), ns.Name, "1G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "")
   604  	}
   605  
   606  	// Create PVs first
   607  	klog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: start")
   608  
   609  	// Create the volumes in a separate goroutine to pop events from
   610  	// watchPV early - it seems it has limited capacity and it gets stuck
   611  	// with >3000 volumes.
   612  	go func() {
   613  		for i := 0; i < objCount; i++ {
   614  			_, _ = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvs[i], metav1.CreateOptions{})
   615  		}
   616  	}()
   617  	// Wait for them to get Available
   618  	for i := 0; i < objCount; i++ {
   619  		waitForAnyPersistentVolumePhase(watchPV, v1.VolumeAvailable)
   620  		klog.V(1).Infof("%d volumes available", i+1)
   621  	}
   622  	klog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: volumes are Available")
   623  
   624  	// Start a separate goroutine that randomly modifies PVs and PVCs while the
   625  	// binder is working. We test that the binder can bind volumes despite
   626  	// users modifying objects underneath.
   627  	stopCh := make(chan struct{}, 0)
   628  	go func() {
   629  		for {
   630  			// Roll a dice and decide a PV or PVC to modify
   631  			if rand.Intn(2) == 0 {
   632  				// Modify PV
   633  				i := rand.Intn(objCount)
   634  				name := "pv-" + strconv.Itoa(i)
   635  				pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), name, metav1.GetOptions{})
   636  				if err != nil {
   637  					// Silently ignore error, the PV may have be already deleted
   638  					// or not exists yet.
   639  					klog.V(4).Infof("Failed to read PV %s: %v", name, err)
   640  					continue
   641  				}
   642  				if pv.Annotations == nil {
   643  					pv.Annotations = map[string]string{"TestAnnotation": fmt.Sprint(rand.Int())}
   644  				} else {
   645  					pv.Annotations["TestAnnotation"] = fmt.Sprint(rand.Int())
   646  				}
   647  				_, err = testClient.CoreV1().PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{})
   648  				if err != nil {
   649  					// Silently ignore error, the PV may have been updated by
   650  					// the controller.
   651  					klog.V(4).Infof("Failed to update PV %s: %v", pv.Name, err)
   652  					continue
   653  				}
   654  				klog.V(4).Infof("Updated PV %s", pv.Name)
   655  			} else {
   656  				// Modify PVC
   657  				i := rand.Intn(objCount)
   658  				name := "pvc-" + strconv.Itoa(i)
   659  				pvc, err := testClient.CoreV1().PersistentVolumeClaims(metav1.NamespaceDefault).Get(context.TODO(), name, metav1.GetOptions{})
   660  				if err != nil {
   661  					// Silently ignore error, the PVC may have be already
   662  					// deleted or not exists yet.
   663  					klog.V(4).Infof("Failed to read PVC %s: %v", name, err)
   664  					continue
   665  				}
   666  				if pvc.Annotations == nil {
   667  					pvc.Annotations = map[string]string{"TestAnnotation": fmt.Sprint(rand.Int())}
   668  				} else {
   669  					pvc.Annotations["TestAnnotation"] = fmt.Sprint(rand.Int())
   670  				}
   671  				_, err = testClient.CoreV1().PersistentVolumeClaims(metav1.NamespaceDefault).Update(context.TODO(), pvc, metav1.UpdateOptions{})
   672  				if err != nil {
   673  					// Silently ignore error, the PVC may have been updated by
   674  					// the controller.
   675  					klog.V(4).Infof("Failed to update PVC %s: %v", pvc.Name, err)
   676  					continue
   677  				}
   678  				klog.V(4).Infof("Updated PVC %s", pvc.Name)
   679  			}
   680  
   681  			select {
   682  			case <-stopCh:
   683  				return
   684  			default:
   685  				continue
   686  			}
   687  		}
   688  	}()
   689  
   690  	// Create the claims, again in a separate goroutine.
   691  	go func() {
   692  		for i := 0; i < objCount; i++ {
   693  			_, _ = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvcs[i], metav1.CreateOptions{})
   694  		}
   695  	}()
   696  
   697  	// wait until the binder pairs all claims
   698  	for i := 0; i < objCount; i++ {
   699  		waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound)
   700  		klog.V(1).Infof("%d claims bound", i+1)
   701  	}
   702  	// wait until the binder pairs all volumes
   703  	for i := 0; i < objCount; i++ {
   704  		waitForPersistentVolumePhase(testClient, pvs[i].Name, watchPV, v1.VolumeBound)
   705  		klog.V(1).Infof("%d claims bound", i+1)
   706  	}
   707  
   708  	klog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: claims are bound")
   709  	close(stopCh)
   710  
   711  	// check that everything is bound to something
   712  	for i := 0; i < objCount; i++ {
   713  		pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), pvs[i].Name, metav1.GetOptions{})
   714  		if err != nil {
   715  			t.Fatalf("Unexpected error getting pv: %v", err)
   716  		}
   717  		if pv.Spec.ClaimRef == nil {
   718  			t.Fatalf("PV %q is not bound", pv.Name)
   719  		}
   720  		klog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name)
   721  
   722  		pvc, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), pvcs[i].Name, metav1.GetOptions{})
   723  		if err != nil {
   724  			t.Fatalf("Unexpected error getting pvc: %v", err)
   725  		}
   726  		if pvc.Spec.VolumeName == "" {
   727  			t.Fatalf("PVC %q is not bound", pvc.Name)
   728  		}
   729  		klog.V(2).Infof("PVC %q is bound to PV %q", pvc.Name, pvc.Spec.VolumeName)
   730  	}
   731  	testSleep()
   732  }
   733  
   734  // TestPersistentVolumeControllerStartup tests startup of the controller.
   735  // The controller should not unbind any volumes when it starts.
   736  func TestPersistentVolumeControllerStartup(t *testing.T) {
   737  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   738  	defer s.TearDownFn()
   739  	namespaceName := "controller-startup"
   740  
   741  	objCount := getObjectCount()
   742  
   743  	const shortSyncPeriod = 2 * time.Second
   744  	syncPeriod := getSyncPeriod(shortSyncPeriod)
   745  
   746  	tCtx := ktesting.Init(t)
   747  	defer tCtx.Cancel("test has completed")
   748  	testClient, binder, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, shortSyncPeriod)
   749  	defer watchPV.Stop()
   750  	defer watchPVC.Stop()
   751  
   752  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   753  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   754  
   755  	// Create *bound* volumes and PVCs
   756  	pvs := make([]*v1.PersistentVolume, objCount)
   757  	pvcs := make([]*v1.PersistentVolumeClaim, objCount)
   758  	for i := 0; i < objCount; i++ {
   759  		pvName := "pv-startup-" + strconv.Itoa(i)
   760  		pvcName := "pvc-startup-" + strconv.Itoa(i)
   761  
   762  		pvc := createPVC(pvcName, ns.Name, "1G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "")
   763  		pvc.Annotations = map[string]string{"annBindCompleted": ""}
   764  		pvc.Spec.VolumeName = pvName
   765  		newPVC, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
   766  		if err != nil {
   767  			t.Fatalf("Cannot create claim %q: %v", pvc.Name, err)
   768  		}
   769  		// Save Bound status as a separate transaction
   770  		newPVC.Status.Phase = v1.ClaimBound
   771  		newPVC, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).UpdateStatus(context.TODO(), newPVC, metav1.UpdateOptions{})
   772  		if err != nil {
   773  			t.Fatalf("Cannot update claim status %q: %v", pvc.Name, err)
   774  		}
   775  		pvcs[i] = newPVC
   776  		// Drain watchPVC with all events generated by the PVC until it's bound
   777  		// We don't want to catch "PVC created with Status.Phase == Pending"
   778  		// later in this test.
   779  		waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound)
   780  
   781  		pv := createPV(pvName, "/tmp/foo"+strconv.Itoa(i), "1G",
   782  			[]v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain)
   783  		claimRef, err := ref.GetReference(legacyscheme.Scheme, newPVC)
   784  		if err != nil {
   785  			klog.V(3).Infof("unexpected error getting claim reference: %v", err)
   786  			return
   787  		}
   788  		pv.Spec.ClaimRef = claimRef
   789  		newPV, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
   790  		if err != nil {
   791  			t.Fatalf("Cannot create volume %q: %v", pv.Name, err)
   792  		}
   793  		// Save Bound status as a separate transaction
   794  		newPV.Status.Phase = v1.VolumeBound
   795  		newPV, err = testClient.CoreV1().PersistentVolumes().UpdateStatus(context.TODO(), newPV, metav1.UpdateOptions{})
   796  		if err != nil {
   797  			t.Fatalf("Cannot update volume status %q: %v", pv.Name, err)
   798  		}
   799  		pvs[i] = newPV
   800  		// Drain watchPV with all events generated by the PV until it's bound
   801  		// We don't want to catch "PV created with Status.Phase == Pending"
   802  		// later in this test.
   803  		waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound)
   804  	}
   805  
   806  	// Start the controller when all PVs and PVCs are already saved in etcd
   807  	informers.Start(tCtx.Done())
   808  	go binder.Run(tCtx)
   809  
   810  	// wait for at least two sync periods for changes. No volume should be
   811  	// Released and no claim should be Lost during this time.
   812  	timer := time.NewTimer(2 * syncPeriod)
   813  	defer timer.Stop()
   814  	finished := false
   815  	for !finished {
   816  		select {
   817  		case volumeEvent := <-watchPV.ResultChan():
   818  			volume, ok := volumeEvent.Object.(*v1.PersistentVolume)
   819  			if !ok {
   820  				continue
   821  			}
   822  			if volume.Status.Phase != v1.VolumeBound {
   823  				t.Errorf("volume %s unexpectedly changed state to %s", volume.Name, volume.Status.Phase)
   824  			}
   825  
   826  		case claimEvent := <-watchPVC.ResultChan():
   827  			claim, ok := claimEvent.Object.(*v1.PersistentVolumeClaim)
   828  			if !ok {
   829  				continue
   830  			}
   831  			if claim.Status.Phase != v1.ClaimBound {
   832  				t.Errorf("claim %s unexpectedly changed state to %s", claim.Name, claim.Status.Phase)
   833  			}
   834  
   835  		case <-timer.C:
   836  			// Wait finished
   837  			klog.V(2).Infof("Wait finished")
   838  			finished = true
   839  		}
   840  	}
   841  
   842  	// check that everything is bound to something
   843  	for i := 0; i < objCount; i++ {
   844  		pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), pvs[i].Name, metav1.GetOptions{})
   845  		if err != nil {
   846  			t.Fatalf("Unexpected error getting pv: %v", err)
   847  		}
   848  		if pv.Spec.ClaimRef == nil {
   849  			t.Fatalf("PV %q is not bound", pv.Name)
   850  		}
   851  		klog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name)
   852  
   853  		pvc, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), pvcs[i].Name, metav1.GetOptions{})
   854  		if err != nil {
   855  			t.Fatalf("Unexpected error getting pvc: %v", err)
   856  		}
   857  		if pvc.Spec.VolumeName == "" {
   858  			t.Fatalf("PVC %q is not bound", pvc.Name)
   859  		}
   860  		klog.V(2).Infof("PVC %q is bound to PV %q", pvc.Name, pvc.Spec.VolumeName)
   861  	}
   862  }
   863  
   864  // TestPersistentVolumeProvisionMultiPVCs tests provisioning of many PVCs.
   865  // This test is configurable by KUBE_INTEGRATION_PV_* variables.
   866  func TestPersistentVolumeProvisionMultiPVCs(t *testing.T) {
   867  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   868  	defer s.TearDownFn()
   869  	namespaceName := "provision-multi-pvs"
   870  
   871  	tCtx := ktesting.Init(t)
   872  	defer tCtx.Cancel("test has completed")
   873  	testClient, binder, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
   874  	defer watchPV.Stop()
   875  	defer watchPVC.Stop()
   876  
   877  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   878  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   879  
   880  	// NOTE: This test cannot run in parallel, because it is creating and deleting
   881  	// non-namespaced objects (PersistenceVolumes and StorageClasses).
   882  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   883  	defer testClient.StorageV1().StorageClasses().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   884  
   885  	storageClass := storage.StorageClass{
   886  		TypeMeta: metav1.TypeMeta{
   887  			Kind: "StorageClass",
   888  		},
   889  		ObjectMeta: metav1.ObjectMeta{
   890  			Name: "gold",
   891  		},
   892  		Provisioner: provisionerPluginName,
   893  	}
   894  	testClient.StorageV1().StorageClasses().Create(context.TODO(), &storageClass, metav1.CreateOptions{})
   895  
   896  	informers.Start(tCtx.Done())
   897  	go binder.Run(tCtx)
   898  
   899  	objCount := getObjectCount()
   900  	pvcs := make([]*v1.PersistentVolumeClaim, objCount)
   901  	for i := 0; i < objCount; i++ {
   902  		pvc := createPVC("pvc-provision-"+strconv.Itoa(i), ns.Name, "1G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "gold")
   903  		pvcs[i] = pvc
   904  	}
   905  
   906  	klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: start")
   907  	// Create the claims in a separate goroutine to pop events from watchPVC
   908  	// early. It gets stuck with >3000 claims.
   909  	go func() {
   910  		for i := 0; i < objCount; i++ {
   911  			_, _ = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvcs[i], metav1.CreateOptions{})
   912  		}
   913  	}()
   914  
   915  	// Wait until the controller provisions and binds all of them
   916  	for i := 0; i < objCount; i++ {
   917  		waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound)
   918  		klog.V(1).Infof("%d claims bound", i+1)
   919  	}
   920  	klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: claims are bound")
   921  
   922  	// check that we have enough bound PVs
   923  	pvList, err := testClient.CoreV1().PersistentVolumes().List(context.TODO(), metav1.ListOptions{})
   924  	if err != nil {
   925  		t.Fatalf("Failed to list volumes: %s", err)
   926  	}
   927  	if len(pvList.Items) != objCount {
   928  		t.Fatalf("Expected to get %d volumes, got %d", objCount, len(pvList.Items))
   929  	}
   930  	for i := 0; i < objCount; i++ {
   931  		pv := &pvList.Items[i]
   932  		if pv.Status.Phase != v1.VolumeBound {
   933  			t.Fatalf("Expected volume %s to be bound, is %s instead", pv.Name, pv.Status.Phase)
   934  		}
   935  		klog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name)
   936  	}
   937  
   938  	// Delete the claims
   939  	for i := 0; i < objCount; i++ {
   940  		_ = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvcs[i].Name, metav1.DeleteOptions{})
   941  	}
   942  
   943  	// Wait for the PVs to get deleted by listing remaining volumes
   944  	// (delete events were unreliable)
   945  	for {
   946  		volumes, err := testClient.CoreV1().PersistentVolumes().List(context.TODO(), metav1.ListOptions{})
   947  		if err != nil {
   948  			t.Fatalf("Failed to list volumes: %v", err)
   949  		}
   950  
   951  		klog.V(1).Infof("%d volumes remaining", len(volumes.Items))
   952  		if len(volumes.Items) == 0 {
   953  			break
   954  		}
   955  		time.Sleep(time.Second)
   956  	}
   957  	klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: volumes are deleted")
   958  }
   959  
   960  // TestPersistentVolumeMultiPVsDiffAccessModes tests binding of one PVC to two
   961  // PVs with different access modes.
   962  func TestPersistentVolumeMultiPVsDiffAccessModes(t *testing.T) {
   963  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd())
   964  	defer s.TearDownFn()
   965  	namespaceName := "multi-pvs-diff-access"
   966  
   967  	tCtx := ktesting.Init(t)
   968  	defer tCtx.Cancel("test has completed")
   969  	testClient, controller, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
   970  	defer watchPV.Stop()
   971  	defer watchPVC.Stop()
   972  
   973  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
   974  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
   975  
   976  	// NOTE: This test cannot run in parallel, because it is creating and deleting
   977  	// non-namespaced objects (PersistenceVolumes).
   978  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   979  
   980  	informers.Start(tCtx.Done())
   981  	go controller.Run(tCtx)
   982  
   983  	// This PV will be claimed, released, and deleted
   984  	pvRwo := createPV("pv-rwo", "/tmp/foo", "10G",
   985  		[]v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain)
   986  	pvRwm := createPV("pv-rwm", "/tmp/bar", "10G",
   987  		[]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain)
   988  
   989  	pvc := createPVC("pvc-rwm", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, "")
   990  
   991  	_, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvRwm, metav1.CreateOptions{})
   992  	if err != nil {
   993  		t.Errorf("Failed to create PersistentVolume: %v", err)
   994  	}
   995  	_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvRwo, metav1.CreateOptions{})
   996  	if err != nil {
   997  		t.Errorf("Failed to create PersistentVolume: %v", err)
   998  	}
   999  	t.Log("volumes created")
  1000  
  1001  	_, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
  1002  	if err != nil {
  1003  		t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
  1004  	}
  1005  	t.Log("claim created")
  1006  
  1007  	// wait until the controller pairs the volume and claim
  1008  	waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound)
  1009  	t.Log("volume bound")
  1010  	waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound)
  1011  	t.Log("claim bound")
  1012  
  1013  	// only RWM PV is bound
  1014  	pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-rwo", metav1.GetOptions{})
  1015  	if err != nil {
  1016  		t.Fatalf("Unexpected error getting pv: %v", err)
  1017  	}
  1018  	if pv.Spec.ClaimRef != nil {
  1019  		t.Fatalf("ReadWriteOnce PV shouldn't be bound")
  1020  	}
  1021  	pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-rwm", metav1.GetOptions{})
  1022  	if err != nil {
  1023  		t.Fatalf("Unexpected error getting pv: %v", err)
  1024  	}
  1025  	if pv.Spec.ClaimRef == nil {
  1026  		t.Fatalf("ReadWriteMany PV should be bound")
  1027  	}
  1028  	if pv.Spec.ClaimRef.Name != pvc.Name {
  1029  		t.Fatalf("Bind mismatch! Expected %s but got %s", pvc.Name, pv.Spec.ClaimRef.Name)
  1030  	}
  1031  
  1032  	// deleting a claim releases the volume
  1033  	if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}); err != nil {
  1034  		t.Errorf("error deleting claim %s", pvc.Name)
  1035  	}
  1036  	t.Log("claim deleted")
  1037  
  1038  	waitForAnyPersistentVolumePhase(watchPV, v1.VolumeReleased)
  1039  	t.Log("volume released")
  1040  }
  1041  
  1042  // TestRetroactiveStorageClassAssignment tests PVC retroactive storage class
  1043  // assignment and binding of PVCs with storage class name set to nil or "" with
  1044  // and without presence of a default SC.
  1045  func TestRetroactiveStorageClassAssignment(t *testing.T) {
  1046  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=DefaultStorageClass"}, framework.SharedEtcd())
  1047  	defer s.TearDownFn()
  1048  	namespaceName := "retro-pvc-sc"
  1049  	defaultStorageClassName := "gold"
  1050  	storageClassName := "silver"
  1051  
  1052  	tCtx := ktesting.Init(t)
  1053  	defer tCtx.Cancel("test has completed")
  1054  	testClient, binder, informers, watchPV, watchPVC := createClients(tCtx, namespaceName, t, s, defaultSyncPeriod)
  1055  	defer watchPV.Stop()
  1056  	defer watchPVC.Stop()
  1057  
  1058  	ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
  1059  	defer framework.DeleteNamespaceOrDie(testClient, ns, t)
  1060  
  1061  	// NOTE: This test cannot run in parallel, because it is creating and deleting
  1062  	// non-namespaced objects (PersistenceVolumes and StorageClasses).
  1063  	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
  1064  	defer testClient.CoreV1().PersistentVolumeClaims(namespaceName).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
  1065  	defer testClient.StorageV1().StorageClasses().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
  1066  
  1067  	// Create non default SC (extra SC - should not be used by any PVC in this test).
  1068  	nonDefaultSC := storage.StorageClass{
  1069  		TypeMeta: metav1.TypeMeta{
  1070  			Kind: "StorageClass",
  1071  		},
  1072  		ObjectMeta: metav1.ObjectMeta{
  1073  			Name: storageClassName,
  1074  			Annotations: map[string]string{
  1075  				util.IsDefaultStorageClassAnnotation: "false",
  1076  			},
  1077  		},
  1078  		Provisioner: provisionerPluginName,
  1079  	}
  1080  	if _, err := testClient.StorageV1().StorageClasses().Create(context.TODO(), &nonDefaultSC, metav1.CreateOptions{}); err != nil {
  1081  		t.Errorf("Failed to create a storage class: %v", err)
  1082  	}
  1083  
  1084  	informers.Start(tCtx.Done())
  1085  	go binder.Run(tCtx)
  1086  
  1087  	klog.V(2).Infof("TestRetroactiveStorageClassAssignment: start")
  1088  
  1089  	// 1. Test that PV with SC set to "" binds to PVC with SC set to nil while default SC does not exist (verifies that feature enablement does not break old behavior).
  1090  	pv1 := createPVWithStorageClass("pv-1", "/tmp/foo", "5G", "",
  1091  		[]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain)
  1092  	_, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv1, metav1.CreateOptions{})
  1093  	if err != nil {
  1094  		t.Errorf("Failed to create PersistentVolume: %v", err)
  1095  	}
  1096  
  1097  	pvc1 := createPVCWithNilStorageClass("pvc-1", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany})
  1098  	_, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc1, metav1.CreateOptions{})
  1099  	if err != nil {
  1100  		t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
  1101  	}
  1102  
  1103  	// Wait until the controller pairs the volume.
  1104  	waitForPersistentVolumePhase(testClient, pv1.Name, watchPV, v1.VolumeBound)
  1105  	t.Log("volume bound")
  1106  	waitForPersistentVolumeClaimPhase(testClient, pvc1.Name, ns.Name, watchPVC, v1.ClaimBound)
  1107  	t.Log("claim bound")
  1108  
  1109  	pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-1", metav1.GetOptions{})
  1110  	if err != nil {
  1111  		t.Fatalf("Unexpected error getting pv: %v", err)
  1112  	}
  1113  	if pv.Spec.ClaimRef == nil {
  1114  		t.Fatalf("PV %s with \"\" storage class should have been bound to PVC %s that has nil storage class", pv1.Name, pvc1.Name)
  1115  	}
  1116  	if pv.Spec.ClaimRef.Name != pvc1.Name {
  1117  		t.Fatalf("Bind mismatch! Expected %s but got %s", pvc1.Name, pv.Spec.ClaimRef.Name)
  1118  	}
  1119  
  1120  	// 2. Test that retroactive SC assignment works - default SC is created after creation of PVC with nil SC.
  1121  	pvcRetro := createPVCWithNilStorageClass("pvc-provision-noclass", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany})
  1122  	if _, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvcRetro, metav1.CreateOptions{}); err != nil {
  1123  		t.Errorf("Failed to create PVC: %v", err)
  1124  	}
  1125  	t.Log("claim created")
  1126  
  1127  	// Create default SC.
  1128  	defaultSC := storage.StorageClass{
  1129  		TypeMeta: metav1.TypeMeta{
  1130  			Kind: "StorageClass",
  1131  		},
  1132  		ObjectMeta: metav1.ObjectMeta{
  1133  			Name: defaultStorageClassName,
  1134  			Annotations: map[string]string{
  1135  				util.IsDefaultStorageClassAnnotation: "true",
  1136  			},
  1137  		},
  1138  		Provisioner: provisionerPluginName,
  1139  	}
  1140  	if _, err := testClient.StorageV1().StorageClasses().Create(context.TODO(), &defaultSC, metav1.CreateOptions{}); err != nil {
  1141  		t.Errorf("Failed to create a storage class: %v", err)
  1142  	}
  1143  
  1144  	// Verify SC was assigned retroactively to PVC.
  1145  	if _, ok := waitForPersistentVolumeClaimStorageClass(t, pvcRetro.Name, defaultStorageClassName, watchPVC, 20*time.Second); !ok {
  1146  		t.Errorf("Expected claim %s to get a storage class %s assigned retroactively", pvcRetro.Name, defaultStorageClassName)
  1147  	}
  1148  
  1149  	waitForPersistentVolumeClaimPhase(testClient, pvcRetro.Name, ns.Name, watchPVC, v1.ClaimBound)
  1150  
  1151  	// 3. Test that a new claim with nil class will still bind to PVs with SC set to "" (if available) and SC will not be assigned retroactively.
  1152  	pv3 := createPVWithStorageClass("pv-3", "/tmp/bar", "5G", "",
  1153  		[]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain)
  1154  	_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv3, metav1.CreateOptions{})
  1155  	if err != nil {
  1156  		t.Errorf("Failed to create PersistentVolume: %v", err)
  1157  	}
  1158  	waitForPersistentVolumePhase(testClient, pv3.Name, watchPV, v1.VolumeAvailable)
  1159  
  1160  	pvc3 := createPVCWithNilStorageClass("pvc-3", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany})
  1161  	if _, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc3, metav1.CreateOptions{}); err != nil {
  1162  		t.Errorf("Failed to create PVC: %v", err)
  1163  	}
  1164  	t.Log("claim created")
  1165  
  1166  	waitForPersistentVolumeClaimPhase(testClient, pvc3.Name, ns.Name, watchPVC, v1.ClaimBound)
  1167  
  1168  	pvc, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), "pvc-3", metav1.GetOptions{})
  1169  	if err != nil {
  1170  		t.Fatalf("Unexpected error getting pv: %v", err)
  1171  	}
  1172  	if pvc.Spec.StorageClassName != nil {
  1173  		t.Errorf("claim %s should still have nil storage class because it bound to existing PV", pvc.Name)
  1174  	}
  1175  
  1176  	// Create another PV which should remain unbound.
  1177  	pvUnbound := createPVWithStorageClass("pv-unbound", "/tmp/bar", "5G", "",
  1178  		[]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain)
  1179  	_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvUnbound, metav1.CreateOptions{})
  1180  	if err != nil {
  1181  		t.Errorf("Failed to create PersistentVolume: %v", err)
  1182  	}
  1183  
  1184  	waitForPersistentVolumePhase(testClient, pvUnbound.Name, watchPV, v1.VolumeAvailable)
  1185  
  1186  	pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-unbound", metav1.GetOptions{})
  1187  	if err != nil {
  1188  		t.Fatalf("Unexpected error getting pv: %v", err)
  1189  	}
  1190  	if pv.Spec.ClaimRef != nil {
  1191  		t.Fatalf("PV %s shouldn't be bound", pvUnbound.Name)
  1192  	}
  1193  
  1194  	// Remove the PV to not interfere with next test.
  1195  	testClient.CoreV1().PersistentVolumes().Delete(context.TODO(), pvUnbound.Name, metav1.DeleteOptions{})
  1196  
  1197  	// 4. Test that PV with SC set to "" binds to PVC with SC set to "" while default SC exists.
  1198  	// This tests that the feature enablement and default SC presence does not break this binding.
  1199  	// If this breaks there would be no way to ever bind PVs with SC set to "".
  1200  	pvc4 := createPVC("pvc-4", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, "")
  1201  	_, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc4, metav1.CreateOptions{})
  1202  	if err != nil {
  1203  		t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
  1204  	}
  1205  
  1206  	pv4 := createPVWithStorageClass("pv-4", "/tmp/bar", "5G", "",
  1207  		[]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain)
  1208  	_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv4, metav1.CreateOptions{})
  1209  	if err != nil {
  1210  		t.Errorf("Failed to create PersistentVolume: %v", err)
  1211  	}
  1212  
  1213  	// Wait until the controller pairs the volume.
  1214  	waitForPersistentVolumePhase(testClient, pv4.Name, watchPV, v1.VolumeBound)
  1215  	t.Log("volume bound")
  1216  	waitForPersistentVolumeClaimPhase(testClient, pvc4.Name, ns.Name, watchPVC, v1.ClaimBound)
  1217  	t.Log("claim bound")
  1218  
  1219  	pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-4", metav1.GetOptions{})
  1220  	if err != nil {
  1221  		t.Fatalf("Unexpected error getting pv: %v", err)
  1222  	}
  1223  	if pv.Spec.ClaimRef == nil {
  1224  		t.Fatalf("PV %s with \"\" storage class should have been bound to PVC %s that also has \"\" storage class", pv4.Name, pvc4.Name)
  1225  	}
  1226  	if pv.Spec.ClaimRef.Name != pvc4.Name {
  1227  		t.Fatalf("Bind mismatch! Expected PV %s to bind to PVC %s but instead it bound to PVC %s", pv.Name, pvc4.Name, pv.Spec.ClaimRef.Name)
  1228  	}
  1229  }
  1230  
  1231  func waitForPersistentVolumePhase(client *clientset.Clientset, pvName string, w watch.Interface, phase v1.PersistentVolumePhase) {
  1232  	// Check if the volume is already in requested phase
  1233  	volume, err := client.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{})
  1234  	if err == nil && volume.Status.Phase == phase {
  1235  		return
  1236  	}
  1237  
  1238  	// Wait for the phase
  1239  	for {
  1240  		event := <-w.ResultChan()
  1241  		volume, ok := event.Object.(*v1.PersistentVolume)
  1242  		if !ok {
  1243  			continue
  1244  		}
  1245  		if volume.Status.Phase == phase && volume.Name == pvName {
  1246  			klog.V(2).Infof("volume %q is %s", volume.Name, phase)
  1247  			break
  1248  		}
  1249  	}
  1250  }
  1251  
  1252  func waitForPersistentVolumeClaimPhase(client *clientset.Clientset, claimName, namespace string, w watch.Interface, phase v1.PersistentVolumeClaimPhase) {
  1253  	// Check if the claim is already in requested phase
  1254  	claim, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(context.TODO(), claimName, metav1.GetOptions{})
  1255  	if err == nil && claim.Status.Phase == phase {
  1256  		return
  1257  	}
  1258  
  1259  	// Wait for the phase
  1260  	for {
  1261  		event := <-w.ResultChan()
  1262  		claim, ok := event.Object.(*v1.PersistentVolumeClaim)
  1263  		if !ok {
  1264  			continue
  1265  		}
  1266  		if claim.Status.Phase == phase && claim.Name == claimName {
  1267  			klog.V(2).Infof("claim %q is %s", claim.Name, phase)
  1268  			break
  1269  		}
  1270  	}
  1271  }
  1272  
  1273  func waitForAnyPersistentVolumePhase(w watch.Interface, phase v1.PersistentVolumePhase) {
  1274  	for {
  1275  		event := <-w.ResultChan()
  1276  		volume, ok := event.Object.(*v1.PersistentVolume)
  1277  		if !ok {
  1278  			continue
  1279  		}
  1280  		if volume.Status.Phase == phase {
  1281  			klog.V(2).Infof("volume %q is %s", volume.Name, phase)
  1282  			break
  1283  		}
  1284  	}
  1285  }
  1286  
  1287  func waitForAnyPersistentVolumeClaimPhase(w watch.Interface, phase v1.PersistentVolumeClaimPhase) {
  1288  	for {
  1289  		event := <-w.ResultChan()
  1290  		claim, ok := event.Object.(*v1.PersistentVolumeClaim)
  1291  		if !ok {
  1292  			continue
  1293  		}
  1294  		if claim.Status.Phase == phase {
  1295  			klog.V(2).Infof("claim %q is %s", claim.Name, phase)
  1296  			break
  1297  		}
  1298  	}
  1299  }
  1300  
  1301  func waitForPersistentVolumeClaimStorageClass(t *testing.T, claimName, scName string, w watch.Interface, duration time.Duration) (*v1.PersistentVolumeClaim, bool) {
  1302  	stopTimer := time.NewTimer(duration)
  1303  	defer stopTimer.Stop()
  1304  
  1305  	// Wait for the storage class
  1306  	for {
  1307  		select {
  1308  		case event := <-w.ResultChan():
  1309  			claim, ok := event.Object.(*v1.PersistentVolumeClaim)
  1310  			if ok {
  1311  				t.Logf("Watching claim %s", claim.Name)
  1312  			} else {
  1313  				t.Errorf("Watch closed unexpectedly")
  1314  			}
  1315  			if claim.Spec.StorageClassName == nil {
  1316  				t.Logf("Claim %v does not yet have expected storage class %v", claim.Name, scName)
  1317  				continue
  1318  			}
  1319  			if *claim.Spec.StorageClassName == scName && claim.Name == claimName {
  1320  				t.Logf("Claim %s now has expected storage class %s", claim.Name, *claim.Spec.StorageClassName)
  1321  				return claim, true
  1322  			}
  1323  		case <-stopTimer.C:
  1324  			return nil, false
  1325  		}
  1326  
  1327  	}
  1328  }
  1329  
  1330  func createClients(ctx context.Context, namespaceName string, t *testing.T, s *kubeapiservertesting.TestServer, syncPeriod time.Duration) (*clientset.Clientset, *persistentvolumecontroller.PersistentVolumeController, informers.SharedInformerFactory, watch.Interface, watch.Interface) {
  1331  	// Use higher QPS and Burst, there is a test for race conditions which
  1332  	// creates many objects and default values were too low.
  1333  	binderConfig := restclient.CopyConfig(s.ClientConfig)
  1334  	binderConfig.QPS = 1000000
  1335  	binderConfig.Burst = 1000000
  1336  	binderClient := clientset.NewForConfigOrDie(binderConfig)
  1337  	testConfig := restclient.CopyConfig(s.ClientConfig)
  1338  	testConfig.QPS = 1000000
  1339  	testConfig.Burst = 1000000
  1340  	testClient := clientset.NewForConfigOrDie(testConfig)
  1341  
  1342  	host := volumetest.NewFakeVolumeHost(t, "/tmp/fake", nil, nil)
  1343  	plugin := &volumetest.FakeVolumePlugin{
  1344  		PluginName:             provisionerPluginName,
  1345  		Host:                   host,
  1346  		Config:                 volume.VolumeConfig{},
  1347  		LastProvisionerOptions: volume.VolumeOptions{},
  1348  		NewAttacherCallCount:   0,
  1349  		NewDetacherCallCount:   0,
  1350  		Mounters:               nil,
  1351  		Unmounters:             nil,
  1352  		Attachers:              nil,
  1353  		Detachers:              nil,
  1354  	}
  1355  	plugins := []volume.VolumePlugin{plugin}
  1356  	cloud := &fakecloud.Cloud{}
  1357  	informers := informers.NewSharedInformerFactory(testClient, getSyncPeriod(syncPeriod))
  1358  	ctrl, err := persistentvolumecontroller.NewController(
  1359  		ctx,
  1360  		persistentvolumecontroller.ControllerParameters{
  1361  			KubeClient:                binderClient,
  1362  			SyncPeriod:                getSyncPeriod(syncPeriod),
  1363  			VolumePlugins:             plugins,
  1364  			Cloud:                     cloud,
  1365  			VolumeInformer:            informers.Core().V1().PersistentVolumes(),
  1366  			ClaimInformer:             informers.Core().V1().PersistentVolumeClaims(),
  1367  			ClassInformer:             informers.Storage().V1().StorageClasses(),
  1368  			PodInformer:               informers.Core().V1().Pods(),
  1369  			NodeInformer:              informers.Core().V1().Nodes(),
  1370  			EnableDynamicProvisioning: true,
  1371  		})
  1372  	if err != nil {
  1373  		t.Fatalf("Failed to construct PersistentVolumes: %v", err)
  1374  	}
  1375  
  1376  	watchPV, err := testClient.CoreV1().PersistentVolumes().Watch(context.TODO(), metav1.ListOptions{})
  1377  	if err != nil {
  1378  		t.Fatalf("Failed to watch PersistentVolumes: %v", err)
  1379  	}
  1380  	watchPVC, err := testClient.CoreV1().PersistentVolumeClaims(namespaceName).Watch(context.TODO(), metav1.ListOptions{})
  1381  	if err != nil {
  1382  		t.Fatalf("Failed to watch PersistentVolumeClaims: %v", err)
  1383  	}
  1384  
  1385  	return testClient, ctrl, informers, watchPV, watchPVC
  1386  }
  1387  
  1388  func createPV(name, path, cap string, mode []v1.PersistentVolumeAccessMode, reclaim v1.PersistentVolumeReclaimPolicy) *v1.PersistentVolume {
  1389  	return &v1.PersistentVolume{
  1390  		ObjectMeta: metav1.ObjectMeta{Name: name},
  1391  		Spec: v1.PersistentVolumeSpec{
  1392  			PersistentVolumeSource:        v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{Path: path}},
  1393  			Capacity:                      v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)},
  1394  			AccessModes:                   mode,
  1395  			PersistentVolumeReclaimPolicy: reclaim,
  1396  		},
  1397  	}
  1398  }
  1399  
  1400  func createPVWithStorageClass(name, path, cap, scName string, mode []v1.PersistentVolumeAccessMode, reclaim v1.PersistentVolumeReclaimPolicy) *v1.PersistentVolume {
  1401  	return &v1.PersistentVolume{
  1402  		ObjectMeta: metav1.ObjectMeta{Name: name},
  1403  		Spec: v1.PersistentVolumeSpec{
  1404  			PersistentVolumeSource:        v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{Path: path}},
  1405  			Capacity:                      v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)},
  1406  			AccessModes:                   mode,
  1407  			PersistentVolumeReclaimPolicy: reclaim,
  1408  			StorageClassName:              scName,
  1409  		},
  1410  	}
  1411  }
  1412  
  1413  func createPVC(name, namespace, cap string, mode []v1.PersistentVolumeAccessMode, class string) *v1.PersistentVolumeClaim {
  1414  	return &v1.PersistentVolumeClaim{
  1415  		ObjectMeta: metav1.ObjectMeta{
  1416  			Name:      name,
  1417  			Namespace: namespace,
  1418  		},
  1419  		Spec: v1.PersistentVolumeClaimSpec{
  1420  			Resources:        v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)}},
  1421  			AccessModes:      mode,
  1422  			StorageClassName: &class,
  1423  		},
  1424  	}
  1425  }
  1426  
  1427  func createPVCWithNilStorageClass(name, namespace, cap string, mode []v1.PersistentVolumeAccessMode) *v1.PersistentVolumeClaim {
  1428  	return &v1.PersistentVolumeClaim{
  1429  		ObjectMeta: metav1.ObjectMeta{
  1430  			Name:      name,
  1431  			Namespace: namespace,
  1432  		},
  1433  		Spec: v1.PersistentVolumeClaimSpec{
  1434  			Resources:   v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)}},
  1435  			AccessModes: mode,
  1436  		},
  1437  	}
  1438  }
  1439  

View as plain text