1
16
17 package apps
18
19 import (
20 "context"
21 "encoding/json"
22 "fmt"
23 "reflect"
24 "regexp"
25 "strconv"
26 "strings"
27 "sync"
28 "time"
29
30 "github.com/google/go-cmp/cmp"
31 "github.com/onsi/ginkgo/v2"
32 "github.com/onsi/gomega"
33
34 appsv1 "k8s.io/api/apps/v1"
35 autoscalingv1 "k8s.io/api/autoscaling/v1"
36 v1 "k8s.io/api/core/v1"
37 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
38 "k8s.io/apimachinery/pkg/fields"
39 klabels "k8s.io/apimachinery/pkg/labels"
40 "k8s.io/apimachinery/pkg/runtime/schema"
41 "k8s.io/apimachinery/pkg/types"
42 "k8s.io/apimachinery/pkg/util/intstr"
43 "k8s.io/apimachinery/pkg/util/strategicpatch"
44 "k8s.io/apimachinery/pkg/util/wait"
45 "k8s.io/apimachinery/pkg/watch"
46 clientset "k8s.io/client-go/kubernetes"
47 "k8s.io/client-go/tools/cache"
48 watchtools "k8s.io/client-go/tools/watch"
49 "k8s.io/client-go/util/retry"
50 "k8s.io/kubernetes/test/e2e/feature"
51 "k8s.io/kubernetes/test/e2e/framework"
52 e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
53 e2enode "k8s.io/kubernetes/test/e2e/framework/node"
54 e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
55 e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
56 e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
57 e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
58 e2estatefulset "k8s.io/kubernetes/test/e2e/framework/statefulset"
59 imageutils "k8s.io/kubernetes/test/utils/image"
60 admissionapi "k8s.io/pod-security-admission/api"
61 "k8s.io/utils/pointer"
62 )
63
64 const (
65 zookeeperManifestPath = "test/e2e/testing-manifests/statefulset/zookeeper"
66 mysqlGaleraManifestPath = "test/e2e/testing-manifests/statefulset/mysql-galera"
67 redisManifestPath = "test/e2e/testing-manifests/statefulset/redis"
68 cockroachDBManifestPath = "test/e2e/testing-manifests/statefulset/cockroachdb"
69
70 restartCluster = true
71
72
73 readTimeout = 60 * time.Second
74
75
76 statefulSetPoll = 10 * time.Second
77
78 statefulSetTimeout = 10 * time.Minute
79
80 statefulPodTimeout = 5 * time.Minute
81
82 testFinalizer = "example.com/test-finalizer"
83 )
84
85 var httpProbe = &v1.Probe{
86 ProbeHandler: v1.ProbeHandler{
87 HTTPGet: &v1.HTTPGetAction{
88 Path: "/index.html",
89 Port: intstr.IntOrString{IntVal: 80},
90 },
91 },
92 PeriodSeconds: 1,
93 SuccessThreshold: 1,
94 FailureThreshold: 1,
95 }
96
97
98
99 var _ = SIGDescribe("StatefulSet", func() {
100 f := framework.NewDefaultFramework("statefulset")
101 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
102 var ns string
103 var c clientset.Interface
104
105 ginkgo.BeforeEach(func() {
106 c = f.ClientSet
107 ns = f.Namespace.Name
108 })
109
110 ginkgo.Describe("Basic StatefulSet functionality [StatefulSetBasic]", func() {
111 ssName := "ss"
112 labels := map[string]string{
113 "foo": "bar",
114 "baz": "blah",
115 }
116 headlessSvcName := "test"
117 var statefulPodMounts, podMounts []v1.VolumeMount
118 var ss *appsv1.StatefulSet
119
120 ginkgo.BeforeEach(func(ctx context.Context) {
121 statefulPodMounts = []v1.VolumeMount{{Name: "datadir", MountPath: "/data/"}}
122 podMounts = []v1.VolumeMount{{Name: "home", MountPath: "/home"}}
123 ss = e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 2, statefulPodMounts, podMounts, labels)
124
125 ginkgo.By("Creating service " + headlessSvcName + " in namespace " + ns)
126 headlessService := e2eservice.CreateServiceSpec(headlessSvcName, "", true, labels)
127 _, err := c.CoreV1().Services(ns).Create(ctx, headlessService, metav1.CreateOptions{})
128 framework.ExpectNoError(err)
129 })
130
131 ginkgo.AfterEach(func(ctx context.Context) {
132 if ginkgo.CurrentSpecReport().Failed() {
133 e2eoutput.DumpDebugInfo(ctx, c, ns)
134 }
135 framework.Logf("Deleting all statefulset in ns %v", ns)
136 e2estatefulset.DeleteAllStatefulSets(ctx, c, ns)
137 })
138
139
140
141 ginkgo.It("should provide basic identity", func(ctx context.Context) {
142 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
143 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
144 *(ss.Spec.Replicas) = 3
145 e2estatefulset.PauseNewPods(ss)
146
147 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
148 framework.ExpectNoError(err)
149
150 ginkgo.By("Saturating stateful set " + ss.Name)
151 e2estatefulset.Saturate(ctx, c, ss)
152
153 ginkgo.By("Verifying statefulset mounted data directory is usable")
154 framework.ExpectNoError(e2estatefulset.CheckMount(ctx, c, ss, "/data"))
155
156 ginkgo.By("Verifying statefulset provides a stable hostname for each pod")
157 framework.ExpectNoError(e2estatefulset.CheckHostname(ctx, c, ss))
158
159 ginkgo.By("Verifying statefulset set proper service name")
160 framework.ExpectNoError(e2estatefulset.CheckServiceName(ss, headlessSvcName))
161
162 cmd := "echo $(hostname) | dd of=/data/hostname conv=fsync"
163 ginkgo.By("Running " + cmd + " in all stateful pods")
164 framework.ExpectNoError(e2estatefulset.ExecInStatefulPods(ctx, c, ss, cmd))
165
166 cmd = "ln -s /data/hostname /data/hostname-symlink"
167 ginkgo.By("Running " + cmd + " in all stateful pods")
168 framework.ExpectNoError(e2estatefulset.ExecInStatefulPods(ctx, c, ss, cmd))
169
170 ginkgo.By("Restarting statefulset " + ss.Name)
171 e2estatefulset.Restart(ctx, c, ss)
172 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
173
174 ginkgo.By("Verifying statefulset mounted data directory is usable")
175 framework.ExpectNoError(e2estatefulset.CheckMount(ctx, c, ss, "/data"))
176
177 cmd = "if [ \"$(cat /data/hostname)\" = \"$(hostname)\" ]; then exit 0; else exit 1; fi"
178 ginkgo.By("Running " + cmd + " in all stateful pods")
179 framework.ExpectNoError(e2estatefulset.ExecInStatefulPods(ctx, c, ss, cmd))
180
181 cmd = "if [ \"$(cat /data/hostname-symlink)\" = \"$(hostname)\" ]; then exit 0; else exit 1; fi"
182 ginkgo.By("Running " + cmd + " in all stateful pods")
183 framework.ExpectNoError(e2estatefulset.ExecInStatefulPods(ctx, c, ss, cmd))
184 })
185
186
187
188 ginkgo.It("should adopt matching orphans and release non-matching pods", func(ctx context.Context) {
189 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
190 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
191 *(ss.Spec.Replicas) = 1
192 e2estatefulset.PauseNewPods(ss)
193
194
195
196 kind := ss.Kind
197 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
198 framework.ExpectNoError(err)
199 ss.Kind = kind
200
201 ginkgo.By("Saturating stateful set " + ss.Name)
202 e2estatefulset.Saturate(ctx, c, ss)
203 pods := e2estatefulset.GetPodList(ctx, c, ss)
204 gomega.Expect(pods.Items).To(gomega.HaveLen(int(*ss.Spec.Replicas)))
205
206 ginkgo.By("Checking that stateful set pods are created with ControllerRef")
207 pod := pods.Items[0]
208 controllerRef := metav1.GetControllerOf(&pod)
209 gomega.Expect(controllerRef).ToNot(gomega.BeNil())
210 gomega.Expect(controllerRef.Kind).To(gomega.Equal(ss.Kind))
211 gomega.Expect(controllerRef.Name).To(gomega.Equal(ss.Name))
212 gomega.Expect(controllerRef.UID).To(gomega.Equal(ss.UID))
213
214 ginkgo.By("Orphaning one of the stateful set's pods")
215 e2epod.NewPodClient(f).Update(ctx, pod.Name, func(pod *v1.Pod) {
216 pod.OwnerReferences = nil
217 })
218
219 ginkgo.By("Checking that the stateful set readopts the pod")
220 gomega.Expect(e2epod.WaitForPodCondition(ctx, c, pod.Namespace, pod.Name, "adopted", statefulSetTimeout,
221 func(pod *v1.Pod) (bool, error) {
222 controllerRef := metav1.GetControllerOf(pod)
223 if controllerRef == nil {
224 return false, nil
225 }
226 if controllerRef.Kind != ss.Kind || controllerRef.Name != ss.Name || controllerRef.UID != ss.UID {
227 return false, fmt.Errorf("pod has wrong controllerRef: %v", controllerRef)
228 }
229 return true, nil
230 },
231 )).To(gomega.Succeed(), "wait for pod %q to be readopted", pod.Name)
232
233 ginkgo.By("Removing the labels from one of the stateful set's pods")
234 prevLabels := pod.Labels
235 e2epod.NewPodClient(f).Update(ctx, pod.Name, func(pod *v1.Pod) {
236 pod.Labels = nil
237 })
238
239 ginkgo.By("Checking that the stateful set releases the pod")
240 gomega.Expect(e2epod.WaitForPodCondition(ctx, c, pod.Namespace, pod.Name, "released", statefulSetTimeout,
241 func(pod *v1.Pod) (bool, error) {
242 controllerRef := metav1.GetControllerOf(pod)
243 if controllerRef != nil {
244 return false, nil
245 }
246 return true, nil
247 },
248 )).To(gomega.Succeed(), "wait for pod %q to be released", pod.Name)
249
250
251 ginkgo.By("Readding labels to the stateful set's pod")
252 e2epod.NewPodClient(f).Update(ctx, pod.Name, func(pod *v1.Pod) {
253 pod.Labels = prevLabels
254 })
255
256 ginkgo.By("Checking that the stateful set readopts the pod")
257 gomega.Expect(e2epod.WaitForPodCondition(ctx, c, pod.Namespace, pod.Name, "adopted", statefulSetTimeout,
258 func(pod *v1.Pod) (bool, error) {
259 controllerRef := metav1.GetControllerOf(pod)
260 if controllerRef == nil {
261 return false, nil
262 }
263 if controllerRef.Kind != ss.Kind || controllerRef.Name != ss.Name || controllerRef.UID != ss.UID {
264 return false, fmt.Errorf("pod has wrong controllerRef: %v", controllerRef)
265 }
266 return true, nil
267 },
268 )).To(gomega.Succeed(), "wait for pod %q to be readopted", pod.Name)
269 })
270
271
272
273 ginkgo.It("should not deadlock when a pod's predecessor fails", func(ctx context.Context) {
274 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
275 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
276 *(ss.Spec.Replicas) = 2
277 e2estatefulset.PauseNewPods(ss)
278
279 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
280 framework.ExpectNoError(err)
281
282 e2estatefulset.WaitForRunning(ctx, c, 1, 0, ss)
283
284 ginkgo.By("Resuming stateful pod at index 0.")
285 e2estatefulset.ResumeNextPod(ctx, c, ss)
286
287 ginkgo.By("Waiting for stateful pod at index 1 to enter running.")
288 e2estatefulset.WaitForRunning(ctx, c, 2, 1, ss)
289
290
291
292
293
294 ginkgo.By("Deleting healthy stateful pod at index 0.")
295 deleteStatefulPodAtIndex(ctx, c, 0, ss)
296
297 ginkgo.By("Confirming stateful pod at index 0 is recreated.")
298 e2estatefulset.WaitForRunning(ctx, c, 2, 1, ss)
299
300 ginkgo.By("Resuming stateful pod at index 1.")
301 e2estatefulset.ResumeNextPod(ctx, c, ss)
302
303 ginkgo.By("Confirming all stateful pods in statefulset are created.")
304 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
305 })
306
307
308
309 ginkgo.It("should perform rolling updates and roll backs of template modifications with PVCs", func(ctx context.Context) {
310 ginkgo.By("Creating a new StatefulSet with PVCs")
311 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
312 *(ss.Spec.Replicas) = 3
313 rollbackTest(ctx, c, ns, ss)
314 })
315
316
321 framework.ConformanceIt("should perform rolling updates and roll backs of template modifications", func(ctx context.Context) {
322 ginkgo.By("Creating a new StatefulSet")
323 ss := e2estatefulset.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels)
324 rollbackTest(ctx, c, ns, ss)
325 })
326
327
332 framework.ConformanceIt("should perform canary updates and phased rolling updates of template modifications", func(ctx context.Context) {
333 ginkgo.By("Creating a new StatefulSet")
334 ss := e2estatefulset.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels)
335 setHTTPProbe(ss)
336 ss.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{
337 Type: appsv1.RollingUpdateStatefulSetStrategyType,
338 RollingUpdate: func() *appsv1.RollingUpdateStatefulSetStrategy {
339 return &appsv1.RollingUpdateStatefulSetStrategy{
340 Partition: pointer.Int32(3),
341 }
342 }(),
343 }
344 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
345 framework.ExpectNoError(err)
346 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
347 ss = waitForStatus(ctx, c, ss)
348 currentRevision, updateRevision := ss.Status.CurrentRevision, ss.Status.UpdateRevision
349 gomega.Expect(currentRevision).To(gomega.Equal(updateRevision), "StatefulSet %s/%s created with update revision %s not equal to current revision %s",
350 ss.Namespace, ss.Name, updateRevision, currentRevision)
351 pods := e2estatefulset.GetPodList(ctx, c, ss)
352 for i := range pods.Items {
353 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, currentRevision), "Pod %s/%s revision %s is not equal to currentRevision %s",
354 pods.Items[i].Namespace,
355 pods.Items[i].Name,
356 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
357 currentRevision)
358 }
359 newImage := NewWebserverImage
360 oldImage := ss.Spec.Template.Spec.Containers[0].Image
361
362 ginkgo.By(fmt.Sprintf("Updating stateful set template: update image from %s to %s", oldImage, newImage))
363 gomega.Expect(oldImage).NotTo(gomega.Equal(newImage), "Incorrect test setup: should update to a different image")
364 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
365 update.Spec.Template.Spec.Containers[0].Image = newImage
366 })
367 framework.ExpectNoError(err)
368
369 ginkgo.By("Creating a new revision")
370 ss = waitForStatus(ctx, c, ss)
371 currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision
372 gomega.Expect(currentRevision).NotTo(gomega.Equal(updateRevision), "Current revision should not equal update revision during rolling update")
373
374 ginkgo.By("Not applying an update when the partition is greater than the number of replicas")
375 for i := range pods.Items {
376 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), "Pod %s/%s has image %s not equal to current image %s",
377 pods.Items[i].Namespace,
378 pods.Items[i].Name,
379 pods.Items[i].Spec.Containers[0].Image,
380 oldImage)
381 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, currentRevision), "Pod %s/%s has revision %s not equal to current revision %s",
382 pods.Items[i].Namespace,
383 pods.Items[i].Name,
384 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
385 currentRevision)
386 }
387
388 ginkgo.By("Performing a canary update")
389 ss.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{
390 Type: appsv1.RollingUpdateStatefulSetStrategyType,
391 RollingUpdate: func() *appsv1.RollingUpdateStatefulSetStrategy {
392 return &appsv1.RollingUpdateStatefulSetStrategy{
393 Partition: pointer.Int32(2),
394 }
395 }(),
396 }
397 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
398 update.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{
399 Type: appsv1.RollingUpdateStatefulSetStrategyType,
400 RollingUpdate: func() *appsv1.RollingUpdateStatefulSetStrategy {
401 return &appsv1.RollingUpdateStatefulSetStrategy{
402 Partition: pointer.Int32(2),
403 }
404 }(),
405 }
406 })
407 framework.ExpectNoError(err)
408 ss, pods = waitForPartitionedRollingUpdate(ctx, c, ss)
409 for i := range pods.Items {
410 if i < int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) {
411 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), "Pod %s/%s has image %s not equal to current image %s",
412 pods.Items[i].Namespace,
413 pods.Items[i].Name,
414 pods.Items[i].Spec.Containers[0].Image,
415 oldImage)
416 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, currentRevision), "Pod %s/%s has revision %s not equal to current revision %s",
417 pods.Items[i].Namespace,
418 pods.Items[i].Name,
419 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
420 currentRevision)
421 } else {
422 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), "Pod %s/%s has image %s not equal to new image %s",
423 pods.Items[i].Namespace,
424 pods.Items[i].Name,
425 pods.Items[i].Spec.Containers[0].Image,
426 newImage)
427 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, updateRevision), "Pod %s/%s has revision %s not equal to new revision %s",
428 pods.Items[i].Namespace,
429 pods.Items[i].Name,
430 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
431 updateRevision)
432 }
433 }
434
435 ginkgo.By("Restoring Pods to the correct revision when they are deleted")
436 deleteStatefulPodAtIndex(ctx, c, 0, ss)
437 deleteStatefulPodAtIndex(ctx, c, 2, ss)
438 e2estatefulset.WaitForRunningAndReady(ctx, c, 3, ss)
439 ss = getStatefulSet(ctx, c, ss.Namespace, ss.Name)
440 pods = e2estatefulset.GetPodList(ctx, c, ss)
441 for i := range pods.Items {
442 if i < int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) {
443 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), "Pod %s/%s has image %s not equal to current image %s",
444 pods.Items[i].Namespace,
445 pods.Items[i].Name,
446 pods.Items[i].Spec.Containers[0].Image,
447 oldImage)
448 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, currentRevision), "Pod %s/%s has revision %s not equal to current revision %s",
449 pods.Items[i].Namespace,
450 pods.Items[i].Name,
451 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
452 currentRevision)
453 } else {
454 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), "Pod %s/%s has image %s not equal to new image %s",
455 pods.Items[i].Namespace,
456 pods.Items[i].Name,
457 pods.Items[i].Spec.Containers[0].Image,
458 newImage)
459 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, updateRevision), "Pod %s/%s has revision %s not equal to new revision %s",
460 pods.Items[i].Namespace,
461 pods.Items[i].Name,
462 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
463 updateRevision)
464 }
465 }
466
467 ginkgo.By("Performing a phased rolling update")
468 for i := int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) - 1; i >= 0; i-- {
469 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
470 update.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{
471 Type: appsv1.RollingUpdateStatefulSetStrategyType,
472 RollingUpdate: func() *appsv1.RollingUpdateStatefulSetStrategy {
473 j := int32(i)
474 return &appsv1.RollingUpdateStatefulSetStrategy{
475 Partition: &j,
476 }
477 }(),
478 }
479 })
480 framework.ExpectNoError(err)
481 ss, pods = waitForPartitionedRollingUpdate(ctx, c, ss)
482 for i := range pods.Items {
483 if i < int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) {
484 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), "Pod %s/%s has image %s not equal to current image %s",
485 pods.Items[i].Namespace,
486 pods.Items[i].Name,
487 pods.Items[i].Spec.Containers[0].Image,
488 oldImage)
489 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, currentRevision), "Pod %s/%s has revision %s not equal to current revision %s",
490 pods.Items[i].Namespace,
491 pods.Items[i].Name,
492 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
493 currentRevision)
494 } else {
495 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), "Pod %s/%s has image %s not equal to new image %s",
496 pods.Items[i].Namespace,
497 pods.Items[i].Name,
498 pods.Items[i].Spec.Containers[0].Image,
499 newImage)
500 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, updateRevision), "Pod %s/%s has revision %s not equal to new revision %s",
501 pods.Items[i].Namespace,
502 pods.Items[i].Name,
503 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
504 updateRevision)
505 }
506 }
507 }
508 gomega.Expect(ss.Status.CurrentRevision).To(gomega.Equal(updateRevision), "StatefulSet %s/%s current revision %s does not equal update revision %s on update completion",
509 ss.Namespace,
510 ss.Name,
511 ss.Status.CurrentRevision,
512 updateRevision)
513
514 })
515
516 ginkgo.It("should perform canary updates and phased rolling updates of template modifications for partiton1 and delete pod-0 without failing container", func(ctx context.Context) {
517 ginkgo.By("Creating a new StatefulSet without failing container")
518 ss := e2estatefulset.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels)
519 deletingPodForRollingUpdatePartitionTest(ctx, f, c, ns, ss)
520 })
521
522 ginkgo.It("should perform canary updates and phased rolling updates of template modifications for partiton1 and delete pod-0 with failing container", func(ctx context.Context) {
523 ginkgo.By("Creating a new StatefulSet with failing container")
524 ss := e2estatefulset.NewStatefulSet("ss3", ns, headlessSvcName, 3, nil, nil, labels)
525 ss.Spec.Template.Spec.Containers = append(ss.Spec.Template.Spec.Containers, v1.Container{
526 Name: "sleep-exit-with-1",
527 Image: imageutils.GetE2EImage(imageutils.BusyBox),
528 Command: []string{"sh", "-c"},
529 Args: []string{`
530 echo "Running in pod $POD_NAME"
531 _term(){
532 echo "Received SIGTERM signal"
533 if [ "${POD_NAME}" = "ss3-0" ]; then
534 exit 1
535 else
536 exit 0
537 fi
538 }
539 trap _term SIGTERM
540 while true; do
541 echo "Running in infinite loop in $POD_NAME"
542 sleep 1
543 done
544 `,
545 },
546 Env: []v1.EnvVar{
547 {
548 Name: "POD_NAME",
549 ValueFrom: &v1.EnvVarSource{
550 FieldRef: &v1.ObjectFieldSelector{
551 APIVersion: "v1",
552 FieldPath: "metadata.name",
553 },
554 },
555 },
556 },
557 })
558 deletingPodForRollingUpdatePartitionTest(ctx, f, c, ns, ss)
559 })
560
561
562
563 ginkgo.It("should implement legacy replacement when the update strategy is OnDelete", func(ctx context.Context) {
564 ginkgo.By("Creating a new StatefulSet")
565 ss := e2estatefulset.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels)
566 setHTTPProbe(ss)
567 ss.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{
568 Type: appsv1.OnDeleteStatefulSetStrategyType,
569 }
570 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
571 framework.ExpectNoError(err)
572 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
573 ss = waitForStatus(ctx, c, ss)
574 currentRevision, updateRevision := ss.Status.CurrentRevision, ss.Status.UpdateRevision
575 gomega.Expect(currentRevision).To(gomega.Equal(updateRevision), "StatefulSet %s/%s created with update revision %s not equal to current revision %s",
576 ss.Namespace, ss.Name, updateRevision, currentRevision)
577 pods := e2estatefulset.GetPodList(ctx, c, ss)
578 for i := range pods.Items {
579 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, currentRevision), "Pod %s/%s revision %s is not equal to current revision %s",
580 pods.Items[i].Namespace,
581 pods.Items[i].Name,
582 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
583 currentRevision)
584 }
585
586 ginkgo.By("Restoring Pods to the current revision")
587 deleteStatefulPodAtIndex(ctx, c, 0, ss)
588 deleteStatefulPodAtIndex(ctx, c, 1, ss)
589 deleteStatefulPodAtIndex(ctx, c, 2, ss)
590 e2estatefulset.WaitForRunningAndReady(ctx, c, 3, ss)
591 ss = getStatefulSet(ctx, c, ss.Namespace, ss.Name)
592 pods = e2estatefulset.GetPodList(ctx, c, ss)
593 for i := range pods.Items {
594 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, currentRevision), "Pod %s/%s revision %s is not equal to current revision %s",
595 pods.Items[i].Namespace,
596 pods.Items[i].Name,
597 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
598 currentRevision)
599 }
600 newImage := NewWebserverImage
601 oldImage := ss.Spec.Template.Spec.Containers[0].Image
602
603 ginkgo.By(fmt.Sprintf("Updating stateful set template: update image from %s to %s", oldImage, newImage))
604 gomega.Expect(oldImage).NotTo(gomega.Equal(newImage), "Incorrect test setup: should update to a different image")
605 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
606 update.Spec.Template.Spec.Containers[0].Image = newImage
607 })
608 framework.ExpectNoError(err)
609
610 ginkgo.By("Creating a new revision")
611 ss = waitForStatus(ctx, c, ss)
612 currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision
613 gomega.Expect(currentRevision).NotTo(gomega.Equal(updateRevision), "Current revision should not equal update revision during rolling update")
614
615 ginkgo.By("Recreating Pods at the new revision")
616 deleteStatefulPodAtIndex(ctx, c, 0, ss)
617 deleteStatefulPodAtIndex(ctx, c, 1, ss)
618 deleteStatefulPodAtIndex(ctx, c, 2, ss)
619 e2estatefulset.WaitForRunningAndReady(ctx, c, 3, ss)
620 ss = getStatefulSet(ctx, c, ss.Namespace, ss.Name)
621 pods = e2estatefulset.GetPodList(ctx, c, ss)
622 for i := range pods.Items {
623 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), "Pod %s/%s has image %s not equal to new image %s",
624 pods.Items[i].Namespace,
625 pods.Items[i].Name,
626 pods.Items[i].Spec.Containers[0].Image,
627 newImage)
628 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, updateRevision), "Pod %s/%s has revision %s not equal to current revision %s",
629 pods.Items[i].Namespace,
630 pods.Items[i].Name,
631 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
632 updateRevision)
633 }
634 })
635
636
641 framework.ConformanceIt("Scaling should happen in predictable order and halt if any stateful pod is unhealthy", f.WithSlow(), func(ctx context.Context) {
642 psLabels := klabels.Set(labels)
643 w := &cache.ListWatch{
644 WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) {
645 options.LabelSelector = psLabels.AsSelector().String()
646 return f.ClientSet.CoreV1().Pods(ns).Watch(ctx, options)
647 },
648 }
649 ginkgo.By("Initializing watcher for selector " + psLabels.String())
650 pl, err := f.ClientSet.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
651 LabelSelector: psLabels.AsSelector().String(),
652 })
653 framework.ExpectNoError(err)
654
655
656 wg := sync.WaitGroup{}
657 var orderErr error
658 wg.Add(1)
659 go func() {
660 defer ginkgo.GinkgoRecover()
661 defer wg.Done()
662
663 expectedOrder := []string{ssName + "-0", ssName + "-1", ssName + "-2"}
664 ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, statefulSetTimeout)
665 defer cancel()
666
667 _, orderErr = watchtools.Until(ctx, pl.ResourceVersion, w, func(event watch.Event) (bool, error) {
668 if event.Type != watch.Added {
669 return false, nil
670 }
671 pod := event.Object.(*v1.Pod)
672 if pod.Name == expectedOrder[0] {
673 expectedOrder = expectedOrder[1:]
674 }
675 return len(expectedOrder) == 0, nil
676 })
677 }()
678
679 ginkgo.By("Creating stateful set " + ssName + " in namespace " + ns)
680 ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, nil, nil, psLabels)
681 setHTTPProbe(ss)
682 ss, err = c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
683 framework.ExpectNoError(err)
684
685 ginkgo.By("Waiting until all stateful set " + ssName + " replicas will be running in namespace " + ns)
686 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
687
688 ginkgo.By("Confirming that stateful set scale up will halt with unhealthy stateful pod")
689 breakHTTPProbe(ctx, c, ss)
690 waitForRunningAndNotReady(ctx, c, *ss.Spec.Replicas, ss)
691 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 0)
692 e2estatefulset.UpdateReplicas(ctx, c, ss, 3)
693 confirmStatefulPodCount(ctx, c, 1, ss, 10*time.Second, true)
694
695 ginkgo.By("Scaling up stateful set " + ssName + " to 3 replicas and waiting until all of them will be running in namespace " + ns)
696 restoreHTTPProbe(ctx, c, ss)
697 e2estatefulset.WaitForRunningAndReady(ctx, c, 3, ss)
698
699 ginkgo.By("Verifying that stateful set " + ssName + " was scaled up in order")
700 wg.Wait()
701 framework.ExpectNoError(orderErr)
702
703 ginkgo.By("Scale down will halt with unhealthy stateful pod")
704 pl, err = f.ClientSet.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
705 LabelSelector: psLabels.AsSelector().String(),
706 })
707 framework.ExpectNoError(err)
708
709
710 wg.Add(1)
711 go func() {
712 defer ginkgo.GinkgoRecover()
713 defer wg.Done()
714
715 expectedOrder := []string{ssName + "-2", ssName + "-1", ssName + "-0"}
716 ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, statefulSetTimeout)
717 defer cancel()
718
719 _, orderErr = watchtools.Until(ctx, pl.ResourceVersion, w, func(event watch.Event) (bool, error) {
720 if event.Type != watch.Deleted {
721 return false, nil
722 }
723 pod := event.Object.(*v1.Pod)
724 if pod.Name == expectedOrder[0] {
725 expectedOrder = expectedOrder[1:]
726 }
727 return len(expectedOrder) == 0, nil
728 })
729 }()
730
731 breakHTTPProbe(ctx, c, ss)
732 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 0)
733 waitForRunningAndNotReady(ctx, c, 3, ss)
734 e2estatefulset.UpdateReplicas(ctx, c, ss, 0)
735 confirmStatefulPodCount(ctx, c, 3, ss, 10*time.Second, true)
736
737 ginkgo.By("Scaling down stateful set " + ssName + " to 0 replicas and waiting until none of pods will run in namespace" + ns)
738 restoreHTTPProbe(ctx, c, ss)
739 e2estatefulset.Scale(ctx, c, ss, 0)
740
741 ginkgo.By("Verifying that stateful set " + ssName + " was scaled down in reverse order")
742 wg.Wait()
743 framework.ExpectNoError(orderErr)
744 })
745
746
751 framework.ConformanceIt("Burst scaling should run to completion even with unhealthy pods", f.WithSlow(), func(ctx context.Context) {
752 psLabels := klabels.Set(labels)
753
754 ginkgo.By("Creating stateful set " + ssName + " in namespace " + ns)
755 ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, nil, nil, psLabels)
756 ss.Spec.PodManagementPolicy = appsv1.ParallelPodManagement
757 setHTTPProbe(ss)
758 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
759 framework.ExpectNoError(err)
760
761 ginkgo.By("Waiting until all stateful set " + ssName + " replicas will be running in namespace " + ns)
762 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
763
764 ginkgo.By("Confirming that stateful set scale up will not halt with unhealthy stateful pod")
765 breakHTTPProbe(ctx, c, ss)
766 waitForRunningAndNotReady(ctx, c, *ss.Spec.Replicas, ss)
767 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 0)
768 e2estatefulset.UpdateReplicas(ctx, c, ss, 3)
769 confirmStatefulPodCount(ctx, c, 3, ss, 10*time.Second, false)
770
771 ginkgo.By("Scaling up stateful set " + ssName + " to 3 replicas and waiting until all of them will be running in namespace " + ns)
772 restoreHTTPProbe(ctx, c, ss)
773 e2estatefulset.WaitForRunningAndReady(ctx, c, 3, ss)
774
775 ginkgo.By("Scale down will not halt with unhealthy stateful pod")
776 breakHTTPProbe(ctx, c, ss)
777 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 0)
778 waitForRunningAndNotReady(ctx, c, 3, ss)
779 e2estatefulset.UpdateReplicas(ctx, c, ss, 0)
780 confirmStatefulPodCount(ctx, c, 0, ss, 10*time.Second, false)
781
782 ginkgo.By("Scaling down stateful set " + ssName + " to 0 replicas and waiting until none of pods will run in namespace" + ns)
783 restoreHTTPProbe(ctx, c, ss)
784 e2estatefulset.Scale(ctx, c, ss, 0)
785 e2estatefulset.WaitForStatusReplicas(ctx, c, ss, 0)
786 })
787
788
793 framework.ConformanceIt("Should recreate evicted statefulset", func(ctx context.Context) {
794 podName := "test-pod"
795 statefulPodName := ssName + "-0"
796 ginkgo.By("Looking for a node to schedule stateful set and pod")
797 node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
798 framework.ExpectNoError(err)
799
800 ginkgo.By("Creating pod with conflicting port in namespace " + f.Namespace.Name)
801 conflictingPort := v1.ContainerPort{HostPort: 21017, ContainerPort: 21017, Name: "conflict"}
802 pod := &v1.Pod{
803 ObjectMeta: metav1.ObjectMeta{
804 Name: podName,
805 },
806 Spec: v1.PodSpec{
807 Containers: []v1.Container{
808 {
809 Name: "webserver",
810 Image: imageutils.GetE2EImage(imageutils.Httpd),
811 Ports: []v1.ContainerPort{conflictingPort},
812 },
813 },
814 NodeName: node.Name,
815 },
816 }
817 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
818 framework.ExpectNoError(err)
819 ginkgo.By("Waiting until pod " + podName + " will start running in namespace " + f.Namespace.Name)
820 if err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name); err != nil {
821 framework.Failf("Pod %v did not start running: %v", podName, err)
822 }
823
824 ginkgo.By("Creating statefulset with conflicting port in namespace " + f.Namespace.Name)
825 ss := e2estatefulset.NewStatefulSet(ssName, f.Namespace.Name, headlessSvcName, 1, nil, nil, labels)
826 statefulPodContainer := &ss.Spec.Template.Spec.Containers[0]
827 statefulPodContainer.Ports = append(statefulPodContainer.Ports, conflictingPort)
828 ss.Spec.Template.Spec.NodeName = node.Name
829 _, err = f.ClientSet.AppsV1().StatefulSets(f.Namespace.Name).Create(ctx, ss, metav1.CreateOptions{})
830 framework.ExpectNoError(err)
831
832 var initialStatefulPodUID types.UID
833 ginkgo.By("Waiting until stateful pod " + statefulPodName + " will be recreated and deleted at least once in namespace " + f.Namespace.Name)
834
835 fieldSelector := fields.OneTermEqualSelector("metadata.name", statefulPodName).String()
836 pl, err := f.ClientSet.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
837 FieldSelector: fieldSelector,
838 })
839 framework.ExpectNoError(err)
840 if len(pl.Items) > 0 {
841 pod := pl.Items[0]
842 framework.Logf("Observed stateful pod in namespace: %v, name: %v, uid: %v, status phase: %v. Waiting for statefulset controller to delete.",
843 pod.Namespace, pod.Name, pod.UID, pod.Status.Phase)
844 initialStatefulPodUID = pod.UID
845 }
846
847 lw := &cache.ListWatch{
848 WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) {
849 options.FieldSelector = fieldSelector
850 return f.ClientSet.CoreV1().Pods(f.Namespace.Name).Watch(ctx, options)
851 },
852 }
853 ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, statefulPodTimeout)
854 defer cancel()
855
856 _, err = watchtools.Until(ctx, pl.ResourceVersion, lw, func(event watch.Event) (bool, error) {
857 pod := event.Object.(*v1.Pod)
858 switch event.Type {
859 case watch.Deleted:
860 framework.Logf("Observed delete event for stateful pod %v in namespace %v", pod.Name, pod.Namespace)
861 if initialStatefulPodUID == "" {
862 return false, nil
863 }
864 return true, nil
865 }
866 framework.Logf("Observed stateful pod in namespace: %v, name: %v, uid: %v, status phase: %v. Waiting for statefulset controller to delete.",
867 pod.Namespace, pod.Name, pod.UID, pod.Status.Phase)
868 initialStatefulPodUID = pod.UID
869 return false, nil
870 })
871 if err != nil {
872 framework.Failf("Pod %v expected to be re-created at least once", statefulPodName)
873 }
874
875 ginkgo.By("Removing pod with conflicting port in namespace " + f.Namespace.Name)
876 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0))
877 framework.ExpectNoError(err)
878
879 ginkgo.By("Waiting when stateful pod " + statefulPodName + " will be recreated in namespace " + f.Namespace.Name + " and will be in running state")
880
881 gomega.Eventually(ctx, func() error {
882 statefulPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, statefulPodName, metav1.GetOptions{})
883 if err != nil {
884 return err
885 }
886 if statefulPod.Status.Phase != v1.PodRunning {
887 return fmt.Errorf("pod %v is not in running phase: %v", statefulPod.Name, statefulPod.Status.Phase)
888 } else if statefulPod.UID == initialStatefulPodUID {
889 return fmt.Errorf("pod %v wasn't recreated: %v == %v", statefulPod.Name, statefulPod.UID, initialStatefulPodUID)
890 }
891 return nil
892 }, statefulPodTimeout, 2*time.Second).Should(gomega.BeNil())
893 })
894
895
902 framework.ConformanceIt("should have a working scale subresource", func(ctx context.Context) {
903 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
904 ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, nil, nil, labels)
905 setHTTPProbe(ss)
906 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
907 framework.ExpectNoError(err)
908 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
909 waitForStatus(ctx, c, ss)
910
911 ginkgo.By("getting scale subresource")
912 scale, err := c.AppsV1().StatefulSets(ns).GetScale(ctx, ssName, metav1.GetOptions{})
913 if err != nil {
914 framework.Failf("Failed to get scale subresource: %v", err)
915 }
916 gomega.Expect(scale.Spec.Replicas).To(gomega.Equal(int32(1)))
917 gomega.Expect(scale.Status.Replicas).To(gomega.Equal(int32(1)))
918
919 ginkgo.By("updating a scale subresource")
920 scale.ResourceVersion = ""
921 scale.Spec.Replicas = 2
922 scaleResult, err := c.AppsV1().StatefulSets(ns).UpdateScale(ctx, ssName, scale, metav1.UpdateOptions{})
923 if err != nil {
924 framework.Failf("Failed to put scale subresource: %v", err)
925 }
926 gomega.Expect(scaleResult.Spec.Replicas).To(gomega.Equal(int32(2)))
927
928 ginkgo.By("verifying the statefulset Spec.Replicas was modified")
929 ss, err = c.AppsV1().StatefulSets(ns).Get(ctx, ssName, metav1.GetOptions{})
930 if err != nil {
931 framework.Failf("Failed to get statefulset resource: %v", err)
932 }
933 gomega.Expect(*(ss.Spec.Replicas)).To(gomega.Equal(int32(2)))
934
935 ginkgo.By("Patch a scale subresource")
936 scale.ResourceVersion = ""
937 scale.Spec.Replicas = 4
938 ssScalePatchPayload, err := json.Marshal(autoscalingv1.Scale{
939 Spec: autoscalingv1.ScaleSpec{
940 Replicas: scale.Spec.Replicas,
941 },
942 })
943 framework.ExpectNoError(err, "Could not Marshal JSON for patch payload")
944
945 _, err = c.AppsV1().StatefulSets(ns).Patch(ctx, ssName, types.StrategicMergePatchType, []byte(ssScalePatchPayload), metav1.PatchOptions{}, "scale")
946 framework.ExpectNoError(err, "Failed to patch stateful set: %v", err)
947
948 ginkgo.By("verifying the statefulset Spec.Replicas was modified")
949 ss, err = c.AppsV1().StatefulSets(ns).Get(ctx, ssName, metav1.GetOptions{})
950 framework.ExpectNoError(err, "Failed to get statefulset resource: %v", err)
951 gomega.Expect(*(ss.Spec.Replicas)).To(gomega.Equal(int32(4)), "statefulset should have 4 replicas")
952 })
953
954
962 framework.ConformanceIt("should list, patch and delete a collection of StatefulSets", func(ctx context.Context) {
963
964 ssPatchReplicas := int32(2)
965 ssPatchImage := imageutils.GetE2EImage(imageutils.Pause)
966 one := int64(1)
967 ssName := "test-ss"
968
969
970 ssPodLabels := map[string]string{
971 "name": "sample-pod",
972 "pod": WebserverImageName,
973 }
974 ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, nil, nil, ssPodLabels)
975 setHTTPProbe(ss)
976 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
977 framework.ExpectNoError(err)
978 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
979 waitForStatus(ctx, c, ss)
980
981 ginkgo.By("patching the StatefulSet")
982 ssPatch, err := json.Marshal(map[string]interface{}{
983 "metadata": map[string]interface{}{
984 "labels": map[string]string{"test-ss": "patched"},
985 },
986 "spec": map[string]interface{}{
987 "replicas": ssPatchReplicas,
988 "template": map[string]interface{}{
989 "spec": map[string]interface{}{
990 "TerminationGracePeriodSeconds": &one,
991 "containers": [1]map[string]interface{}{{
992 "name": ssName,
993 "image": ssPatchImage,
994 }},
995 },
996 },
997 },
998 })
999 framework.ExpectNoError(err, "failed to Marshal StatefulSet JSON patch")
1000 _, err = f.ClientSet.AppsV1().StatefulSets(ns).Patch(ctx, ssName, types.StrategicMergePatchType, []byte(ssPatch), metav1.PatchOptions{})
1001 framework.ExpectNoError(err, "failed to patch Set")
1002 ss, err = c.AppsV1().StatefulSets(ns).Get(ctx, ssName, metav1.GetOptions{})
1003 framework.ExpectNoError(err, "Failed to get statefulset resource: %v", err)
1004 gomega.Expect(*(ss.Spec.Replicas)).To(gomega.Equal(ssPatchReplicas), "statefulset should have 2 replicas")
1005 gomega.Expect(ss.Spec.Template.Spec.Containers[0].Image).To(gomega.Equal(ssPatchImage), "statefulset not using ssPatchImage. Is using %v", ss.Spec.Template.Spec.Containers[0].Image)
1006 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
1007 waitForStatus(ctx, c, ss)
1008
1009 ginkgo.By("Listing all StatefulSets")
1010 ssList, err := c.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{LabelSelector: "test-ss=patched"})
1011 framework.ExpectNoError(err, "failed to list StatefulSets")
1012 gomega.Expect(ssList.Items).To(gomega.HaveLen(1), "filtered list wasn't found")
1013
1014 ginkgo.By("Delete all of the StatefulSets")
1015 err = c.AppsV1().StatefulSets(ns).DeleteCollection(ctx, metav1.DeleteOptions{GracePeriodSeconds: &one}, metav1.ListOptions{LabelSelector: "test-ss=patched"})
1016 framework.ExpectNoError(err, "failed to delete StatefulSets")
1017
1018 ginkgo.By("Verify that StatefulSets have been deleted")
1019 ssList, err = c.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{LabelSelector: "test-ss=patched"})
1020 framework.ExpectNoError(err, "failed to list StatefulSets")
1021 gomega.Expect(ssList.Items).To(gomega.BeEmpty(), "filtered list should have no Statefulsets")
1022 })
1023
1024
1031 framework.ConformanceIt("should validate Statefulset Status endpoints", func(ctx context.Context) {
1032 ssClient := c.AppsV1().StatefulSets(ns)
1033 labelSelector := "e2e=testing"
1034
1035 w := &cache.ListWatch{
1036 WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
1037 options.LabelSelector = labelSelector
1038 return ssClient.Watch(ctx, options)
1039 },
1040 }
1041 ssList, err := c.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
1042 framework.ExpectNoError(err, "failed to list StatefulSets")
1043
1044 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1045 ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, nil, nil, labels)
1046 setHTTPProbe(ss)
1047 ss, err = c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1048 framework.ExpectNoError(err)
1049 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
1050 waitForStatus(ctx, c, ss)
1051
1052 ginkgo.By("Patch Statefulset to include a label")
1053 payload := []byte(`{"metadata":{"labels":{"e2e":"testing"}}}`)
1054 ss, err = ssClient.Patch(ctx, ssName, types.StrategicMergePatchType, payload, metav1.PatchOptions{})
1055 framework.ExpectNoError(err)
1056
1057 ginkgo.By("Getting /status")
1058 ssResource := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "statefulsets"}
1059 ssStatusUnstructured, err := f.DynamicClient.Resource(ssResource).Namespace(ns).Get(ctx, ssName, metav1.GetOptions{}, "status")
1060 framework.ExpectNoError(err, "Failed to fetch the status of replica set %s in namespace %s", ssName, ns)
1061 ssStatusBytes, err := json.Marshal(ssStatusUnstructured)
1062 framework.ExpectNoError(err, "Failed to marshal unstructured response. %v", err)
1063
1064 var ssStatus appsv1.StatefulSet
1065 err = json.Unmarshal(ssStatusBytes, &ssStatus)
1066 framework.ExpectNoError(err, "Failed to unmarshal JSON bytes to a Statefulset object type")
1067 framework.Logf("StatefulSet %s has Conditions: %#v", ssName, ssStatus.Status.Conditions)
1068
1069 ginkgo.By("updating the StatefulSet Status")
1070 var statusToUpdate, updatedStatus *appsv1.StatefulSet
1071
1072 err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
1073 statusToUpdate, err = ssClient.Get(ctx, ssName, metav1.GetOptions{})
1074 framework.ExpectNoError(err, "Unable to retrieve statefulset %s", ssName)
1075
1076 statusToUpdate.Status.Conditions = append(statusToUpdate.Status.Conditions, appsv1.StatefulSetCondition{
1077 Type: "StatusUpdate",
1078 Status: "True",
1079 Reason: "E2E",
1080 Message: "Set from e2e test",
1081 })
1082
1083 updatedStatus, err = ssClient.UpdateStatus(ctx, statusToUpdate, metav1.UpdateOptions{})
1084 return err
1085 })
1086 framework.ExpectNoError(err, "Failed to update status. %v", err)
1087 framework.Logf("updatedStatus.Conditions: %#v", updatedStatus.Status.Conditions)
1088
1089 ginkgo.By("watching for the statefulset status to be updated")
1090
1091 ctxUntil, cancel := context.WithTimeout(ctx, statefulSetTimeout)
1092 defer cancel()
1093
1094 _, err = watchtools.Until(ctxUntil, ssList.ResourceVersion, w, func(event watch.Event) (bool, error) {
1095
1096 if e, ok := event.Object.(*appsv1.StatefulSet); ok {
1097 found := e.ObjectMeta.Name == ss.ObjectMeta.Name &&
1098 e.ObjectMeta.Namespace == ss.ObjectMeta.Namespace &&
1099 e.ObjectMeta.Labels["e2e"] == ss.ObjectMeta.Labels["e2e"]
1100 if !found {
1101 framework.Logf("Observed Statefulset %v in namespace %v with annotations: %v & Conditions: %v", ss.ObjectMeta.Name, ss.ObjectMeta.Namespace, ss.Annotations, ss.Status.Conditions)
1102 return false, nil
1103 }
1104 for _, cond := range e.Status.Conditions {
1105 if cond.Type == "StatusUpdate" &&
1106 cond.Reason == "E2E" &&
1107 cond.Message == "Set from e2e test" {
1108 framework.Logf("Found Statefulset %v in namespace %v with labels: %v annotations: %v & Conditions: %v", ss.ObjectMeta.Name, ss.ObjectMeta.Namespace, ss.ObjectMeta.Labels, ss.Annotations, cond)
1109 return found, nil
1110 }
1111 framework.Logf("Observed Statefulset %v in namespace %v with annotations: %v & Conditions: %v", ss.ObjectMeta.Name, ss.ObjectMeta.Namespace, ss.Annotations, cond)
1112 }
1113 }
1114 object := strings.Split(fmt.Sprintf("%v", event.Object), "{")[0]
1115 framework.Logf("Observed %v event: %+v", object, event.Type)
1116 return false, nil
1117 })
1118 framework.ExpectNoError(err, "failed to locate Statefulset %v in namespace %v", ss.ObjectMeta.Name, ns)
1119 framework.Logf("Statefulset %s has an updated status", ssName)
1120
1121 ginkgo.By("patching the Statefulset Status")
1122 payload = []byte(`{"status":{"conditions":[{"type":"StatusPatched","status":"True"}]}}`)
1123 framework.Logf("Patch payload: %v", string(payload))
1124
1125 patchedStatefulSet, err := ssClient.Patch(ctx, ssName, types.MergePatchType, payload, metav1.PatchOptions{}, "status")
1126 framework.ExpectNoError(err, "Failed to patch status. %v", err)
1127 framework.Logf("Patched status conditions: %#v", patchedStatefulSet.Status.Conditions)
1128
1129 ginkgo.By("watching for the Statefulset status to be patched")
1130 ctxUntil, cancel = context.WithTimeout(ctx, statefulSetTimeout)
1131
1132 _, err = watchtools.Until(ctxUntil, ssList.ResourceVersion, w, func(event watch.Event) (bool, error) {
1133
1134 defer cancel()
1135 if e, ok := event.Object.(*appsv1.StatefulSet); ok {
1136 found := e.ObjectMeta.Name == ss.ObjectMeta.Name &&
1137 e.ObjectMeta.Namespace == ss.ObjectMeta.Namespace &&
1138 e.ObjectMeta.Labels["e2e"] == ss.ObjectMeta.Labels["e2e"]
1139 if !found {
1140 framework.Logf("Observed Statefulset %v in namespace %v with annotations: %v & Conditions: %v", ss.ObjectMeta.Name, ss.ObjectMeta.Namespace, ss.Annotations, ss.Status.Conditions)
1141 return false, nil
1142 }
1143 for _, cond := range e.Status.Conditions {
1144 if cond.Type == "StatusPatched" {
1145 framework.Logf("Found Statefulset %v in namespace %v with labels: %v annotations: %v & Conditions: %v", ss.ObjectMeta.Name, ss.ObjectMeta.Namespace, ss.ObjectMeta.Labels, ss.Annotations, cond)
1146 return found, nil
1147 }
1148 framework.Logf("Observed Statefulset %v in namespace %v with annotations: %v & Conditions: %v", ss.ObjectMeta.Name, ss.ObjectMeta.Namespace, ss.Annotations, cond)
1149 }
1150 }
1151 object := strings.Split(fmt.Sprintf("%v", event.Object), "{")[0]
1152 framework.Logf("Observed %v event: %+v", object, event.Type)
1153 return false, nil
1154 })
1155 })
1156 })
1157
1158 f.Describe("Deploy clustered applications", feature.StatefulSet, framework.WithSlow(), func() {
1159 var appTester *clusterAppTester
1160
1161 ginkgo.BeforeEach(func(ctx context.Context) {
1162 appTester = &clusterAppTester{client: c, ns: ns}
1163 })
1164
1165 ginkgo.AfterEach(func(ctx context.Context) {
1166 if ginkgo.CurrentSpecReport().Failed() {
1167 e2eoutput.DumpDebugInfo(ctx, c, ns)
1168 }
1169 framework.Logf("Deleting all statefulset in ns %v", ns)
1170 e2estatefulset.DeleteAllStatefulSets(ctx, c, ns)
1171 })
1172
1173
1174
1175 ginkgo.It("should creating a working zookeeper cluster", func(ctx context.Context) {
1176 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
1177 appTester.statefulPod = &zookeeperTester{client: c}
1178 appTester.run(ctx)
1179 })
1180
1181
1182
1183 ginkgo.It("should creating a working redis cluster", func(ctx context.Context) {
1184 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
1185 appTester.statefulPod = &redisTester{client: c}
1186 appTester.run(ctx)
1187 })
1188
1189
1190
1191 ginkgo.It("should creating a working mysql cluster", func(ctx context.Context) {
1192 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
1193 appTester.statefulPod = &mysqlGaleraTester{client: c}
1194 appTester.run(ctx)
1195 })
1196
1197
1198
1199 ginkgo.It("should creating a working CockroachDB cluster", func(ctx context.Context) {
1200 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
1201 appTester.statefulPod = &cockroachDBTester{client: c}
1202 appTester.run(ctx)
1203 })
1204 })
1205
1206
1207
1208 ginkgo.It("MinReadySeconds should be honored when enabled", func(ctx context.Context) {
1209 ssName := "test-ss"
1210 headlessSvcName := "test"
1211
1212 ssPodLabels := map[string]string{
1213 "name": "sample-pod",
1214 "pod": WebserverImageName,
1215 }
1216 ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, nil, nil, ssPodLabels)
1217 setHTTPProbe(ss)
1218 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1219 framework.ExpectNoError(err)
1220 e2estatefulset.WaitForStatusAvailableReplicas(ctx, c, ss, 1)
1221 })
1222
1223 ginkgo.It("AvailableReplicas should get updated accordingly when MinReadySeconds is enabled", func(ctx context.Context) {
1224 ssName := "test-ss"
1225 headlessSvcName := "test"
1226
1227 ssPodLabels := map[string]string{
1228 "name": "sample-pod",
1229 "pod": WebserverImageName,
1230 }
1231 ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 2, nil, nil, ssPodLabels)
1232 ss.Spec.MinReadySeconds = 30
1233 setHTTPProbe(ss)
1234 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1235 framework.ExpectNoError(err)
1236 e2estatefulset.WaitForStatusAvailableReplicas(ctx, c, ss, 0)
1237
1238 time.Sleep(5 * time.Second)
1239 ss, err = c.AppsV1().StatefulSets(ns).Get(ctx, ss.Name, metav1.GetOptions{})
1240 framework.ExpectNoError(err)
1241 if ss.Status.AvailableReplicas != 0 {
1242 framework.Failf("invalid number of availableReplicas: expected=%v received=%v", 0, ss.Status.AvailableReplicas)
1243 }
1244 e2estatefulset.WaitForStatusAvailableReplicas(ctx, c, ss, 2)
1245
1246 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
1247 update.Spec.MinReadySeconds = 3600
1248 })
1249 framework.ExpectNoError(err)
1250
1251 e2estatefulset.WaitForStatusAvailableReplicas(ctx, c, ss, 0)
1252
1253 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
1254 update.Spec.MinReadySeconds = 0
1255 })
1256 framework.ExpectNoError(err)
1257 e2estatefulset.WaitForStatusAvailableReplicas(ctx, c, ss, 2)
1258
1259 ginkgo.By("check availableReplicas are shown in status")
1260 out, err := e2ekubectl.RunKubectl(ns, "get", "statefulset", ss.Name, "-o=yaml")
1261 framework.ExpectNoError(err)
1262 if !strings.Contains(out, "availableReplicas: 2") {
1263 framework.Failf("invalid number of availableReplicas: expected=%v received=%v", 2, out)
1264 }
1265 })
1266
1267 ginkgo.Describe("Non-retain StatefulSetPersistentVolumeClaimPolicy", func() {
1268 ssName := "ss"
1269 labels := map[string]string{
1270 "foo": "bar",
1271 "baz": "blah",
1272 }
1273 headlessSvcName := "test"
1274 var statefulPodMounts, podMounts []v1.VolumeMount
1275 var ss *appsv1.StatefulSet
1276
1277 ginkgo.BeforeEach(func(ctx context.Context) {
1278 statefulPodMounts = []v1.VolumeMount{{Name: "datadir", MountPath: "/data/"}}
1279 podMounts = []v1.VolumeMount{{Name: "home", MountPath: "/home"}}
1280 ss = e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 2, statefulPodMounts, podMounts, labels)
1281
1282 ginkgo.By("Creating service " + headlessSvcName + " in namespace " + ns)
1283 headlessService := e2eservice.CreateServiceSpec(headlessSvcName, "", true, labels)
1284 _, err := c.CoreV1().Services(ns).Create(ctx, headlessService, metav1.CreateOptions{})
1285 framework.ExpectNoError(err)
1286 })
1287
1288 ginkgo.AfterEach(func(ctx context.Context) {
1289 if ginkgo.CurrentSpecReport().Failed() {
1290 e2eoutput.DumpDebugInfo(ctx, c, ns)
1291 }
1292 framework.Logf("Deleting all statefulset in ns %v", ns)
1293 e2estatefulset.DeleteAllStatefulSets(ctx, c, ns)
1294 })
1295
1296 ginkgo.It("should delete PVCs with a WhenDeleted policy", func(ctx context.Context) {
1297 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
1298 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1299 *(ss.Spec.Replicas) = 3
1300 ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
1301 WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
1302 }
1303 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1304 framework.ExpectNoError(err)
1305
1306 ginkgo.By("Confirming all 3 PVCs exist with their owner refs")
1307 err = verifyStatefulSetPVCsExistWithOwnerRefs(ctx, c, ss, []int{0, 1, 2}, true, false)
1308 framework.ExpectNoError(err)
1309
1310 ginkgo.By("Deleting stateful set " + ss.Name)
1311 err = c.AppsV1().StatefulSets(ns).Delete(ctx, ss.Name, metav1.DeleteOptions{})
1312 framework.ExpectNoError(err)
1313
1314 ginkgo.By("Verifying PVCs deleted")
1315 err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{})
1316 framework.ExpectNoError(err)
1317 })
1318
1319 ginkgo.It("should delete PVCs with a OnScaledown policy", func(ctx context.Context) {
1320 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
1321 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1322 *(ss.Spec.Replicas) = 3
1323 ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
1324 WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
1325 }
1326 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1327 framework.ExpectNoError(err)
1328
1329 ginkgo.By("Confirming all 3 PVCs exist")
1330 err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0, 1, 2})
1331 framework.ExpectNoError(err)
1332
1333 ginkgo.By("Scaling stateful set " + ss.Name + " to one replica")
1334 ss, err = e2estatefulset.Scale(ctx, c, ss, 1)
1335 framework.ExpectNoError(err)
1336
1337 ginkgo.By("Verifying all but one PVC deleted")
1338 err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0})
1339 framework.ExpectNoError(err)
1340 })
1341
1342 ginkgo.It("should delete PVCs after adopting pod (WhenDeleted)", func(ctx context.Context) {
1343 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
1344 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1345 *(ss.Spec.Replicas) = 3
1346 ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
1347 WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
1348 }
1349 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1350 framework.ExpectNoError(err)
1351
1352 ginkgo.By("Confirming all 3 PVCs exist with their owner refs")
1353 err = verifyStatefulSetPVCsExistWithOwnerRefs(ctx, c, ss, []int{0, 1, 2}, true, false)
1354 framework.ExpectNoError(err)
1355
1356 ginkgo.By("Orphaning the 3rd pod")
1357 patch, err := json.Marshal(metav1.ObjectMeta{
1358 OwnerReferences: []metav1.OwnerReference{},
1359 })
1360 framework.ExpectNoError(err, "Could not Marshal JSON for patch payload")
1361 _, err = c.CoreV1().Pods(ns).Patch(ctx, fmt.Sprintf("%s-2", ss.Name), types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "")
1362 framework.ExpectNoError(err, "Could not patch payload")
1363
1364 ginkgo.By("Deleting stateful set " + ss.Name)
1365 err = c.AppsV1().StatefulSets(ns).Delete(ctx, ss.Name, metav1.DeleteOptions{})
1366 framework.ExpectNoError(err)
1367
1368 ginkgo.By("Verifying PVCs deleted")
1369 err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{})
1370 framework.ExpectNoError(err)
1371 })
1372
1373 ginkgo.It("should delete PVCs after adopting pod (WhenScaled)", func(ctx context.Context) {
1374 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
1375 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1376 *(ss.Spec.Replicas) = 3
1377 ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
1378 WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
1379 }
1380 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1381 framework.ExpectNoError(err)
1382
1383 ginkgo.By("Confirming all 3 PVCs exist")
1384 err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0, 1, 2})
1385 framework.ExpectNoError(err)
1386
1387 ginkgo.By("Orphaning the 3rd pod")
1388 patch, err := json.Marshal(metav1.ObjectMeta{
1389 OwnerReferences: []metav1.OwnerReference{},
1390 })
1391 framework.ExpectNoError(err, "Could not Marshal JSON for patch payload")
1392 _, err = c.CoreV1().Pods(ns).Patch(ctx, fmt.Sprintf("%s-2", ss.Name), types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "")
1393 framework.ExpectNoError(err, "Could not patch payload")
1394
1395 ginkgo.By("Scaling stateful set " + ss.Name + " to one replica")
1396 ss, err = e2estatefulset.Scale(ctx, c, ss, 1)
1397 framework.ExpectNoError(err)
1398
1399 ginkgo.By("Verifying all but one PVC deleted")
1400 err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0})
1401 framework.ExpectNoError(err)
1402 })
1403 })
1404
1405 ginkgo.Describe("Automatically recreate PVC for pending pod when PVC is missing", func() {
1406 ssName := "ss"
1407 labels := map[string]string{
1408 "foo": "bar",
1409 "baz": "blah",
1410 }
1411 headlessSvcName := "test"
1412 var statefulPodMounts []v1.VolumeMount
1413 var ss *appsv1.StatefulSet
1414
1415 ginkgo.BeforeEach(func(ctx context.Context) {
1416 statefulPodMounts = []v1.VolumeMount{{Name: "datadir", MountPath: "/data/"}}
1417 ss = e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, statefulPodMounts, nil, labels)
1418 })
1419
1420 ginkgo.AfterEach(func(ctx context.Context) {
1421 if ginkgo.CurrentSpecReport().Failed() {
1422 e2eoutput.DumpDebugInfo(ctx, c, ns)
1423 }
1424 framework.Logf("Deleting all statefulset in ns %v", ns)
1425 e2estatefulset.DeleteAllStatefulSets(ctx, c, ns)
1426 })
1427
1428 f.It("PVC should be recreated when pod is pending due to missing PVC", f.WithDisruptive(), f.WithSerial(), func(ctx context.Context) {
1429 e2epv.SkipIfNoDefaultStorageClass(ctx, c)
1430
1431 readyNode, err := e2enode.GetRandomReadySchedulableNode(ctx, c)
1432 framework.ExpectNoError(err)
1433 hostLabel := "kubernetes.io/hostname"
1434 hostLabelVal := readyNode.Labels[hostLabel]
1435
1436 ss.Spec.Template.Spec.NodeSelector = map[string]string{hostLabel: hostLabelVal}
1437 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1438 _, err = c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1439 framework.ExpectNoError(err)
1440
1441 ginkgo.By("Confirming PVC exists")
1442 err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0})
1443 framework.ExpectNoError(err)
1444
1445 ginkgo.By("Confirming Pod is ready")
1446 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 1)
1447 podName := getStatefulSetPodNameAtIndex(0, ss)
1448 pod, err := c.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
1449 framework.ExpectNoError(err)
1450
1451 nodeName := pod.Spec.NodeName
1452 gomega.Expect(nodeName).To(gomega.Equal(readyNode.Name))
1453 node, err := c.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
1454 framework.ExpectNoError(err)
1455
1456 oldData, err := json.Marshal(node)
1457 framework.ExpectNoError(err)
1458
1459 node.Spec.Unschedulable = true
1460
1461 newData, err := json.Marshal(node)
1462 framework.ExpectNoError(err)
1463
1464
1465 patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
1466 framework.ExpectNoError(err)
1467 ginkgo.By("Cordoning Node")
1468 _, err = c.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{})
1469 framework.ExpectNoError(err)
1470 cordoned := true
1471
1472 defer func() {
1473 if cordoned {
1474 uncordonNode(ctx, c, oldData, newData, nodeName)
1475 }
1476 }()
1477
1478
1479 e2enode.WaitForNodeSchedulable(ctx, c, nodeName, 10*time.Second, false)
1480
1481 ginkgo.By("Deleting Pod")
1482 err = c.CoreV1().Pods(ns).Delete(ctx, podName, metav1.DeleteOptions{})
1483 framework.ExpectNoError(err)
1484
1485
1486 waitForStatusCurrentReplicas(ctx, c, ss, 1)
1487 _, err = c.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
1488 framework.ExpectNoError(err)
1489
1490 pvcList, err := c.CoreV1().PersistentVolumeClaims(ns).List(ctx, metav1.ListOptions{LabelSelector: klabels.Everything().String()})
1491 framework.ExpectNoError(err)
1492 gomega.Expect(pvcList.Items).To(gomega.HaveLen(1))
1493 pvcName := pvcList.Items[0].Name
1494
1495 ginkgo.By("Deleting PVC")
1496 err = c.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, pvcName, metav1.DeleteOptions{})
1497 framework.ExpectNoError(err)
1498
1499 uncordonNode(ctx, c, oldData, newData, nodeName)
1500 cordoned = false
1501
1502 ginkgo.By("Confirming PVC recreated")
1503 err = verifyStatefulSetPVCsExist(ctx, c, ss, []int{0})
1504 framework.ExpectNoError(err)
1505
1506 ginkgo.By("Confirming Pod is ready after being recreated")
1507 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 1)
1508 pod, err = c.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
1509 framework.ExpectNoError(err)
1510 gomega.Expect(pod.Spec.NodeName).To(gomega.Equal(readyNode.Name))
1511 })
1512 })
1513
1514 ginkgo.Describe("Scaling StatefulSetStartOrdinal", func() {
1515 ssName := "ss"
1516 labels := map[string]string{
1517 "foo": "bar",
1518 "baz": "blah",
1519 }
1520 headlessSvcName := "test"
1521 var ss *appsv1.StatefulSet
1522
1523 ginkgo.BeforeEach(func(ctx context.Context) {
1524 ss = e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 2, nil, nil, labels)
1525
1526 ginkgo.By("Creating service " + headlessSvcName + " in namespace " + ns)
1527 headlessService := e2eservice.CreateServiceSpec(headlessSvcName, "", true, labels)
1528 _, err := c.CoreV1().Services(ns).Create(ctx, headlessService, metav1.CreateOptions{})
1529 framework.ExpectNoError(err)
1530 })
1531
1532 ginkgo.AfterEach(func(ctx context.Context) {
1533 if ginkgo.CurrentSpecReport().Failed() {
1534 e2eoutput.DumpDebugInfo(ctx, c, ns)
1535 }
1536 framework.Logf("Deleting all statefulset in ns %v", ns)
1537 e2estatefulset.DeleteAllStatefulSets(ctx, c, ns)
1538 })
1539
1540 ginkgo.It("Setting .start.ordinal", func(ctx context.Context) {
1541 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1542 *(ss.Spec.Replicas) = 2
1543 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1544 framework.ExpectNoError(err)
1545 waitForStatus(ctx, c, ss)
1546 e2estatefulset.WaitForStatusReplicas(ctx, c, ss, 2)
1547 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 2)
1548
1549 ginkgo.By("Confirming 2 replicas, with start ordinal 0")
1550 pods := e2estatefulset.GetPodList(ctx, c, ss)
1551 err = expectPodNames(pods, []string{"ss-0", "ss-1"})
1552 framework.ExpectNoError(err)
1553
1554 ginkgo.By("Setting .spec.replicas = 3 .spec.ordinals.start = 2")
1555 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
1556 update.Spec.Ordinals = &appsv1.StatefulSetOrdinals{
1557 Start: 2,
1558 }
1559 *(update.Spec.Replicas) = 3
1560 })
1561 framework.ExpectNoError(err)
1562
1563
1564
1565 waitForStatus(ctx, c, ss)
1566 waitForPodNames(ctx, c, ss, []string{"ss-2", "ss-3", "ss-4"})
1567 ginkgo.By("Confirming 3 replicas, with start ordinal 2")
1568 e2estatefulset.WaitForStatusReplicas(ctx, c, ss, 3)
1569 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 3)
1570 })
1571
1572 ginkgo.It("Increasing .start.ordinal", func(ctx context.Context) {
1573 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1574 *(ss.Spec.Replicas) = 2
1575 ss.Spec.Ordinals = &appsv1.StatefulSetOrdinals{
1576 Start: 2,
1577 }
1578 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1579 framework.ExpectNoError(err)
1580 waitForStatus(ctx, c, ss)
1581 e2estatefulset.WaitForStatusReplicas(ctx, c, ss, 2)
1582 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 2)
1583
1584 ginkgo.By("Confirming 2 replicas, with start ordinal 2")
1585 pods := e2estatefulset.GetPodList(ctx, c, ss)
1586 err = expectPodNames(pods, []string{"ss-2", "ss-3"})
1587 framework.ExpectNoError(err)
1588
1589 ginkgo.By("Increasing .spec.ordinals.start = 4")
1590 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
1591 update.Spec.Ordinals = &appsv1.StatefulSetOrdinals{
1592 Start: 4,
1593 }
1594 })
1595 framework.ExpectNoError(err)
1596
1597
1598
1599 ginkgo.By("Confirming 2 replicas, with start ordinal 4")
1600 waitForStatus(ctx, c, ss)
1601 waitForPodNames(ctx, c, ss, []string{"ss-4", "ss-5"})
1602 e2estatefulset.WaitForStatusReplicas(ctx, c, ss, 2)
1603 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 2)
1604 })
1605
1606 ginkgo.It("Decreasing .start.ordinal", func(ctx context.Context) {
1607 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1608 *(ss.Spec.Replicas) = 2
1609 ss.Spec.Ordinals = &appsv1.StatefulSetOrdinals{
1610 Start: 3,
1611 }
1612 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1613 framework.ExpectNoError(err)
1614 waitForStatus(ctx, c, ss)
1615 e2estatefulset.WaitForStatusReplicas(ctx, c, ss, 2)
1616 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 2)
1617
1618 ginkgo.By("Confirming 2 replicas, with start ordinal 3")
1619 pods := e2estatefulset.GetPodList(ctx, c, ss)
1620 err = expectPodNames(pods, []string{"ss-3", "ss-4"})
1621 framework.ExpectNoError(err)
1622
1623 ginkgo.By("Decreasing .spec.ordinals.start = 2")
1624 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
1625 update.Spec.Ordinals = &appsv1.StatefulSetOrdinals{
1626 Start: 2,
1627 }
1628 })
1629 framework.ExpectNoError(err)
1630
1631
1632
1633 ginkgo.By("Confirming 2 replicas, with start ordinal 2")
1634 waitForStatus(ctx, c, ss)
1635 waitForPodNames(ctx, c, ss, []string{"ss-2", "ss-3"})
1636 e2estatefulset.WaitForStatusReplicas(ctx, c, ss, 2)
1637 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 2)
1638 })
1639
1640 ginkgo.It("Removing .start.ordinal", func(ctx context.Context) {
1641 ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
1642 *(ss.Spec.Replicas) = 2
1643 ss.Spec.Ordinals = &appsv1.StatefulSetOrdinals{
1644 Start: 3,
1645 }
1646 _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1647 framework.ExpectNoError(err)
1648 e2estatefulset.WaitForStatusReplicas(ctx, c, ss, 2)
1649 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 2)
1650
1651 ginkgo.By("Confirming 2 replicas, with start ordinal 3")
1652 pods := e2estatefulset.GetPodList(ctx, c, ss)
1653 err = expectPodNames(pods, []string{"ss-3", "ss-4"})
1654 framework.ExpectNoError(err)
1655
1656 ginkgo.By("Removing .spec.ordinals")
1657 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
1658 update.Spec.Ordinals = nil
1659 })
1660 framework.ExpectNoError(err)
1661
1662
1663
1664 framework.Logf("Confirming 2 replicas, with start ordinal 0")
1665 waitForStatus(ctx, c, ss)
1666 waitForPodNames(ctx, c, ss, []string{"ss-0", "ss-1"})
1667 e2estatefulset.WaitForStatusReplicas(ctx, c, ss, 2)
1668 e2estatefulset.WaitForStatusReadyReplicas(ctx, c, ss, 2)
1669 })
1670 })
1671 })
1672
1673 func uncordonNode(ctx context.Context, c clientset.Interface, oldData, newData []byte, nodeName string) {
1674 ginkgo.By("Uncordoning Node")
1675
1676 revertPatchBytes, err := strategicpatch.CreateTwoWayMergePatch(newData, oldData, v1.Node{})
1677 framework.ExpectNoError(err)
1678 _, err = c.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, revertPatchBytes, metav1.PatchOptions{})
1679 framework.ExpectNoError(err)
1680 }
1681
1682 func kubectlExecWithRetries(ns string, args ...string) (out string) {
1683 var err error
1684 for i := 0; i < 3; i++ {
1685 if out, err = e2ekubectl.RunKubectl(ns, args...); err == nil {
1686 return
1687 }
1688 framework.Logf("Retrying %v:\nerror %v\nstdout %v", args, err, out)
1689 }
1690 framework.Failf("Failed to execute \"%v\" with retries: %v", args, err)
1691 return
1692 }
1693
1694 type statefulPodTester interface {
1695 deploy(ctx context.Context, ns string) *appsv1.StatefulSet
1696 write(statefulPodIndex int, kv map[string]string)
1697 read(statefulPodIndex int, key string) string
1698 name() string
1699 }
1700
1701 type clusterAppTester struct {
1702 ns string
1703 statefulPod statefulPodTester
1704 client clientset.Interface
1705 }
1706
1707 func (c *clusterAppTester) run(ctx context.Context) {
1708 ginkgo.By("Deploying " + c.statefulPod.name())
1709 ss := c.statefulPod.deploy(ctx, c.ns)
1710
1711 ginkgo.By("Creating foo:bar in member with index 0")
1712 c.statefulPod.write(0, map[string]string{"foo": "bar"})
1713
1714 switch c.statefulPod.(type) {
1715 case *mysqlGaleraTester:
1716
1717 default:
1718 if restartCluster {
1719 ginkgo.By("Restarting stateful set " + ss.Name)
1720 e2estatefulset.Restart(ctx, c.client, ss)
1721 e2estatefulset.WaitForRunningAndReady(ctx, c.client, *ss.Spec.Replicas, ss)
1722 }
1723 }
1724
1725 ginkgo.By("Reading value under foo from member with index 2")
1726 if err := pollReadWithTimeout(ctx, c.statefulPod, 2, "foo", "bar"); err != nil {
1727 framework.Failf("%v", err)
1728 }
1729 }
1730
1731 type zookeeperTester struct {
1732 ss *appsv1.StatefulSet
1733 client clientset.Interface
1734 }
1735
1736 func (z *zookeeperTester) name() string {
1737 return "zookeeper"
1738 }
1739
1740 func (z *zookeeperTester) deploy(ctx context.Context, ns string) *appsv1.StatefulSet {
1741 z.ss = e2estatefulset.CreateStatefulSet(ctx, z.client, zookeeperManifestPath, ns)
1742 return z.ss
1743 }
1744
1745 func (z *zookeeperTester) write(statefulPodIndex int, kv map[string]string) {
1746 name := fmt.Sprintf("%v-%d", z.ss.Name, statefulPodIndex)
1747 for k, v := range kv {
1748 cmd := fmt.Sprintf("/opt/zookeeper/bin/zkCli.sh create /%v %v", k, v)
1749 framework.Logf(e2ekubectl.RunKubectlOrDie(z.ss.Namespace, "exec", name, "--", "/bin/sh", "-c", cmd))
1750 }
1751 }
1752
1753 func (z *zookeeperTester) read(statefulPodIndex int, key string) string {
1754 name := fmt.Sprintf("%v-%d", z.ss.Name, statefulPodIndex)
1755 cmd := fmt.Sprintf("/opt/zookeeper/bin/zkCli.sh get /%v", key)
1756 return lastLine(e2ekubectl.RunKubectlOrDie(z.ss.Namespace, "exec", name, "--", "/bin/sh", "-c", cmd))
1757 }
1758
1759 type mysqlGaleraTester struct {
1760 ss *appsv1.StatefulSet
1761 client clientset.Interface
1762 }
1763
1764 func (m *mysqlGaleraTester) name() string {
1765 return "mysql: galera"
1766 }
1767
1768 func (m *mysqlGaleraTester) mysqlExec(cmd, ns, podName string) string {
1769 cmd = fmt.Sprintf("/usr/bin/mysql -u root -B -e '%v'", cmd)
1770
1771
1772
1773 return kubectlExecWithRetries(ns, "exec", podName, "--", "/bin/sh", "-c", cmd)
1774 }
1775
1776 func (m *mysqlGaleraTester) deploy(ctx context.Context, ns string) *appsv1.StatefulSet {
1777 m.ss = e2estatefulset.CreateStatefulSet(ctx, m.client, mysqlGaleraManifestPath, ns)
1778
1779 framework.Logf("Deployed statefulset %v, initializing database", m.ss.Name)
1780 for _, cmd := range []string{
1781 "create database statefulset;",
1782 "use statefulset; create table foo (k varchar(20), v varchar(20));",
1783 } {
1784 framework.Logf(m.mysqlExec(cmd, ns, fmt.Sprintf("%v-0", m.ss.Name)))
1785 }
1786 return m.ss
1787 }
1788
1789 func (m *mysqlGaleraTester) write(statefulPodIndex int, kv map[string]string) {
1790 name := fmt.Sprintf("%v-%d", m.ss.Name, statefulPodIndex)
1791 for k, v := range kv {
1792 cmd := fmt.Sprintf("use statefulset; insert into foo (k, v) values (\"%v\", \"%v\");", k, v)
1793 framework.Logf(m.mysqlExec(cmd, m.ss.Namespace, name))
1794 }
1795 }
1796
1797 func (m *mysqlGaleraTester) read(statefulPodIndex int, key string) string {
1798 name := fmt.Sprintf("%v-%d", m.ss.Name, statefulPodIndex)
1799 return lastLine(m.mysqlExec(fmt.Sprintf("use statefulset; select v from foo where k=\"%v\";", key), m.ss.Namespace, name))
1800 }
1801
1802 type redisTester struct {
1803 ss *appsv1.StatefulSet
1804 client clientset.Interface
1805 }
1806
1807 func (m *redisTester) name() string {
1808 return "redis: master/slave"
1809 }
1810
1811 func (m *redisTester) redisExec(cmd, ns, podName string) string {
1812 cmd = fmt.Sprintf("/opt/redis/redis-cli -h %v %v", podName, cmd)
1813 return e2ekubectl.RunKubectlOrDie(ns, "exec", podName, "--", "/bin/sh", "-c", cmd)
1814 }
1815
1816 func (m *redisTester) deploy(ctx context.Context, ns string) *appsv1.StatefulSet {
1817 m.ss = e2estatefulset.CreateStatefulSet(ctx, m.client, redisManifestPath, ns)
1818 return m.ss
1819 }
1820
1821 func (m *redisTester) write(statefulPodIndex int, kv map[string]string) {
1822 name := fmt.Sprintf("%v-%d", m.ss.Name, statefulPodIndex)
1823 for k, v := range kv {
1824 framework.Logf(m.redisExec(fmt.Sprintf("SET %v %v", k, v), m.ss.Namespace, name))
1825 }
1826 }
1827
1828 func (m *redisTester) read(statefulPodIndex int, key string) string {
1829 name := fmt.Sprintf("%v-%d", m.ss.Name, statefulPodIndex)
1830 return lastLine(m.redisExec(fmt.Sprintf("GET %v", key), m.ss.Namespace, name))
1831 }
1832
1833 type cockroachDBTester struct {
1834 ss *appsv1.StatefulSet
1835 client clientset.Interface
1836 }
1837
1838 func (c *cockroachDBTester) name() string {
1839 return "CockroachDB"
1840 }
1841
1842 func (c *cockroachDBTester) cockroachDBExec(cmd, ns, podName string) string {
1843 cmd = fmt.Sprintf("/cockroach/cockroach sql --insecure --host %s.cockroachdb -e \"%v\"", podName, cmd)
1844 return e2ekubectl.RunKubectlOrDie(ns, "exec", podName, "--", "/bin/sh", "-c", cmd)
1845 }
1846
1847 func (c *cockroachDBTester) deploy(ctx context.Context, ns string) *appsv1.StatefulSet {
1848 c.ss = e2estatefulset.CreateStatefulSet(ctx, c.client, cockroachDBManifestPath, ns)
1849 framework.Logf("Deployed statefulset %v, initializing database", c.ss.Name)
1850 for _, cmd := range []string{
1851 "CREATE DATABASE IF NOT EXISTS foo;",
1852 "CREATE TABLE IF NOT EXISTS foo.bar (k STRING PRIMARY KEY, v STRING);",
1853 } {
1854 framework.Logf(c.cockroachDBExec(cmd, ns, fmt.Sprintf("%v-0", c.ss.Name)))
1855 }
1856 return c.ss
1857 }
1858
1859 func (c *cockroachDBTester) write(statefulPodIndex int, kv map[string]string) {
1860 name := fmt.Sprintf("%v-%d", c.ss.Name, statefulPodIndex)
1861 for k, v := range kv {
1862 cmd := fmt.Sprintf("UPSERT INTO foo.bar VALUES ('%v', '%v');", k, v)
1863 framework.Logf(c.cockroachDBExec(cmd, c.ss.Namespace, name))
1864 }
1865 }
1866 func (c *cockroachDBTester) read(statefulPodIndex int, key string) string {
1867 name := fmt.Sprintf("%v-%d", c.ss.Name, statefulPodIndex)
1868 return lastLine(c.cockroachDBExec(fmt.Sprintf("SELECT v FROM foo.bar WHERE k='%v';", key), c.ss.Namespace, name))
1869 }
1870
1871 func lastLine(out string) string {
1872 outLines := strings.Split(strings.Trim(out, "\n"), "\n")
1873 return outLines[len(outLines)-1]
1874 }
1875
1876 func pollReadWithTimeout(ctx context.Context, statefulPod statefulPodTester, statefulPodNumber int, key, expectedVal string) error {
1877 err := wait.PollUntilContextTimeout(ctx, time.Second, readTimeout, true, func(ctx context.Context) (bool, error) {
1878 val := statefulPod.read(statefulPodNumber, key)
1879 if val == "" {
1880 return false, nil
1881 } else if val != expectedVal {
1882 return false, fmt.Errorf("expected value %v, found %v", expectedVal, val)
1883 }
1884 return true, nil
1885 })
1886
1887 if wait.Interrupted(err) {
1888 return fmt.Errorf("timed out when trying to read value for key %v from stateful pod %d", key, statefulPodNumber)
1889 }
1890 return err
1891 }
1892
1893
1894
1895 func rollbackTest(ctx context.Context, c clientset.Interface, ns string, ss *appsv1.StatefulSet) {
1896 setHTTPProbe(ss)
1897 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
1898 framework.ExpectNoError(err)
1899 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
1900 ss = waitForStatus(ctx, c, ss)
1901 currentRevision, updateRevision := ss.Status.CurrentRevision, ss.Status.UpdateRevision
1902 gomega.Expect(currentRevision).To(gomega.Equal(updateRevision), "StatefulSet %s/%s created with update revision %s not equal to current revision %s",
1903 ss.Namespace, ss.Name, updateRevision, currentRevision)
1904 pods := e2estatefulset.GetPodList(ctx, c, ss)
1905 for i := range pods.Items {
1906 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, currentRevision), "Pod %s/%s revision %s is not equal to current revision %s",
1907 pods.Items[i].Namespace,
1908 pods.Items[i].Name,
1909 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
1910 currentRevision)
1911 }
1912 e2estatefulset.SortStatefulPods(pods)
1913 err = breakPodHTTPProbe(ss, &pods.Items[1])
1914 framework.ExpectNoError(err)
1915 ss, _ = waitForPodNotReady(ctx, c, ss, pods.Items[1].Name)
1916 newImage := NewWebserverImage
1917 oldImage := ss.Spec.Template.Spec.Containers[0].Image
1918
1919 ginkgo.By(fmt.Sprintf("Updating StatefulSet template: update image from %s to %s", oldImage, newImage))
1920 gomega.Expect(oldImage).NotTo(gomega.Equal(newImage), "Incorrect test setup: should update to a different image")
1921 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
1922 update.Spec.Template.Spec.Containers[0].Image = newImage
1923 })
1924 framework.ExpectNoError(err)
1925
1926 ginkgo.By("Creating a new revision")
1927 ss = waitForStatus(ctx, c, ss)
1928 currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision
1929 gomega.Expect(currentRevision).NotTo(gomega.Equal(updateRevision), "Current revision should not equal update revision during rolling update")
1930
1931 ginkgo.By("Updating Pods in reverse ordinal order")
1932 pods = e2estatefulset.GetPodList(ctx, c, ss)
1933 e2estatefulset.SortStatefulPods(pods)
1934 err = restorePodHTTPProbe(ss, &pods.Items[1])
1935 framework.ExpectNoError(err)
1936 ss, _ = e2estatefulset.WaitForPodReady(ctx, c, ss, pods.Items[1].Name)
1937 ss, pods = waitForRollingUpdate(ctx, c, ss)
1938 gomega.Expect(ss.Status.CurrentRevision).To(gomega.Equal(updateRevision), "StatefulSet %s/%s current revision %s does not equal update revision %s on update completion",
1939 ss.Namespace,
1940 ss.Name,
1941 ss.Status.CurrentRevision,
1942 updateRevision)
1943 for i := range pods.Items {
1944 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), "Pod %s/%s has image %s not have new image %s",
1945 pods.Items[i].Namespace,
1946 pods.Items[i].Name,
1947 pods.Items[i].Spec.Containers[0].Image,
1948 newImage)
1949 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, updateRevision), "Pod %s/%s revision %s is not equal to update revision %s",
1950 pods.Items[i].Namespace,
1951 pods.Items[i].Name,
1952 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
1953 updateRevision)
1954 }
1955
1956 ginkgo.By("Rolling back to a previous revision")
1957 err = breakPodHTTPProbe(ss, &pods.Items[1])
1958 framework.ExpectNoError(err)
1959 ss, _ = waitForPodNotReady(ctx, c, ss, pods.Items[1].Name)
1960 priorRevision := currentRevision
1961 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
1962 update.Spec.Template.Spec.Containers[0].Image = oldImage
1963 })
1964 framework.ExpectNoError(err)
1965 ss = waitForStatus(ctx, c, ss)
1966 currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision
1967 gomega.Expect(priorRevision).To(gomega.Equal(updateRevision), "Prior revision should equal update revision during roll back")
1968 gomega.Expect(currentRevision).NotTo(gomega.Equal(updateRevision), "Current revision should not equal update revision during roll back")
1969
1970 ginkgo.By("Rolling back update in reverse ordinal order")
1971 pods = e2estatefulset.GetPodList(ctx, c, ss)
1972 e2estatefulset.SortStatefulPods(pods)
1973 restorePodHTTPProbe(ss, &pods.Items[1])
1974 ss, _ = e2estatefulset.WaitForPodReady(ctx, c, ss, pods.Items[1].Name)
1975 ss, pods = waitForRollingUpdate(ctx, c, ss)
1976 gomega.Expect(ss.Status.CurrentRevision).To(gomega.Equal(priorRevision), "StatefulSet %s/%s current revision %s does not equal prior revision %s on rollback completion",
1977 ss.Namespace,
1978 ss.Name,
1979 ss.Status.CurrentRevision,
1980 updateRevision)
1981
1982 for i := range pods.Items {
1983 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), "Pod %s/%s has image %s not equal to previous image %s",
1984 pods.Items[i].Namespace,
1985 pods.Items[i].Name,
1986 pods.Items[i].Spec.Containers[0].Image,
1987 oldImage)
1988 gomega.Expect(pods.Items[i].Labels).To(gomega.HaveKeyWithValue(appsv1.StatefulSetRevisionLabel, priorRevision), "Pod %s/%s revision %s is not equal to prior revision %s",
1989 pods.Items[i].Namespace,
1990 pods.Items[i].Name,
1991 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
1992 priorRevision)
1993 }
1994 }
1995
1996
1997 func deletingPodForRollingUpdatePartitionTest(ctx context.Context, f *framework.Framework, c clientset.Interface, ns string, ss *appsv1.StatefulSet) {
1998 setHTTPProbe(ss)
1999 ss.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{
2000 Type: appsv1.RollingUpdateStatefulSetStrategyType,
2001 RollingUpdate: func() *appsv1.RollingUpdateStatefulSetStrategy {
2002 return &appsv1.RollingUpdateStatefulSetStrategy{
2003 Partition: pointer.Int32(1),
2004 }
2005 }(),
2006 }
2007 ss, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
2008 framework.ExpectNoError(err)
2009 e2estatefulset.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
2010 ss = waitForStatus(ctx, c, ss)
2011 currentRevision, updateRevision := ss.Status.CurrentRevision, ss.Status.UpdateRevision
2012 gomega.Expect(currentRevision).To(gomega.Equal(updateRevision), fmt.Sprintf("StatefulSet %s/%s created with update revision %s not equal to current revision %s",
2013 ss.Namespace, ss.Name, updateRevision, currentRevision))
2014 pods := e2estatefulset.GetPodList(ctx, c, ss)
2015 for i := range pods.Items {
2016 gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(currentRevision), fmt.Sprintf("Pod %s/%s revision %s is not equal to currentRevision %s",
2017 pods.Items[i].Namespace,
2018 pods.Items[i].Name,
2019 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
2020 currentRevision))
2021 }
2022
2023 ginkgo.By("Adding finalizer for pod-0")
2024 pod0name := getStatefulSetPodNameAtIndex(0, ss)
2025 pod0, err := c.CoreV1().Pods(ns).Get(ctx, pod0name, metav1.GetOptions{})
2026 framework.ExpectNoError(err)
2027 pod0.Finalizers = append(pod0.Finalizers, testFinalizer)
2028 pod0, err = c.CoreV1().Pods(ss.Namespace).Update(ctx, pod0, metav1.UpdateOptions{})
2029 framework.ExpectNoError(err)
2030 pods.Items[0] = *pod0
2031 defer e2epod.NewPodClient(f).RemoveFinalizer(ctx, pod0.Name, testFinalizer)
2032
2033 ginkgo.By("Updating image on StatefulSet")
2034 newImage := NewWebserverImage
2035 oldImage := ss.Spec.Template.Spec.Containers[0].Image
2036 ginkgo.By(fmt.Sprintf("Updating stateful set template: update image from %s to %s", oldImage, newImage))
2037 gomega.Expect(oldImage).ToNot(gomega.Equal(newImage), "Incorrect test setup: should update to a different image")
2038 ss, err = updateStatefulSetWithRetries(ctx, c, ns, ss.Name, func(update *appsv1.StatefulSet) {
2039 update.Spec.Template.Spec.Containers[0].Image = newImage
2040 })
2041 framework.ExpectNoError(err)
2042
2043 ginkgo.By("Creating a new revision")
2044 ss = waitForStatus(ctx, c, ss)
2045 currentRevision, updateRevision = ss.Status.CurrentRevision, ss.Status.UpdateRevision
2046 gomega.Expect(currentRevision).ToNot(gomega.Equal(updateRevision), "Current revision should not equal update revision during rolling update")
2047
2048 ginkgo.By("Await for all replicas running, all are updated but pod-0")
2049 e2estatefulset.WaitForState(ctx, c, ss, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) {
2050 ss = set2
2051 pods = pods2
2052 if ss.Status.UpdatedReplicas == *ss.Spec.Replicas-1 && ss.Status.Replicas == *ss.Spec.Replicas && ss.Status.ReadyReplicas == *ss.Spec.Replicas {
2053
2054 return true, nil
2055 }
2056 return false, nil
2057 })
2058
2059 ginkgo.By("Verify pod images before pod-0 deletion and recreation")
2060 for i := range pods.Items {
2061 if i < int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) {
2062 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), fmt.Sprintf("Pod %s/%s has image %s not equal to oldimage image %s",
2063 pods.Items[i].Namespace,
2064 pods.Items[i].Name,
2065 pods.Items[i].Spec.Containers[0].Image,
2066 oldImage))
2067 gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(currentRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to current revision %s",
2068 pods.Items[i].Namespace,
2069 pods.Items[i].Name,
2070 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
2071 currentRevision))
2072 } else {
2073 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), fmt.Sprintf("Pod %s/%s has image %s not equal to new image %s",
2074 pods.Items[i].Namespace,
2075 pods.Items[i].Name,
2076 pods.Items[i].Spec.Containers[0].Image,
2077 newImage))
2078 gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(updateRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to new revision %s",
2079 pods.Items[i].Namespace,
2080 pods.Items[i].Name,
2081 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
2082 updateRevision))
2083 }
2084 }
2085
2086 ginkgo.By("Deleting the pod-0 so that kubelet terminates it and StatefulSet controller recreates it")
2087 deleteStatefulPodAtIndex(ctx, c, 0, ss)
2088 ginkgo.By("Await for two replicas to be updated, while the pod-0 is not running")
2089 e2estatefulset.WaitForState(ctx, c, ss, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) {
2090 ss = set2
2091 pods = pods2
2092 return ss.Status.ReadyReplicas == *ss.Spec.Replicas-1, nil
2093 })
2094
2095 ginkgo.By(fmt.Sprintf("Removing finalizer from pod-0 (%v/%v) to allow recreation", pod0.Namespace, pod0.Name))
2096 e2epod.NewPodClient(f).RemoveFinalizer(ctx, pod0.Name, testFinalizer)
2097
2098 ginkgo.By("Await for recreation of pod-0, so that all replicas are running")
2099 e2estatefulset.WaitForState(ctx, c, ss, func(set2 *appsv1.StatefulSet, pods2 *v1.PodList) (bool, error) {
2100 ss = set2
2101 pods = pods2
2102 return ss.Status.ReadyReplicas == *ss.Spec.Replicas, nil
2103 })
2104
2105 ginkgo.By("Verify pod images after pod-0 deletion and recreation")
2106 pods = e2estatefulset.GetPodList(ctx, c, ss)
2107 for i := range pods.Items {
2108 if i < int(*ss.Spec.UpdateStrategy.RollingUpdate.Partition) {
2109 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(oldImage), fmt.Sprintf("Pod %s/%s has image %s not equal to current image %s",
2110 pods.Items[i].Namespace,
2111 pods.Items[i].Name,
2112 pods.Items[i].Spec.Containers[0].Image,
2113 oldImage))
2114 gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(currentRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to current revision %s",
2115 pods.Items[i].Namespace,
2116 pods.Items[i].Name,
2117 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
2118 currentRevision))
2119 } else {
2120 gomega.Expect(pods.Items[i].Spec.Containers[0].Image).To(gomega.Equal(newImage), fmt.Sprintf("Pod %s/%s has image %s not equal to new image %s",
2121 pods.Items[i].Namespace,
2122 pods.Items[i].Name,
2123 pods.Items[i].Spec.Containers[0].Image,
2124 newImage))
2125 gomega.Expect(pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel]).To(gomega.Equal(updateRevision), fmt.Sprintf("Pod %s/%s has revision %s not equal to new revision %s",
2126 pods.Items[i].Namespace,
2127 pods.Items[i].Name,
2128 pods.Items[i].Labels[appsv1.StatefulSetRevisionLabel],
2129 updateRevision))
2130 }
2131 }
2132 }
2133
2134
2135
2136 func confirmStatefulPodCount(ctx context.Context, c clientset.Interface, count int, ss *appsv1.StatefulSet, timeout time.Duration, hard bool) {
2137 start := time.Now()
2138 deadline := start.Add(timeout)
2139 for t := time.Now(); t.Before(deadline) && ctx.Err() == nil; t = time.Now() {
2140 podList := e2estatefulset.GetPodList(ctx, c, ss)
2141 statefulPodCount := len(podList.Items)
2142 if statefulPodCount != count {
2143 e2epod.LogPodStates(podList.Items)
2144 if hard {
2145 framework.Failf("StatefulSet %v scaled unexpectedly scaled to %d -> %d replicas", ss.Name, count, len(podList.Items))
2146 } else {
2147 framework.Logf("StatefulSet %v has not reached scale %d, at %d", ss.Name, count, statefulPodCount)
2148 }
2149 time.Sleep(1 * time.Second)
2150 continue
2151 }
2152 framework.Logf("Verifying statefulset %v doesn't scale past %d for another %+v", ss.Name, count, deadline.Sub(t))
2153 time.Sleep(1 * time.Second)
2154 }
2155 }
2156
2157
2158
2159
2160 func setHTTPProbe(ss *appsv1.StatefulSet) {
2161 ss.Spec.Template.Spec.Containers[0].ReadinessProbe = httpProbe
2162 }
2163
2164
2165 func breakHTTPProbe(ctx context.Context, c clientset.Interface, ss *appsv1.StatefulSet) error {
2166 path := httpProbe.HTTPGet.Path
2167 if path == "" {
2168 return fmt.Errorf("path expected to be not empty: %v", path)
2169 }
2170
2171 cmd := fmt.Sprintf("mv -v /usr/local/apache2/htdocs%v /tmp/ || true", path)
2172 return e2estatefulset.ExecInStatefulPods(ctx, c, ss, cmd)
2173 }
2174
2175
2176 func breakPodHTTPProbe(ss *appsv1.StatefulSet, pod *v1.Pod) error {
2177 path := httpProbe.HTTPGet.Path
2178 if path == "" {
2179 return fmt.Errorf("path expected to be not empty: %v", path)
2180 }
2181
2182 cmd := fmt.Sprintf("mv -v /usr/local/apache2/htdocs%v /tmp/ || true", path)
2183 stdout, err := e2eoutput.RunHostCmdWithRetries(pod.Namespace, pod.Name, cmd, statefulSetPoll, statefulPodTimeout)
2184 framework.Logf("stdout of %v on %v: %v", cmd, pod.Name, stdout)
2185 return err
2186 }
2187
2188
2189 func restoreHTTPProbe(ctx context.Context, c clientset.Interface, ss *appsv1.StatefulSet) error {
2190 path := httpProbe.HTTPGet.Path
2191 if path == "" {
2192 return fmt.Errorf("path expected to be not empty: %v", path)
2193 }
2194
2195 cmd := fmt.Sprintf("mv -v /tmp%v /usr/local/apache2/htdocs/ || true", path)
2196 return e2estatefulset.ExecInStatefulPods(ctx, c, ss, cmd)
2197 }
2198
2199
2200 func restorePodHTTPProbe(ss *appsv1.StatefulSet, pod *v1.Pod) error {
2201 path := httpProbe.HTTPGet.Path
2202 if path == "" {
2203 return fmt.Errorf("path expected to be not empty: %v", path)
2204 }
2205
2206 cmd := fmt.Sprintf("mv -v /tmp%v /usr/local/apache2/htdocs/ || true", path)
2207 stdout, err := e2eoutput.RunHostCmdWithRetries(pod.Namespace, pod.Name, cmd, statefulSetPoll, statefulPodTimeout)
2208 framework.Logf("stdout of %v on %v: %v", cmd, pod.Name, stdout)
2209 return err
2210 }
2211
2212
2213 func deleteStatefulPodAtIndex(ctx context.Context, c clientset.Interface, index int, ss *appsv1.StatefulSet) {
2214 name := getStatefulSetPodNameAtIndex(index, ss)
2215 noGrace := int64(0)
2216 if err := c.CoreV1().Pods(ss.Namespace).Delete(ctx, name, metav1.DeleteOptions{GracePeriodSeconds: &noGrace}); err != nil {
2217 framework.Failf("Failed to delete stateful pod %v for StatefulSet %v/%v: %v", name, ss.Namespace, ss.Name, err)
2218 }
2219 }
2220
2221
2222 func getStatefulSetPodNameAtIndex(index int, ss *appsv1.StatefulSet) string {
2223
2224
2225 return fmt.Sprintf("%v-%v", ss.Name, index)
2226 }
2227
2228 type updateStatefulSetFunc func(*appsv1.StatefulSet)
2229
2230
2231 func updateStatefulSetWithRetries(ctx context.Context, c clientset.Interface, namespace, name string, applyUpdate updateStatefulSetFunc) (statefulSet *appsv1.StatefulSet, err error) {
2232 statefulSets := c.AppsV1().StatefulSets(namespace)
2233 var updateErr error
2234 pollErr := wait.PollWithContext(ctx, 10*time.Millisecond, 1*time.Minute, func(ctx context.Context) (bool, error) {
2235 if statefulSet, err = statefulSets.Get(ctx, name, metav1.GetOptions{}); err != nil {
2236 return false, err
2237 }
2238
2239 applyUpdate(statefulSet)
2240 if statefulSet, err = statefulSets.Update(ctx, statefulSet, metav1.UpdateOptions{}); err == nil {
2241 framework.Logf("Updating stateful set %s", name)
2242 return true, nil
2243 }
2244 updateErr = err
2245 return false, nil
2246 })
2247 if wait.Interrupted(pollErr) {
2248 pollErr = fmt.Errorf("couldn't apply the provided updated to stateful set %q: %v", name, updateErr)
2249 }
2250 return statefulSet, pollErr
2251 }
2252
2253
2254 func getStatefulSet(ctx context.Context, c clientset.Interface, namespace, name string) *appsv1.StatefulSet {
2255 ss, err := c.AppsV1().StatefulSets(namespace).Get(ctx, name, metav1.GetOptions{})
2256 if err != nil {
2257 framework.Failf("Failed to get StatefulSet %s/%s: %v", namespace, name, err)
2258 }
2259 return ss
2260 }
2261
2262
2263 func verifyStatefulSetPVCsExist(ctx context.Context, c clientset.Interface, ss *appsv1.StatefulSet, claimIds []int) error {
2264 idSet := map[int]struct{}{}
2265 for _, id := range claimIds {
2266 idSet[id] = struct{}{}
2267 }
2268 return wait.PollImmediate(e2estatefulset.StatefulSetPoll, e2estatefulset.StatefulSetTimeout, func() (bool, error) {
2269 pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(ctx, metav1.ListOptions{LabelSelector: klabels.Everything().String()})
2270 if err != nil {
2271 framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err)
2272 return false, nil
2273 }
2274 for _, claim := range ss.Spec.VolumeClaimTemplates {
2275 pvcNameRE := regexp.MustCompile(fmt.Sprintf("^%s-%s-([0-9]+)$", claim.Name, ss.Name))
2276 seenPVCs := map[int]struct{}{}
2277 for _, pvc := range pvcList.Items {
2278 matches := pvcNameRE.FindStringSubmatch(pvc.Name)
2279 if len(matches) != 2 {
2280 continue
2281 }
2282 ordinal, err := strconv.ParseInt(matches[1], 10, 32)
2283 if err != nil {
2284 framework.Logf("ERROR: bad pvc name %s (%v)", pvc.Name, err)
2285 return false, err
2286 }
2287 if _, found := idSet[int(ordinal)]; !found {
2288 return false, nil
2289 } else {
2290 seenPVCs[int(ordinal)] = struct{}{}
2291 }
2292 }
2293 if len(seenPVCs) != len(idSet) {
2294 framework.Logf("Found %d of %d PVCs", len(seenPVCs), len(idSet))
2295 return false, nil
2296 }
2297 }
2298 return true, nil
2299 })
2300 }
2301
2302
2303 func verifyStatefulSetPVCsExistWithOwnerRefs(ctx context.Context, c clientset.Interface, ss *appsv1.StatefulSet, claimIndicies []int, wantSetRef, wantPodRef bool) error {
2304 indexSet := map[int]struct{}{}
2305 for _, id := range claimIndicies {
2306 indexSet[id] = struct{}{}
2307 }
2308 set := getStatefulSet(ctx, c, ss.Namespace, ss.Name)
2309 setUID := set.GetUID()
2310 if setUID == "" {
2311 framework.Failf("Statefulset %s missing UID", ss.Name)
2312 }
2313 return wait.PollImmediate(e2estatefulset.StatefulSetPoll, e2estatefulset.StatefulSetTimeout, func() (bool, error) {
2314 pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(ctx, metav1.ListOptions{LabelSelector: klabels.Everything().String()})
2315 if err != nil {
2316 framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err)
2317 return false, nil
2318 }
2319 for _, claim := range ss.Spec.VolumeClaimTemplates {
2320 pvcNameRE := regexp.MustCompile(fmt.Sprintf("^%s-%s-([0-9]+)$", claim.Name, ss.Name))
2321 seenPVCs := map[int]struct{}{}
2322 for _, pvc := range pvcList.Items {
2323 matches := pvcNameRE.FindStringSubmatch(pvc.Name)
2324 if len(matches) != 2 {
2325 continue
2326 }
2327 ordinal, err := strconv.ParseInt(matches[1], 10, 32)
2328 if err != nil {
2329 framework.Logf("ERROR: bad pvc name %s (%v)", pvc.Name, err)
2330 return false, err
2331 }
2332 if _, found := indexSet[int(ordinal)]; !found {
2333 framework.Logf("Unexpected, retrying")
2334 return false, nil
2335 }
2336 var foundSetRef, foundPodRef bool
2337 for _, ref := range pvc.GetOwnerReferences() {
2338 if ref.Kind == "StatefulSet" && ref.UID == setUID {
2339 foundSetRef = true
2340 }
2341 if ref.Kind == "Pod" {
2342 podName := fmt.Sprintf("%s-%d", ss.Name, ordinal)
2343 pod, err := c.CoreV1().Pods(ss.Namespace).Get(ctx, podName, metav1.GetOptions{})
2344 if err != nil {
2345 framework.Logf("Pod %s not found, retrying (%v)", podName, err)
2346 return false, nil
2347 }
2348 podUID := pod.GetUID()
2349 if podUID == "" {
2350 framework.Failf("Pod %s is missing UID", pod.Name)
2351 }
2352 if ref.UID == podUID {
2353 foundPodRef = true
2354 }
2355 }
2356 }
2357 if foundSetRef == wantSetRef && foundPodRef == wantPodRef {
2358 seenPVCs[int(ordinal)] = struct{}{}
2359 }
2360 }
2361 if len(seenPVCs) != len(indexSet) {
2362 framework.Logf("Only %d PVCs, retrying", len(seenPVCs))
2363 return false, nil
2364 }
2365 }
2366 return true, nil
2367 })
2368 }
2369
2370
2371
2372
2373 func expectPodNames(actualPods *v1.PodList, expectedPodNames []string) error {
2374 e2estatefulset.SortStatefulPods(actualPods)
2375 pods := []string{}
2376 for _, pod := range actualPods.Items {
2377
2378
2379 if e2epod.IsPodActive(&pod) {
2380 pods = append(pods, pod.Name)
2381 }
2382 }
2383 if !reflect.DeepEqual(expectedPodNames, pods) {
2384 diff := cmp.Diff(expectedPodNames, pods)
2385 return fmt.Errorf("pod names don't match, diff (- for expected, + for actual):\n%s", diff)
2386 }
2387 return nil
2388 }
2389
View as plain text