Source file src/k8s.io/kubernetes/test/e2e_node/container_lifecycle_test.go

Documentation: k8s.io/kubernetes/test/e2e_node

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package e2enode
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	"github.com/onsi/ginkgo/v2"
    25  	"github.com/onsi/gomega"
    26  	v1 "k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
    29  	admissionapi "k8s.io/pod-security-admission/api"
    30  
    31  	"k8s.io/kubernetes/test/e2e/framework"
    32  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    33  	"k8s.io/kubernetes/test/e2e/nodefeature"
    34  	imageutils "k8s.io/kubernetes/test/utils/image"
    35  )
    36  
    37  const (
    38  	PostStartPrefix = "PostStart"
    39  	PreStopPrefix   = "PreStop"
    40  )
    41  
    42  var containerRestartPolicyAlways = v1.ContainerRestartPolicyAlways
    43  
    44  func prefixedName(namePrefix string, name string) string {
    45  	return fmt.Sprintf("%s-%s", namePrefix, name)
    46  }
    47  
    48  var _ = SIGDescribe(framework.WithNodeConformance(), "Containers Lifecycle", func() {
    49  	f := framework.NewDefaultFramework("containers-lifecycle-test")
    50  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    51  
    52  	ginkgo.It("should launch init container serially before a regular container", func() {
    53  
    54  		init1 := "init-1"
    55  		init2 := "init-2"
    56  		init3 := "init-3"
    57  		regular1 := "regular-1"
    58  
    59  		podSpec := &v1.Pod{
    60  			ObjectMeta: metav1.ObjectMeta{
    61  				Name: "initcontainer-test-pod",
    62  			},
    63  			Spec: v1.PodSpec{
    64  				RestartPolicy: v1.RestartPolicyNever,
    65  				InitContainers: []v1.Container{
    66  					{
    67  						Name:  init1,
    68  						Image: busyboxImage,
    69  						Command: ExecCommand(init1, execCommand{
    70  							Delay:    1,
    71  							ExitCode: 0,
    72  						}),
    73  					},
    74  					{
    75  						Name:  init2,
    76  						Image: busyboxImage,
    77  						Command: ExecCommand(init2, execCommand{
    78  							Delay:    1,
    79  							ExitCode: 0,
    80  						}),
    81  					},
    82  					{
    83  						Name:  init3,
    84  						Image: busyboxImage,
    85  						Command: ExecCommand(init3, execCommand{
    86  							Delay:    1,
    87  							ExitCode: 0,
    88  						}),
    89  					},
    90  				},
    91  				Containers: []v1.Container{
    92  					{
    93  						Name:  regular1,
    94  						Image: busyboxImage,
    95  						Command: ExecCommand(regular1, execCommand{
    96  							StartDelay: 5,
    97  							Delay:      1,
    98  							ExitCode:   0,
    99  						}),
   100  						StartupProbe: &v1.Probe{
   101  							ProbeHandler: v1.ProbeHandler{
   102  								Exec: &v1.ExecAction{
   103  									Command: []string{
   104  										"test",
   105  										"-f",
   106  										"started",
   107  									},
   108  								},
   109  							},
   110  						},
   111  					},
   112  				},
   113  			},
   114  		}
   115  
   116  		preparePod(podSpec)
   117  
   118  		/// generates an out file output like:
   119  		//
   120  		// 1682076093 4905.79 init-1 Starting 0
   121  		// 1682076093 4905.80 init-1 Started
   122  		// 1682076093 4905.80 init-1 Delaying 1
   123  		// 1682076094 4906.80 init-1 Exiting
   124  		// 1682076095 4907.70 init-2 Starting 0
   125  		// 1682076095 4907.71 init-2 Started
   126  		// 1682076095 4907.71 init-2 Delaying 1
   127  		// 1682076096 4908.71 init-2 Exiting
   128  		// 1682076097 4909.74 init-3 Starting 0
   129  		// 1682076097 4909.74 init-3 Started
   130  		// 1682076097 4909.74 init-3 Delaying 1
   131  		// 1682076098 4910.75 init-3 Exiting
   132  		// 1682076099 4911.70 regular-1 Starting 5
   133  		// 1682076104 4916.71 regular-1 Started
   134  		// 1682076104 4916.71 regular-1 Delaying 1
   135  		// 1682076105 4917.72 regular-1 Exiting
   136  
   137  		client := e2epod.NewPodClient(f)
   138  		podSpec = client.Create(context.TODO(), podSpec)
   139  		ginkgo.By("Waiting for the pod to finish")
   140  		err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 1*time.Minute)
   141  		framework.ExpectNoError(err)
   142  
   143  		ginkgo.By("Parsing results")
   144  		podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
   145  		framework.ExpectNoError(err)
   146  		results := parseOutput(context.TODO(), f, podSpec)
   147  
   148  		// which we then use to make assertions regarding container ordering
   149  		ginkgo.By("Analyzing results")
   150  		framework.ExpectNoError(results.StartsBefore(init1, init2))
   151  		framework.ExpectNoError(results.ExitsBefore(init1, init2))
   152  
   153  		framework.ExpectNoError(results.StartsBefore(init2, init3))
   154  		framework.ExpectNoError(results.ExitsBefore(init2, init3))
   155  
   156  		framework.ExpectNoError(results.StartsBefore(init3, regular1))
   157  		framework.ExpectNoError(results.ExitsBefore(init3, regular1))
   158  	})
   159  
   160  	ginkgo.It("should not launch regular containers if an init container fails", func() {
   161  
   162  		init1 := "init-1"
   163  		regular1 := "regular-1"
   164  
   165  		podSpec := &v1.Pod{
   166  			ObjectMeta: metav1.ObjectMeta{
   167  				Name: "initcontainer-test-pod-failure",
   168  			},
   169  			Spec: v1.PodSpec{
   170  				RestartPolicy: v1.RestartPolicyNever,
   171  				InitContainers: []v1.Container{
   172  					{
   173  						Name:  init1,
   174  						Image: busyboxImage,
   175  						Command: ExecCommand(init1, execCommand{
   176  							Delay:    1,
   177  							ExitCode: 1,
   178  						}),
   179  					},
   180  				},
   181  				Containers: []v1.Container{
   182  					{
   183  						Name:  regular1,
   184  						Image: busyboxImage,
   185  						Command: ExecCommand(regular1, execCommand{
   186  							Delay:    1,
   187  							ExitCode: 0,
   188  						}),
   189  					},
   190  				},
   191  			},
   192  		}
   193  
   194  		preparePod(podSpec)
   195  
   196  		client := e2epod.NewPodClient(f)
   197  		podSpec = client.Create(context.TODO(), podSpec)
   198  		ginkgo.By("Waiting for the pod to fail")
   199  		err := e2epod.WaitForPodFailedReason(context.TODO(), f.ClientSet, podSpec, "", 1*time.Minute)
   200  		framework.ExpectNoError(err)
   201  
   202  		ginkgo.By("Parsing results")
   203  		podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
   204  		framework.ExpectNoError(err)
   205  		results := parseOutput(context.TODO(), f, podSpec)
   206  
   207  		ginkgo.By("Analyzing results")
   208  		// init container should start and exit with an error, and the regular container should never start
   209  		framework.ExpectNoError(results.Starts(init1))
   210  		framework.ExpectNoError(results.Exits(init1))
   211  
   212  		framework.ExpectNoError(results.DoesntStart(regular1))
   213  	})
   214  
   215  	ginkgo.It("should run Init container to completion before call to PostStart of regular container", func() {
   216  		init1 := "init-1"
   217  		regular1 := "regular-1"
   218  
   219  		podSpec := &v1.Pod{
   220  			ObjectMeta: metav1.ObjectMeta{
   221  				Name: "initcontainer-test-pod-with-post-start",
   222  			},
   223  			Spec: v1.PodSpec{
   224  				RestartPolicy: v1.RestartPolicyNever,
   225  				InitContainers: []v1.Container{
   226  					{
   227  						Name:  init1,
   228  						Image: busyboxImage,
   229  						Command: ExecCommand(init1, execCommand{
   230  							Delay:    1,
   231  							ExitCode: 0,
   232  						}),
   233  					},
   234  				},
   235  				Containers: []v1.Container{
   236  					{
   237  						Name:  regular1,
   238  						Image: busyboxImage,
   239  						Command: ExecCommand(regular1, execCommand{
   240  							// Allocate sufficient time for its postStart hook
   241  							// to complete.
   242  							// Note that we've observed approximately a 2s
   243  							// delay before the postStart hook is called.
   244  							// 10s > 1s + 2s(estimated maximum delay) + other possible delays
   245  							Delay:    10,
   246  							ExitCode: 0,
   247  						}),
   248  						Lifecycle: &v1.Lifecycle{
   249  							PostStart: &v1.LifecycleHandler{
   250  								Exec: &v1.ExecAction{
   251  									Command: ExecCommand(prefixedName(PostStartPrefix, regular1), execCommand{
   252  										Delay:         1,
   253  										ExitCode:      0,
   254  										ContainerName: regular1,
   255  									}),
   256  								},
   257  							},
   258  						},
   259  					},
   260  				},
   261  			},
   262  		}
   263  
   264  		preparePod(podSpec)
   265  
   266  		client := e2epod.NewPodClient(f)
   267  		podSpec = client.Create(context.TODO(), podSpec)
   268  		ginkgo.By("Waiting for the pod to finish")
   269  		err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 1*time.Minute)
   270  		framework.ExpectNoError(err)
   271  
   272  		ginkgo.By("Parsing results")
   273  		podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
   274  		framework.ExpectNoError(err)
   275  		results := parseOutput(context.TODO(), f, podSpec)
   276  
   277  		ginkgo.By("Analyzing results")
   278  		// init container should start and exit with an error, and the regular container should never start
   279  		framework.ExpectNoError(results.StartsBefore(init1, prefixedName(PostStartPrefix, regular1)))
   280  		framework.ExpectNoError(results.ExitsBefore(init1, prefixedName(PostStartPrefix, regular1)))
   281  
   282  		framework.ExpectNoError(results.RunTogether(regular1, prefixedName(PostStartPrefix, regular1)))
   283  	})
   284  
   285  	ginkgo.It("should restart failing container when pod restartPolicy is Always", func() {
   286  
   287  		regular1 := "regular-1"
   288  
   289  		podSpec := &v1.Pod{
   290  			ObjectMeta: metav1.ObjectMeta{
   291  				Name: "container-must-be-restarted",
   292  			},
   293  			Spec: v1.PodSpec{
   294  				RestartPolicy: v1.RestartPolicyAlways,
   295  				Containers: []v1.Container{
   296  					{
   297  						Name:  regular1,
   298  						Image: busyboxImage,
   299  						Command: ExecCommand(regular1, execCommand{
   300  							Delay:    1,
   301  							ExitCode: 1,
   302  						}),
   303  					},
   304  				},
   305  			},
   306  		}
   307  
   308  		preparePod(podSpec)
   309  
   310  		client := e2epod.NewPodClient(f)
   311  		podSpec = client.Create(context.TODO(), podSpec)
   312  		ginkgo.By("Waiting for the pod, it will not finish")
   313  		err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 3, 2*time.Minute)
   314  		framework.ExpectNoError(err)
   315  
   316  		ginkgo.By("Parsing results")
   317  		podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
   318  		framework.ExpectNoError(err)
   319  		results := parseOutput(context.TODO(), f, podSpec)
   320  
   321  		ginkgo.By("Analyzing results")
   322  		// container must be restarted
   323  		framework.ExpectNoError(results.Starts(regular1))
   324  		framework.ExpectNoError(results.StartsBefore(regular1, regular1))
   325  		framework.ExpectNoError(results.ExitsBefore(regular1, regular1))
   326  	})
   327  
   328  	ginkgo.It("should not launch second container before PostStart of the first container completed", func() {
   329  
   330  		regular1 := "regular-1"
   331  		regular2 := "regular-2"
   332  
   333  		podSpec := &v1.Pod{
   334  			ObjectMeta: metav1.ObjectMeta{
   335  				Name: "post-start-blocks-second-container",
   336  			},
   337  			Spec: v1.PodSpec{
   338  				RestartPolicy: v1.RestartPolicyNever,
   339  				Containers: []v1.Container{
   340  					{
   341  						Name:  regular1,
   342  						Image: busyboxImage,
   343  						Command: ExecCommand(regular1, execCommand{
   344  							// Allocate sufficient time for its postStart hook
   345  							// to complete.
   346  							// Note that we've observed approximately a 2s
   347  							// delay before the postStart hook is called.
   348  							// 10s > 1s + 2s(estimated maximum delay) + other possible delays
   349  							Delay:    10,
   350  							ExitCode: 0,
   351  						}),
   352  						Lifecycle: &v1.Lifecycle{
   353  							PostStart: &v1.LifecycleHandler{
   354  								Exec: &v1.ExecAction{
   355  									Command: ExecCommand(prefixedName(PostStartPrefix, regular1), execCommand{
   356  										Delay:         1,
   357  										ExitCode:      0,
   358  										ContainerName: regular1,
   359  									}),
   360  								},
   361  							},
   362  						},
   363  					},
   364  					{
   365  						Name:  regular2,
   366  						Image: busyboxImage,
   367  						Command: ExecCommand(regular2, execCommand{
   368  							Delay:    1,
   369  							ExitCode: 0,
   370  						}),
   371  					},
   372  				},
   373  			},
   374  		}
   375  
   376  		preparePod(podSpec)
   377  
   378  		client := e2epod.NewPodClient(f)
   379  		podSpec = client.Create(context.TODO(), podSpec)
   380  		ginkgo.By("Waiting for the pod to finish")
   381  		err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 1*time.Minute)
   382  		framework.ExpectNoError(err)
   383  
   384  		ginkgo.By("Parsing results")
   385  		podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
   386  		framework.ExpectNoError(err)
   387  		results := parseOutput(context.TODO(), f, podSpec)
   388  
   389  		ginkgo.By("Analyzing results")
   390  		// second container should not start before the PostStart of a first container completed
   391  		framework.ExpectNoError(results.StartsBefore(prefixedName(PostStartPrefix, regular1), regular2))
   392  		framework.ExpectNoError(results.ExitsBefore(prefixedName(PostStartPrefix, regular1), regular2))
   393  	})
   394  
   395  	ginkgo.When("have init container in a Pod with restartPolicy=Never", func() {
   396  
   397  		ginkgo.When("an init container fails to start because of a bad image", ginkgo.Ordered, func() {
   398  
   399  			init1 := "init1-1"
   400  			regular1 := "regular-1"
   401  
   402  			podSpec := &v1.Pod{
   403  				ObjectMeta: metav1.ObjectMeta{
   404  					Name: "bad-image",
   405  				},
   406  				Spec: v1.PodSpec{
   407  					RestartPolicy: v1.RestartPolicyNever,
   408  					InitContainers: []v1.Container{
   409  						{
   410  							Name:  init1,
   411  							Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage),
   412  							Command: ExecCommand(init1, execCommand{
   413  								Delay:    600,
   414  								ExitCode: 0,
   415  							}),
   416  						},
   417  					},
   418  					Containers: []v1.Container{
   419  						{
   420  							Name:  regular1,
   421  							Image: busyboxImage,
   422  							Command: ExecCommand(regular1, execCommand{
   423  								Delay:    1,
   424  								ExitCode: 0,
   425  							}),
   426  						},
   427  					},
   428  				},
   429  			}
   430  
   431  			preparePod(podSpec)
   432  			var results containerOutputList
   433  
   434  			ginkgo.It("should mark a Pod as failed and produce log", func() {
   435  				client := e2epod.NewPodClient(f)
   436  				podSpec = client.Create(context.TODO(), podSpec)
   437  
   438  				err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart)
   439  				framework.ExpectNoError(err)
   440  
   441  				podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
   442  				framework.ExpectNoError(err)
   443  				results = parseOutput(context.TODO(), f, podSpec)
   444  			})
   445  			ginkgo.It("should not start an init container", func() {
   446  				framework.ExpectNoError(results.DoesntStart(init1))
   447  			})
   448  			ginkgo.It("should not start a regular container", func() {
   449  				framework.ExpectNoError(results.DoesntStart(regular1))
   450  			})
   451  		})
   452  	})
   453  
   454  	ginkgo.It("shouldn't restart init containers upon regular container restart", func() {
   455  		init1 := "init-1"
   456  		init2 := "init-2"
   457  		init3 := "init-3"
   458  		regular1 := "regular-1"
   459  
   460  		podSpec := &v1.Pod{
   461  			ObjectMeta: metav1.ObjectMeta{
   462  				Name: "initcontainer-test-pod",
   463  			},
   464  			Spec: v1.PodSpec{
   465  				RestartPolicy: v1.RestartPolicyAlways,
   466  				InitContainers: []v1.Container{
   467  					{
   468  						Name:  init1,
   469  						Image: busyboxImage,
   470  						Command: ExecCommand(init1, execCommand{
   471  							Delay:    1,
   472  							ExitCode: 0,
   473  						}),
   474  					},
   475  					{
   476  						Name:  init2,
   477  						Image: busyboxImage,
   478  						Command: ExecCommand(init2, execCommand{
   479  							Delay:    1,
   480  							ExitCode: 0,
   481  						}),
   482  					},
   483  					{
   484  						Name:  init3,
   485  						Image: busyboxImage,
   486  						Command: ExecCommand(init3, execCommand{
   487  							Delay:    1,
   488  							ExitCode: 0,
   489  						}),
   490  					},
   491  				},
   492  				Containers: []v1.Container{
   493  					{
   494  						Name:  regular1,
   495  						Image: busyboxImage,
   496  						Command: ExecCommand(regular1, execCommand{
   497  							Delay:    10,
   498  							ExitCode: -1,
   499  						}),
   500  					},
   501  				},
   502  			},
   503  		}
   504  
   505  		preparePod(podSpec)
   506  
   507  		client := e2epod.NewPodClient(f)
   508  		podSpec = client.Create(context.TODO(), podSpec)
   509  		ginkgo.By("Waiting for the pod to restart a few times")
   510  		err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 3, 2*time.Minute)
   511  		framework.ExpectNoError(err)
   512  
   513  		ginkgo.By("Parsing results")
   514  		podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
   515  		framework.ExpectNoError(err)
   516  		results := parseOutput(context.TODO(), f, podSpec)
   517  
   518  		ginkgo.By("Analyzing results")
   519  		framework.ExpectNoError(results.StartsBefore(init1, init2))
   520  		framework.ExpectNoError(results.ExitsBefore(init1, init2))
   521  
   522  		framework.ExpectNoError(results.StartsBefore(init2, init3))
   523  		framework.ExpectNoError(results.ExitsBefore(init2, init3))
   524  
   525  		framework.ExpectNoError(results.StartsBefore(init3, regular1))
   526  		framework.ExpectNoError(results.ExitsBefore(init3, regular1))
   527  
   528  		// ensure that the init containers never restarted
   529  		framework.ExpectNoError(results.HasNotRestarted(init1))
   530  		framework.ExpectNoError(results.HasNotRestarted(init2))
   531  		framework.ExpectNoError(results.HasNotRestarted(init3))
   532  		// while the regular container did
   533  		framework.ExpectNoError(results.HasRestarted(regular1))
   534  	})
   535  
   536  	ginkgo.When("a pod cannot terminate gracefully", func() {
   537  		testPod := func(name string, gracePeriod int64) *v1.Pod {
   538  			return &v1.Pod{
   539  				ObjectMeta: metav1.ObjectMeta{
   540  					Name: name,
   541  				},
   542  				Spec: v1.PodSpec{
   543  					Containers: []v1.Container{
   544  						{
   545  							Name:  "busybox",
   546  							Image: imageutils.GetE2EImage(imageutils.BusyBox),
   547  							Command: []string{
   548  								"sleep",
   549  								"10000",
   550  							},
   551  						},
   552  					},
   553  					TerminationGracePeriodSeconds: &gracePeriod,
   554  				},
   555  			}
   556  		}
   557  
   558  		// To account for the time it takes to delete the pod, we add a buffer. Its sized
   559  		// so that we allow up to 2x the grace time to delete the pod. Its extra large to
   560  		// reduce test flakes.
   561  		bufferSeconds := int64(30)
   562  
   563  		f.It("should respect termination grace period seconds", f.WithNodeConformance(), func() {
   564  			client := e2epod.NewPodClient(f)
   565  			gracePeriod := int64(30)
   566  
   567  			ginkgo.By("creating a pod with a termination grace period seconds")
   568  			pod := testPod("pod-termination-grace-period", gracePeriod)
   569  			pod = client.Create(context.TODO(), pod)
   570  
   571  			ginkgo.By("ensuring the pod is running")
   572  			err := e2epod.WaitForPodRunningInNamespace(context.TODO(), f.ClientSet, pod)
   573  			framework.ExpectNoError(err)
   574  
   575  			ginkgo.By("deleting the pod gracefully")
   576  			err = client.Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
   577  			framework.ExpectNoError(err)
   578  
   579  			ginkgo.By("ensuring the pod is terminated within the grace period seconds + buffer seconds")
   580  			err = e2epod.WaitForPodNotFoundInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, time.Duration(gracePeriod+bufferSeconds)*time.Second)
   581  			framework.ExpectNoError(err)
   582  		})
   583  
   584  		f.It("should respect termination grace period seconds with long-running preStop hook", f.WithNodeConformance(), func() {
   585  			client := e2epod.NewPodClient(f)
   586  			gracePeriod := int64(30)
   587  
   588  			ginkgo.By("creating a pod with a termination grace period seconds and long-running preStop hook")
   589  			pod := testPod("pod-termination-grace-period", gracePeriod)
   590  			pod.Spec.Containers[0].Lifecycle = &v1.Lifecycle{
   591  				PreStop: &v1.LifecycleHandler{
   592  					Exec: &v1.ExecAction{
   593  						Command: []string{
   594  							"sleep",
   595  							"10000",
   596  						},
   597  					},
   598  				},
   599  			}
   600  			pod = client.Create(context.TODO(), pod)
   601  
   602  			ginkgo.By("ensuring the pod is running")
   603  			err := e2epod.WaitForPodRunningInNamespace(context.TODO(), f.ClientSet, pod)
   604  			framework.ExpectNoError(err)
   605  
   606  			ginkgo.By("deleting the pod gracefully")
   607  			err = client.Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
   608  			framework.ExpectNoError(err)
   609  
   610  			ginkgo.By("ensuring the pod is terminated within the grace period seconds + buffer seconds")
   611  			err = e2epod.WaitForPodNotFoundInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, time.Duration(gracePeriod+bufferSeconds)*time.Second)
   612  			framework.ExpectNoError(err)
   613  		})
   614  	})
   615  
   616  	ginkgo.It("should call the container's preStop hook and terminate it if its startup probe fails", func() {
   617  		regular1 := "regular-1"
   618  
   619  		podSpec := &v1.Pod{
   620  			ObjectMeta: metav1.ObjectMeta{
   621  				Name: "test-pod",
   622  			},
   623  			Spec: v1.PodSpec{
   624  				RestartPolicy: v1.RestartPolicyNever,
   625  				Containers: []v1.Container{
   626  					{
   627  						Name:  regular1,
   628  						Image: busyboxImage,
   629  						Command: ExecCommand(regular1, execCommand{
   630  							Delay:              100,
   631  							TerminationSeconds: 15,
   632  							ExitCode:           0,
   633  						}),
   634  						StartupProbe: &v1.Probe{
   635  							ProbeHandler: v1.ProbeHandler{
   636  								Exec: &v1.ExecAction{
   637  									Command: []string{
   638  										"sh",
   639  										"-c",
   640  										"exit 1",
   641  									},
   642  								},
   643  							},
   644  							InitialDelaySeconds: 10,
   645  							FailureThreshold:    1,
   646  						},
   647  						Lifecycle: &v1.Lifecycle{
   648  							PreStop: &v1.LifecycleHandler{
   649  								Exec: &v1.ExecAction{
   650  									Command: ExecCommand(prefixedName(PreStopPrefix, regular1), execCommand{
   651  										Delay:         1,
   652  										ExitCode:      0,
   653  										ContainerName: regular1,
   654  									}),
   655  								},
   656  							},
   657  						},
   658  					},
   659  				},
   660  			},
   661  		}
   662  
   663  		preparePod(podSpec)
   664  
   665  		client := e2epod.NewPodClient(f)
   666  		podSpec = client.Create(context.TODO(), podSpec)
   667  
   668  		ginkgo.By("Waiting for the pod to complete")
   669  		err := e2epod.WaitForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace)
   670  		framework.ExpectNoError(err)
   671  
   672  		ginkgo.By("Parsing results")
   673  		podSpec, err = client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
   674  		framework.ExpectNoError(err)
   675  		results := parseOutput(context.TODO(), f, podSpec)
   676  
   677  		ginkgo.By("Analyzing results")
   678  		framework.ExpectNoError(results.RunTogether(regular1, prefixedName(PreStopPrefix, regular1)))
   679  		framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, regular1)))
   680  		framework.ExpectNoError(results.Exits(regular1))
   681  	})
   682  
   683  	ginkgo.It("should call the container's preStop hook and terminate it if its liveness probe fails", func() {
   684  		regular1 := "regular-1"
   685  
   686  		podSpec := &v1.Pod{
   687  			ObjectMeta: metav1.ObjectMeta{
   688  				Name: "test-pod",
   689  			},
   690  			Spec: v1.PodSpec{
   691  				RestartPolicy: v1.RestartPolicyNever,
   692  				Containers: []v1.Container{
   693  					{
   694  						Name:  regular1,
   695  						Image: busyboxImage,
   696  						Command: ExecCommand(regular1, execCommand{
   697  							Delay:              100,
   698  							TerminationSeconds: 15,
   699  							ExitCode:           0,
   700  						}),
   701  						LivenessProbe: &v1.Probe{
   702  							ProbeHandler: v1.ProbeHandler{
   703  								Exec: &v1.ExecAction{
   704  									Command: []string{
   705  										"sh",
   706  										"-c",
   707  										"exit 1",
   708  									},
   709  								},
   710  							},
   711  							InitialDelaySeconds: 10,
   712  							FailureThreshold:    1,
   713  						},
   714  						Lifecycle: &v1.Lifecycle{
   715  							PreStop: &v1.LifecycleHandler{
   716  								Exec: &v1.ExecAction{
   717  									Command: ExecCommand(prefixedName(PreStopPrefix, regular1), execCommand{
   718  										Delay:         1,
   719  										ExitCode:      0,
   720  										ContainerName: regular1,
   721  									}),
   722  								},
   723  							},
   724  						},
   725  					},
   726  				},
   727  			},
   728  		}
   729  
   730  		preparePod(podSpec)
   731  
   732  		client := e2epod.NewPodClient(f)
   733  		podSpec = client.Create(context.TODO(), podSpec)
   734  
   735  		ginkgo.By("Waiting for the pod to complete")
   736  		err := e2epod.WaitForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace)
   737  		framework.ExpectNoError(err)
   738  
   739  		ginkgo.By("Parsing results")
   740  		podSpec, err = client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
   741  		framework.ExpectNoError(err)
   742  		results := parseOutput(context.TODO(), f, podSpec)
   743  
   744  		ginkgo.By("Analyzing results")
   745  		framework.ExpectNoError(results.RunTogether(regular1, prefixedName(PreStopPrefix, regular1)))
   746  		framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, regular1)))
   747  		framework.ExpectNoError(results.Exits(regular1))
   748  	})
   749  })
   750  
   751  var _ = SIGDescribe(framework.WithSerial(), "Containers Lifecycle", func() {
   752  	f := framework.NewDefaultFramework("containers-lifecycle-test-serial")
   753  	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
   754  
   755  	ginkgo.It("should restart the containers in right order after the node reboot", func(ctx context.Context) {
   756  		init1 := "init-1"
   757  		init2 := "init-2"
   758  		init3 := "init-3"
   759  		regular1 := "regular-1"
   760  
   761  		podLabels := map[string]string{
   762  			"test":      "containers-lifecycle-test-serial",
   763  			"namespace": f.Namespace.Name,
   764  		}
   765  		pod := &v1.Pod{
   766  			ObjectMeta: metav1.ObjectMeta{
   767  				Name:   "initialized-pod",
   768  				Labels: podLabels,
   769  			},
   770  			Spec: v1.PodSpec{
   771  				RestartPolicy: v1.RestartPolicyAlways,
   772  				InitContainers: []v1.Container{
   773  					{
   774  						Name:  init1,
   775  						Image: busyboxImage,
   776  						Command: ExecCommand(init1, execCommand{
   777  							Delay:    5,
   778  							ExitCode: 0,
   779  						}),
   780  					},
   781  					{
   782  						Name:  init2,
   783  						Image: busyboxImage,
   784  						Command: ExecCommand(init2, execCommand{
   785  							Delay:    5,
   786  							ExitCode: 0,
   787  						}),
   788  					},
   789  					{
   790  						Name:  init3,
   791  						Image: busyboxImage,
   792  						Command: ExecCommand(init3, execCommand{
   793  							Delay:    5,
   794  							ExitCode: 0,
   795  						}),
   796  					},
   797  				},
   798  				Containers: []v1.Container{
   799  					{
   800  						Name:  regular1,
   801  						Image: busyboxImage,
   802  						Command: ExecCommand(regular1, execCommand{
   803  							Delay:    30,
   804  							ExitCode: 0,
   805  						}),
   806  					},
   807  				},
   808  			},
   809  		}
   810  		preparePod(pod)
   811  
   812  		client := e2epod.NewPodClient(f)
   813  		pod = client.Create(ctx, pod)
   814  		ginkgo.By("Waiting for the pod to be initialized and run")
   815  		err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
   816  		framework.ExpectNoError(err)
   817  
   818  		ginkgo.By("Getting the current pod sandbox ID")
   819  		rs, _, err := getCRIClient()
   820  		framework.ExpectNoError(err)
   821  
   822  		sandboxes, err := rs.ListPodSandbox(ctx, &runtimeapi.PodSandboxFilter{
   823  			LabelSelector: podLabels,
   824  		})
   825  		framework.ExpectNoError(err)
   826  		gomega.Expect(sandboxes).To(gomega.HaveLen(1))
   827  		podSandboxID := sandboxes[0].Id
   828  
   829  		ginkgo.By("Stopping the kubelet")
   830  		restartKubelet := stopKubelet()
   831  		gomega.Eventually(ctx, func() bool {
   832  			return kubeletHealthCheck(kubeletHealthCheckURL)
   833  		}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse())
   834  
   835  		ginkgo.By("Stopping the pod sandbox to simulate the node reboot")
   836  		err = rs.StopPodSandbox(ctx, podSandboxID)
   837  		framework.ExpectNoError(err)
   838  
   839  		ginkgo.By("Restarting the kubelet")
   840  		restartKubelet()
   841  		gomega.Eventually(ctx, func() bool {
   842  			return kubeletHealthCheck(kubeletHealthCheckURL)
   843  		}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue())
   844  
   845  		ginkgo.By("Waiting for the pod to be re-initialized and run")
   846  		err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "re-initialized", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) {
   847  			if pod.Status.ContainerStatuses[0].RestartCount < 2 {
   848  				return false, nil
   849  			}
   850  			if pod.Status.Phase != v1.PodRunning {
   851  				return false, nil
   852  			}
   853  			return true, nil
   854  		})
   855  		framework.ExpectNoError(err)
   856  
   857  		ginkgo.By("Parsing results")
   858  		pod, err = client.Get(ctx, pod.Name, metav1.GetOptions{})
   859  		framework.ExpectNoError(err)
   860  		results := parseOutput(context.TODO(), f, pod)
   861  
   862  		ginkgo.By("Analyzing results")
   863  		init1Started, err := results.FindIndex(init1, "Started", 0)
   864  		framework.ExpectNoError(err)
   865  		init2Started, err := results.FindIndex(init2, "Started", 0)
   866  		framework.ExpectNoError(err)
   867  		init3Started, err := results.FindIndex(init3, "Started", 0)
   868  		framework.ExpectNoError(err)
   869  		regular1Started, err := results.FindIndex(regular1, "Started", 0)
   870  		framework.ExpectNoError(err)
   871  
   872  		init1Restarted, err := results.FindIndex(init1, "Started", init1Started+1)
   873  		framework.ExpectNoError(err)
   874  		init2Restarted, err := results.FindIndex(init2, "Started", init2Started+1)
   875  		framework.ExpectNoError(err)
   876  		init3Restarted, err := results.FindIndex(init3, "Started", init3Started+1)
   877  		framework.ExpectNoError(err)
   878  		regular1Restarted, err := results.FindIndex(regular1, "Started", regular1Started+1)
   879  		framework.ExpectNoError(err)
   880  
   881  		framework.ExpectNoError(init1Started.IsBefore(init2Started))
   882  		framework.ExpectNoError(init2Started.IsBefore(init3Started))
   883  		framework.ExpectNoError(init3Started.IsBefore(regular1Started))
   884  
   885  		framework.ExpectNoError(init1Restarted.IsBefore(init2Restarted))
   886  		framework.ExpectNoError(init2Restarted.IsBefore(init3Restarted))
   887  		framework.ExpectNoError(init3Restarted.IsBefore(regular1Restarted))
   888  	})
   889  })
   890  
   891  var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func() {
   892  	f := framework.NewDefaultFramework("containers-lifecycle-test")
   893  	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
   894  
   895  	ginkgo.When("using a Pod with restartPolicy=Never, three init container and two restartable init containers", ginkgo.Ordered, func() {
   896  
   897  		init1 := "init-1"
   898  		restartableInit1 := "restartable-init-1"
   899  		init2 := "init-2"
   900  		restartableInit2 := "restartable-init-2"
   901  		init3 := "init-3"
   902  		regular1 := "regular-1"
   903  
   904  		podSpec := &v1.Pod{
   905  			ObjectMeta: metav1.ObjectMeta{
   906  				Name: "restartable-init-containers-start-serially",
   907  			},
   908  			Spec: v1.PodSpec{
   909  				RestartPolicy: v1.RestartPolicyNever,
   910  				InitContainers: []v1.Container{
   911  					{
   912  						Name:  init1,
   913  						Image: busyboxImage,
   914  						Command: ExecCommand(init1, execCommand{
   915  							Delay:    1,
   916  							ExitCode: 0,
   917  						}),
   918  					},
   919  					{
   920  						Name:  restartableInit1,
   921  						Image: busyboxImage,
   922  						Command: ExecCommand(restartableInit1, execCommand{
   923  							Delay:    600,
   924  							ExitCode: 0,
   925  						}),
   926  						RestartPolicy: &containerRestartPolicyAlways,
   927  					},
   928  					{
   929  						Name:  init2,
   930  						Image: busyboxImage,
   931  						Command: ExecCommand(init2, execCommand{
   932  							Delay:    1,
   933  							ExitCode: 0,
   934  						}),
   935  					},
   936  					{
   937  						Name:  restartableInit2,
   938  						Image: busyboxImage,
   939  						Command: ExecCommand(restartableInit2, execCommand{
   940  							Delay:    600,
   941  							ExitCode: 0,
   942  						}),
   943  						RestartPolicy: &containerRestartPolicyAlways,
   944  					},
   945  					{
   946  						Name:  init3,
   947  						Image: busyboxImage,
   948  						Command: ExecCommand(init3, execCommand{
   949  							Delay:    1,
   950  							ExitCode: 0,
   951  						}),
   952  					},
   953  				},
   954  				Containers: []v1.Container{
   955  					{
   956  						Name:  regular1,
   957  						Image: busyboxImage,
   958  						Command: ExecCommand(regular1, execCommand{
   959  							Delay:    1,
   960  							ExitCode: 0,
   961  						}),
   962  					},
   963  				},
   964  			},
   965  		}
   966  
   967  		preparePod(podSpec)
   968  		var results containerOutputList
   969  
   970  		ginkgo.It("should finish and produce log", func() {
   971  			client := e2epod.NewPodClient(f)
   972  			podSpec = client.Create(context.TODO(), podSpec)
   973  
   974  			err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute)
   975  			framework.ExpectNoError(err)
   976  
   977  			podSpec, err := client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
   978  			framework.ExpectNoError(err)
   979  
   980  			// pod should exit successfully
   981  			gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
   982  
   983  			results = parseOutput(context.TODO(), f, podSpec)
   984  		})
   985  
   986  		ginkgo.It("should run the first init container to completion before starting first restartable init container", func() {
   987  			framework.ExpectNoError(results.StartsBefore(init1, restartableInit1))
   988  			framework.ExpectNoError(results.ExitsBefore(init1, restartableInit1))
   989  		})
   990  
   991  		ginkgo.It("should start first restartable init container before starting second init container", func() {
   992  			framework.ExpectNoError(results.StartsBefore(restartableInit1, init2))
   993  		})
   994  
   995  		ginkgo.It("should run first init container and first restartable init container together", func() {
   996  			framework.ExpectNoError(results.RunTogether(restartableInit1, init2))
   997  		})
   998  
   999  		ginkgo.It("should run second init container to completion before starting second restartable init container", func() {
  1000  			framework.ExpectNoError(results.StartsBefore(init2, restartableInit2))
  1001  			framework.ExpectNoError(results.ExitsBefore(init2, restartableInit2))
  1002  		})
  1003  
  1004  		ginkgo.It("should start second restartable init container before third init container", func() {
  1005  			framework.ExpectNoError(results.StartsBefore(restartableInit2, init3))
  1006  		})
  1007  
  1008  		ginkgo.It("should run both restartable init containers and third init container together", func() {
  1009  			framework.ExpectNoError(results.RunTogether(restartableInit2, restartableInit1))
  1010  			framework.ExpectNoError(results.RunTogether(restartableInit1, init3))
  1011  			framework.ExpectNoError(results.RunTogether(restartableInit2, init3))
  1012  		})
  1013  
  1014  		ginkgo.It("should run third init container to completion before starting regular container", func() {
  1015  			framework.ExpectNoError(results.StartsBefore(init3, regular1))
  1016  			framework.ExpectNoError(results.ExitsBefore(init3, regular1))
  1017  		})
  1018  
  1019  		ginkgo.It("should run both restartable init containers and a regular container together", func() {
  1020  			framework.ExpectNoError(results.RunTogether(restartableInit1, regular1))
  1021  			framework.ExpectNoError(results.RunTogether(restartableInit2, regular1))
  1022  		})
  1023  	})
  1024  
  1025  	ginkgo.When("using a restartable init container in a Pod with restartPolicy=Never", func() {
  1026  		ginkgo.When("a restartable init container runs continuously", ginkgo.Ordered, func() {
  1027  
  1028  			restartableInit1 := "restartable-init-1"
  1029  			regular1 := "regular-1"
  1030  
  1031  			podSpec := &v1.Pod{
  1032  				ObjectMeta: metav1.ObjectMeta{
  1033  					Name: "restartable-init-container-run-continuously",
  1034  				},
  1035  				Spec: v1.PodSpec{
  1036  					RestartPolicy: v1.RestartPolicyNever,
  1037  					InitContainers: []v1.Container{
  1038  						{
  1039  							Name:  restartableInit1,
  1040  							Image: busyboxImage,
  1041  							Command: ExecCommand(restartableInit1, execCommand{
  1042  								Delay:    600,
  1043  								ExitCode: 0,
  1044  							}),
  1045  							RestartPolicy: &containerRestartPolicyAlways,
  1046  						},
  1047  					},
  1048  					Containers: []v1.Container{
  1049  						{
  1050  							Name:  regular1,
  1051  							Image: busyboxImage,
  1052  							Command: ExecCommand(regular1, execCommand{
  1053  								Delay:    1,
  1054  								ExitCode: 0,
  1055  							}),
  1056  						},
  1057  					},
  1058  				},
  1059  			}
  1060  
  1061  			preparePod(podSpec)
  1062  			var results containerOutputList
  1063  
  1064  			ginkgo.It("should complete a Pod successfully and produce log", func() {
  1065  				client := e2epod.NewPodClient(f)
  1066  				podSpec = client.Create(context.TODO(), podSpec)
  1067  
  1068  				err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute)
  1069  				framework.ExpectNoError(err)
  1070  
  1071  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1072  				framework.ExpectNoError(err)
  1073  
  1074  				// pod should exit successfully
  1075  				gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
  1076  
  1077  				results = parseOutput(context.TODO(), f, podSpec)
  1078  			})
  1079  			ginkgo.It("should not restart a restartable init container", func() {
  1080  				framework.ExpectNoError(results.DoesntStartAfter(restartableInit1, regular1))
  1081  			})
  1082  			ginkgo.It("should run a regular container to completion", func() {
  1083  				framework.ExpectNoError(results.Exits(regular1))
  1084  			})
  1085  		})
  1086  
  1087  		ginkgo.When("a restartable init container fails to start because of a bad image", ginkgo.Ordered, func() {
  1088  
  1089  			restartableInit1 := "restartable-init-1"
  1090  			regular1 := "regular-1"
  1091  
  1092  			podSpec := &v1.Pod{
  1093  				ObjectMeta: metav1.ObjectMeta{
  1094  					Name: "restartable-init-runs-with-pod",
  1095  				},
  1096  				Spec: v1.PodSpec{
  1097  					RestartPolicy: v1.RestartPolicyNever,
  1098  					InitContainers: []v1.Container{
  1099  						{
  1100  							Name:  restartableInit1,
  1101  							Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage),
  1102  							Command: ExecCommand(restartableInit1, execCommand{
  1103  								Delay:    600,
  1104  								ExitCode: 0,
  1105  							}),
  1106  							RestartPolicy: &containerRestartPolicyAlways,
  1107  						},
  1108  					},
  1109  					Containers: []v1.Container{
  1110  						{
  1111  							Name:  regular1,
  1112  							Image: busyboxImage,
  1113  							Command: ExecCommand(regular1, execCommand{
  1114  								Delay:    1,
  1115  								ExitCode: 0,
  1116  							}),
  1117  						},
  1118  					},
  1119  				},
  1120  			}
  1121  
  1122  			preparePod(podSpec)
  1123  			var results containerOutputList
  1124  
  1125  			ginkgo.It("should mark a Pod as failed and produce log", func() {
  1126  				client := e2epod.NewPodClient(f)
  1127  				podSpec = client.Create(context.TODO(), podSpec)
  1128  
  1129  				// restartable init container should be in image pull backoff
  1130  				err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart)
  1131  				framework.ExpectNoError(err)
  1132  
  1133  				podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
  1134  				framework.ExpectNoError(err)
  1135  				results = parseOutput(context.TODO(), f, podSpec)
  1136  			})
  1137  			ginkgo.It("should not start a restartable init container", func() {
  1138  				framework.ExpectNoError(results.DoesntStart(restartableInit1))
  1139  			})
  1140  			ginkgo.It("should not start a regular container", func() {
  1141  				framework.ExpectNoError(results.DoesntStart(regular1))
  1142  			})
  1143  		})
  1144  
  1145  		// TODO: add a test case similar to one above, but with startup probe never succeeding
  1146  
  1147  		ginkgo.When("a restartable init container starts and exits with exit code 0 continuously", ginkgo.Ordered, func() {
  1148  
  1149  			restartableInit1 := "restartable-init-1"
  1150  			init1 := "init-1"
  1151  			regular1 := "regular-1"
  1152  
  1153  			podSpec := &v1.Pod{
  1154  				ObjectMeta: metav1.ObjectMeta{
  1155  					Name: "restartable-init-container-exit-0-continuously",
  1156  				},
  1157  				Spec: v1.PodSpec{
  1158  					RestartPolicy: v1.RestartPolicyNever,
  1159  					InitContainers: []v1.Container{
  1160  						{
  1161  							Name:  restartableInit1,
  1162  							Image: busyboxImage,
  1163  							Command: ExecCommand(restartableInit1, execCommand{
  1164  								Delay:    1,
  1165  								ExitCode: 0,
  1166  							}),
  1167  							RestartPolicy: &containerRestartPolicyAlways,
  1168  						},
  1169  						{
  1170  							Name:  init1,
  1171  							Image: busyboxImage,
  1172  							Command: ExecCommand(init1, execCommand{
  1173  								Delay:    5,
  1174  								ExitCode: 0,
  1175  							}),
  1176  						},
  1177  					},
  1178  					Containers: []v1.Container{
  1179  						{
  1180  							Name:  regular1,
  1181  							Image: busyboxImage,
  1182  							Command: ExecCommand(regular1, execCommand{
  1183  								Delay:    60,
  1184  								ExitCode: 0,
  1185  							}),
  1186  						},
  1187  					},
  1188  				},
  1189  			}
  1190  
  1191  			preparePod(podSpec)
  1192  			var results containerOutputList
  1193  
  1194  			ginkgo.It("should complete a Pod successfully and produce log", func() {
  1195  				client := e2epod.NewPodClient(f)
  1196  				podSpec = client.Create(context.TODO(), podSpec)
  1197  
  1198  				err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute)
  1199  				framework.ExpectNoError(err)
  1200  
  1201  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1202  				framework.ExpectNoError(err)
  1203  
  1204  				// pod should exit successfully
  1205  				gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
  1206  				results = parseOutput(context.TODO(), f, podSpec)
  1207  			})
  1208  			ginkgo.It("should restart a restartable init container before the regular container started", func() {
  1209  				framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1))
  1210  			})
  1211  			ginkgo.It("should restart a restartable init container after the regular container started", func() {
  1212  				framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1))
  1213  			})
  1214  			ginkgo.It("should run a regular container to completion", func() {
  1215  				framework.ExpectNoError(results.Exits(regular1))
  1216  			})
  1217  		})
  1218  
  1219  		ginkgo.When("a restartable init container starts and exits with exit code 1 continuously", ginkgo.Ordered, func() {
  1220  			restartableInit1 := "restartable-init-1"
  1221  			init1 := "init-1"
  1222  			regular1 := "regular-1"
  1223  
  1224  			podSpec := &v1.Pod{
  1225  				ObjectMeta: metav1.ObjectMeta{
  1226  					Name: "restartable-init-container-exit-1-continuously",
  1227  				},
  1228  				Spec: v1.PodSpec{
  1229  					RestartPolicy: v1.RestartPolicyNever,
  1230  					InitContainers: []v1.Container{
  1231  						{
  1232  							Name:  restartableInit1,
  1233  							Image: busyboxImage,
  1234  							Command: ExecCommand(restartableInit1, execCommand{
  1235  								Delay:    1,
  1236  								ExitCode: 1,
  1237  							}),
  1238  							RestartPolicy: &containerRestartPolicyAlways,
  1239  						},
  1240  						{
  1241  							Name:  init1,
  1242  							Image: busyboxImage,
  1243  							Command: ExecCommand(init1, execCommand{
  1244  								Delay:    5,
  1245  								ExitCode: 0,
  1246  							}),
  1247  						},
  1248  					},
  1249  					Containers: []v1.Container{
  1250  						{
  1251  							Name:  regular1,
  1252  							Image: busyboxImage,
  1253  							Command: ExecCommand(regular1, execCommand{
  1254  								Delay:    60,
  1255  								ExitCode: 0,
  1256  							}),
  1257  						},
  1258  					},
  1259  				},
  1260  			}
  1261  
  1262  			preparePod(podSpec)
  1263  			var results containerOutputList
  1264  
  1265  			ginkgo.It("should complete a Pod successfully and produce log", func() {
  1266  				client := e2epod.NewPodClient(f)
  1267  				podSpec = client.Create(context.TODO(), podSpec)
  1268  
  1269  				err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute)
  1270  				framework.ExpectNoError(err)
  1271  
  1272  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1273  				framework.ExpectNoError(err)
  1274  
  1275  				// pod should exit successfully
  1276  				gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
  1277  
  1278  				results = parseOutput(context.TODO(), f, podSpec)
  1279  			})
  1280  			ginkgo.It("should restart a restartable init container before the regular container started", func() {
  1281  				framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1))
  1282  			})
  1283  			ginkgo.It("should restart a restartable init container after the regular container started", func() {
  1284  				framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1))
  1285  			})
  1286  			ginkgo.It("should run a regular container to completion", func() {
  1287  				framework.ExpectNoError(results.Exits(regular1))
  1288  			})
  1289  		})
  1290  
  1291  		ginkgo.When("an Init container before restartable init container fails", ginkgo.Ordered, func() {
  1292  
  1293  			init1 := "init-1"
  1294  			restartableInit1 := "restartable-init-1"
  1295  			regular1 := "regular-1"
  1296  
  1297  			podSpec := &v1.Pod{
  1298  				ObjectMeta: metav1.ObjectMeta{
  1299  					Name: "init-container-fails-before-restartable-init-starts",
  1300  				},
  1301  				Spec: v1.PodSpec{
  1302  					RestartPolicy: v1.RestartPolicyNever,
  1303  					InitContainers: []v1.Container{
  1304  						{
  1305  							Name:  init1,
  1306  							Image: busyboxImage,
  1307  							Command: ExecCommand(init1, execCommand{
  1308  								Delay:    1,
  1309  								ExitCode: 1,
  1310  							}),
  1311  						},
  1312  						{
  1313  							Name:  restartableInit1,
  1314  							Image: busyboxImage,
  1315  							Command: ExecCommand(restartableInit1, execCommand{
  1316  								Delay:    600,
  1317  								ExitCode: 0,
  1318  							}),
  1319  							RestartPolicy: &containerRestartPolicyAlways,
  1320  						},
  1321  					},
  1322  					Containers: []v1.Container{
  1323  						{
  1324  							Name:  regular1,
  1325  							Image: busyboxImage,
  1326  							Command: ExecCommand(regular1, execCommand{
  1327  								Delay:    600,
  1328  								ExitCode: 0,
  1329  							}),
  1330  						},
  1331  					},
  1332  				},
  1333  			}
  1334  
  1335  			preparePod(podSpec)
  1336  			var results containerOutputList
  1337  
  1338  			ginkgo.It("should mark a Pod as failed and produce log", func() {
  1339  				client := e2epod.NewPodClient(f)
  1340  				podSpec = client.Create(context.TODO(), podSpec)
  1341  
  1342  				err := e2epod.WaitForPodFailedReason(context.TODO(), f.ClientSet, podSpec, "", 1*time.Minute)
  1343  				framework.ExpectNoError(err)
  1344  
  1345  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1346  				framework.ExpectNoError(err)
  1347  				results = parseOutput(context.TODO(), f, podSpec)
  1348  			})
  1349  			ginkgo.It("should mark an Init container as failed", func() {
  1350  				framework.ExpectNoError(results.Exits(init1))
  1351  			})
  1352  			ginkgo.It("should not start restartable init container", func() {
  1353  				framework.ExpectNoError(results.DoesntStart(restartableInit1))
  1354  			})
  1355  		})
  1356  
  1357  		ginkgo.When("an Init container after restartable init container fails", ginkgo.Ordered, func() {
  1358  
  1359  			init1 := "init-1"
  1360  			restartableInit1 := "restartable-init-1"
  1361  			regular1 := "regular-1"
  1362  
  1363  			podSpec := &v1.Pod{
  1364  				ObjectMeta: metav1.ObjectMeta{
  1365  					Name: "restartable-init-container-fails-before-init-container",
  1366  				},
  1367  				Spec: v1.PodSpec{
  1368  					RestartPolicy: v1.RestartPolicyNever,
  1369  					InitContainers: []v1.Container{
  1370  						{
  1371  							Name:  restartableInit1,
  1372  							Image: busyboxImage,
  1373  							Command: ExecCommand(restartableInit1, execCommand{
  1374  								Delay:    1,
  1375  								ExitCode: 1,
  1376  							}),
  1377  							RestartPolicy: &containerRestartPolicyAlways,
  1378  						},
  1379  						{
  1380  							Name:  init1,
  1381  							Image: busyboxImage,
  1382  							Command: ExecCommand(init1, execCommand{
  1383  								Delay:    1,
  1384  								ExitCode: 1,
  1385  							}),
  1386  						},
  1387  					},
  1388  					Containers: []v1.Container{
  1389  						{
  1390  							Name:  regular1,
  1391  							Image: busyboxImage,
  1392  							Command: ExecCommand(regular1, execCommand{
  1393  								Delay:    600,
  1394  								ExitCode: 0,
  1395  							}),
  1396  						},
  1397  					},
  1398  				},
  1399  			}
  1400  
  1401  			preparePod(podSpec)
  1402  			var results containerOutputList
  1403  
  1404  			ginkgo.It("should mark a Pod as failed and produce log", func() {
  1405  				client := e2epod.NewPodClient(f)
  1406  				podSpec = client.Create(context.TODO(), podSpec)
  1407  
  1408  				err := e2epod.WaitForPodFailedReason(context.TODO(), f.ClientSet, podSpec, "", 1*time.Minute)
  1409  				framework.ExpectNoError(err)
  1410  
  1411  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1412  				framework.ExpectNoError(err)
  1413  				results = parseOutput(context.TODO(), f, podSpec)
  1414  			})
  1415  			ginkgo.It("should mark an Init container as failed", func() {
  1416  				framework.ExpectNoError(results.Exits(init1))
  1417  			})
  1418  			// TODO: how will we be able to test it if restartable init container
  1419  			// will never fail and there will be no termination log? Or will be?
  1420  			ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() {
  1421  				framework.ExpectNoError(results.RunTogether(restartableInit1, init1))
  1422  			})
  1423  		})
  1424  	})
  1425  
  1426  	ginkgo.When("using a restartable init container in a Pod with restartPolicy=OnFailure", ginkgo.Ordered, func() {
  1427  		// this test case the same as for restartPolicy=Never
  1428  		ginkgo.When("a restartable init container runs continuously", func() {
  1429  
  1430  			restartableInit1 := "restartable-init-1"
  1431  			regular1 := "regular-1"
  1432  
  1433  			podSpec := &v1.Pod{
  1434  				ObjectMeta: metav1.ObjectMeta{
  1435  					Name: "restartable-init-container-run-continuously",
  1436  				},
  1437  				Spec: v1.PodSpec{
  1438  					RestartPolicy: v1.RestartPolicyOnFailure,
  1439  					InitContainers: []v1.Container{
  1440  						{
  1441  							Name:  restartableInit1,
  1442  							Image: busyboxImage,
  1443  							Command: ExecCommand(restartableInit1, execCommand{
  1444  								Delay:    600,
  1445  								ExitCode: 0,
  1446  							}),
  1447  							RestartPolicy: &containerRestartPolicyAlways,
  1448  						},
  1449  					},
  1450  					Containers: []v1.Container{
  1451  						{
  1452  							Name:  regular1,
  1453  							Image: busyboxImage,
  1454  							Command: ExecCommand(regular1, execCommand{
  1455  								Delay:    1,
  1456  								ExitCode: 0,
  1457  							}),
  1458  						},
  1459  					},
  1460  				},
  1461  			}
  1462  
  1463  			preparePod(podSpec)
  1464  			var results containerOutputList
  1465  
  1466  			ginkgo.It("should complete a Pod successfully and produce log", func() {
  1467  				client := e2epod.NewPodClient(f)
  1468  				podSpec = client.Create(context.TODO(), podSpec)
  1469  
  1470  				err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute)
  1471  				framework.ExpectNoError(err)
  1472  
  1473  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1474  				framework.ExpectNoError(err)
  1475  
  1476  				// pod should exit successfully
  1477  				gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
  1478  
  1479  				results = parseOutput(context.TODO(), f, podSpec)
  1480  			})
  1481  			ginkgo.It("should not restart a restartable init container", func() {
  1482  				framework.ExpectNoError(results.DoesntStartAfter(restartableInit1, regular1))
  1483  			})
  1484  			ginkgo.It("should run a regular container to completion", func() {
  1485  				framework.ExpectNoError(results.Exits(regular1))
  1486  			})
  1487  		})
  1488  
  1489  		ginkgo.When("a restartable init container fails to start because of a bad image", ginkgo.Ordered, func() {
  1490  
  1491  			restartableInit1 := "restartable-init-1"
  1492  			regular1 := "regular-1"
  1493  
  1494  			podSpec := &v1.Pod{
  1495  				ObjectMeta: metav1.ObjectMeta{
  1496  					Name: "restartable-init-runs-with-pod",
  1497  				},
  1498  				Spec: v1.PodSpec{
  1499  					RestartPolicy: v1.RestartPolicyOnFailure,
  1500  					InitContainers: []v1.Container{
  1501  						{
  1502  							Name:  restartableInit1,
  1503  							Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage),
  1504  							Command: ExecCommand(restartableInit1, execCommand{
  1505  								Delay:    600,
  1506  								ExitCode: 0,
  1507  							}),
  1508  							RestartPolicy: &containerRestartPolicyAlways,
  1509  						},
  1510  					},
  1511  					Containers: []v1.Container{
  1512  						{
  1513  							Name:  regular1,
  1514  							Image: busyboxImage,
  1515  							Command: ExecCommand(regular1, execCommand{
  1516  								Delay:    1,
  1517  								ExitCode: 0,
  1518  							}),
  1519  						},
  1520  					},
  1521  				},
  1522  			}
  1523  
  1524  			preparePod(podSpec)
  1525  			var results containerOutputList
  1526  
  1527  			ginkgo.It("should mark a Pod as failed and produce log", func() {
  1528  				client := e2epod.NewPodClient(f)
  1529  				podSpec = client.Create(context.TODO(), podSpec)
  1530  
  1531  				// restartable init container should be in image pull backoff
  1532  				err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart)
  1533  				framework.ExpectNoError(err)
  1534  
  1535  				podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
  1536  				framework.ExpectNoError(err)
  1537  				results = parseOutput(context.TODO(), f, podSpec)
  1538  			})
  1539  			ginkgo.It("should not start a restartable init container", func() {
  1540  				framework.ExpectNoError(results.DoesntStart(restartableInit1))
  1541  			})
  1542  			ginkgo.It("should not start a regular container", func() {
  1543  				framework.ExpectNoError(results.DoesntStart(regular1))
  1544  			})
  1545  		})
  1546  
  1547  		// TODO: add a test case similar to one above, but with startup probe never succeeding
  1548  
  1549  		// this test case the same as for restartPolicy=Never
  1550  		ginkgo.When("a restartable init container starts and exits with exit code 0 continuously", ginkgo.Ordered, func() {
  1551  
  1552  			restartableInit1 := "restartable-init-1"
  1553  			init1 := "init-1"
  1554  			regular1 := "regular-1"
  1555  
  1556  			podSpec := &v1.Pod{
  1557  				ObjectMeta: metav1.ObjectMeta{
  1558  					Name: "restartable-init-container-exit-0-continuously",
  1559  				},
  1560  				Spec: v1.PodSpec{
  1561  					RestartPolicy: v1.RestartPolicyOnFailure,
  1562  					InitContainers: []v1.Container{
  1563  						{
  1564  							Name:  restartableInit1,
  1565  							Image: busyboxImage,
  1566  							Command: ExecCommand(restartableInit1, execCommand{
  1567  								Delay:    1,
  1568  								ExitCode: 0,
  1569  							}),
  1570  							RestartPolicy: &containerRestartPolicyAlways,
  1571  						},
  1572  						{
  1573  							Name:  init1,
  1574  							Image: busyboxImage,
  1575  							Command: ExecCommand(init1, execCommand{
  1576  								Delay:    5,
  1577  								ExitCode: 0,
  1578  							}),
  1579  						},
  1580  					},
  1581  					Containers: []v1.Container{
  1582  						{
  1583  							Name:  regular1,
  1584  							Image: busyboxImage,
  1585  							Command: ExecCommand(regular1, execCommand{
  1586  								Delay:    60,
  1587  								ExitCode: 0,
  1588  							}),
  1589  						},
  1590  					},
  1591  				},
  1592  			}
  1593  
  1594  			preparePod(podSpec)
  1595  			var results containerOutputList
  1596  
  1597  			ginkgo.It("should complete a Pod successfully and produce log", func() {
  1598  				client := e2epod.NewPodClient(f)
  1599  				podSpec = client.Create(context.TODO(), podSpec)
  1600  
  1601  				err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute)
  1602  				framework.ExpectNoError(err)
  1603  
  1604  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1605  				framework.ExpectNoError(err)
  1606  
  1607  				// pod should exit successfully
  1608  				gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
  1609  
  1610  				results = parseOutput(context.TODO(), f, podSpec)
  1611  			})
  1612  			ginkgo.It("should restart a restartable init container before the regular container started", func() {
  1613  				framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1))
  1614  			})
  1615  			ginkgo.It("should restart a restartable init container after the regular container started", func() {
  1616  				framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1))
  1617  			})
  1618  			ginkgo.It("should run a regular container to completion", func() {
  1619  				framework.ExpectNoError(results.Exits(regular1))
  1620  			})
  1621  		})
  1622  
  1623  		// this test case the same as for restartPolicy=Never
  1624  		ginkgo.When("a restartable init container starts and exits with exit code 1 continuously", ginkgo.Ordered, func() {
  1625  
  1626  			restartableInit1 := "restartable-init-1"
  1627  			init1 := "init-1"
  1628  			regular1 := "regular-1"
  1629  
  1630  			podSpec := &v1.Pod{
  1631  				ObjectMeta: metav1.ObjectMeta{
  1632  					Name: "restartable-init-container-exit-1-continuously",
  1633  				},
  1634  				Spec: v1.PodSpec{
  1635  					RestartPolicy: v1.RestartPolicyOnFailure,
  1636  					InitContainers: []v1.Container{
  1637  						{
  1638  							Name:  restartableInit1,
  1639  							Image: busyboxImage,
  1640  							Command: ExecCommand(restartableInit1, execCommand{
  1641  								Delay:    1,
  1642  								ExitCode: 1,
  1643  							}),
  1644  							RestartPolicy: &containerRestartPolicyAlways,
  1645  						},
  1646  						{
  1647  							Name:  init1,
  1648  							Image: busyboxImage,
  1649  							Command: ExecCommand(init1, execCommand{
  1650  								Delay:    5,
  1651  								ExitCode: 0,
  1652  							}),
  1653  						},
  1654  					},
  1655  					Containers: []v1.Container{
  1656  						{
  1657  							Name:  regular1,
  1658  							Image: busyboxImage,
  1659  							Command: ExecCommand(regular1, execCommand{
  1660  								Delay:    60,
  1661  								ExitCode: 0,
  1662  							}),
  1663  						},
  1664  					},
  1665  				},
  1666  			}
  1667  
  1668  			preparePod(podSpec)
  1669  			var results containerOutputList
  1670  
  1671  			ginkgo.It("should complete a Pod successfully and produce log", func() {
  1672  				client := e2epod.NewPodClient(f)
  1673  				podSpec = client.Create(context.TODO(), podSpec)
  1674  
  1675  				err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute)
  1676  				framework.ExpectNoError(err)
  1677  
  1678  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1679  				framework.ExpectNoError(err)
  1680  
  1681  				// pod should exit successfully
  1682  				gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
  1683  
  1684  				results = parseOutput(context.TODO(), f, podSpec)
  1685  			})
  1686  			ginkgo.It("should restart a restartable init container before the regular container started", func() {
  1687  				framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1))
  1688  			})
  1689  			ginkgo.It("should restart a restartable init container after the regular container started", func() {
  1690  				framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1))
  1691  			})
  1692  			ginkgo.It("should run a regular container to completion", func() {
  1693  				framework.ExpectNoError(results.Exits(regular1))
  1694  			})
  1695  		})
  1696  
  1697  		ginkgo.When("an Init container before restartable init container continuously fails", ginkgo.Ordered, func() {
  1698  
  1699  			init1 := "init-1"
  1700  			restartableInit1 := "restartable-init-1"
  1701  			regular1 := "regular-1"
  1702  
  1703  			podSpec := &v1.Pod{
  1704  				ObjectMeta: metav1.ObjectMeta{
  1705  					Name: "init-container-fails-before-restartable-init-starts",
  1706  				},
  1707  				Spec: v1.PodSpec{
  1708  					RestartPolicy: v1.RestartPolicyOnFailure,
  1709  					InitContainers: []v1.Container{
  1710  						{
  1711  							Name:  init1,
  1712  							Image: busyboxImage,
  1713  							Command: ExecCommand(init1, execCommand{
  1714  								Delay:    1,
  1715  								ExitCode: 1,
  1716  							}),
  1717  						},
  1718  						{
  1719  							Name:  restartableInit1,
  1720  							Image: busyboxImage,
  1721  							Command: ExecCommand(restartableInit1, execCommand{
  1722  								Delay:    600,
  1723  								ExitCode: 0,
  1724  							}),
  1725  							RestartPolicy: &containerRestartPolicyAlways,
  1726  						},
  1727  					},
  1728  					Containers: []v1.Container{
  1729  						{
  1730  							Name:  regular1,
  1731  							Image: busyboxImage,
  1732  							Command: ExecCommand(regular1, execCommand{
  1733  								Delay:    600,
  1734  								ExitCode: 0,
  1735  							}),
  1736  						},
  1737  					},
  1738  				},
  1739  			}
  1740  
  1741  			preparePod(podSpec)
  1742  			var results containerOutputList
  1743  
  1744  			ginkgo.It("should continuously run Pod keeping it Pending", func() {
  1745  				client := e2epod.NewPodClient(f)
  1746  				podSpec = client.Create(context.TODO(), podSpec)
  1747  
  1748  				err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) {
  1749  					if pod.Status.Phase != v1.PodPending {
  1750  						return false, fmt.Errorf("pod should be in pending phase")
  1751  					}
  1752  					if len(pod.Status.InitContainerStatuses) < 1 {
  1753  						return false, nil
  1754  					}
  1755  					containerStatus := pod.Status.InitContainerStatuses[0]
  1756  					return containerStatus.RestartCount >= 3, nil
  1757  				})
  1758  				framework.ExpectNoError(err)
  1759  
  1760  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1761  				framework.ExpectNoError(err)
  1762  				results = parseOutput(context.TODO(), f, podSpec)
  1763  			})
  1764  			ginkgo.It("should have Init container restartCount greater than 0", func() {
  1765  				framework.ExpectNoError(results.HasRestarted(init1))
  1766  			})
  1767  			ginkgo.It("should not start restartable init container", func() {
  1768  				framework.ExpectNoError(results.DoesntStart(restartableInit1))
  1769  			})
  1770  		})
  1771  
  1772  		ginkgo.When("an Init container after restartable init container fails", ginkgo.Ordered, func() {
  1773  
  1774  			init1 := "init-1"
  1775  			restartableInit1 := "restartable-init-1"
  1776  			regular1 := "regular-1"
  1777  
  1778  			podSpec := &v1.Pod{
  1779  				ObjectMeta: metav1.ObjectMeta{
  1780  					Name: "restartable-init-container-fails-before-init-container",
  1781  				},
  1782  				Spec: v1.PodSpec{
  1783  					RestartPolicy: v1.RestartPolicyOnFailure,
  1784  					InitContainers: []v1.Container{
  1785  						{
  1786  							Name:  restartableInit1,
  1787  							Image: busyboxImage,
  1788  							Command: ExecCommand(restartableInit1, execCommand{
  1789  								Delay:    1,
  1790  								ExitCode: 1,
  1791  							}),
  1792  							RestartPolicy: &containerRestartPolicyAlways,
  1793  						},
  1794  						{
  1795  							Name:  init1,
  1796  							Image: busyboxImage,
  1797  							Command: ExecCommand(init1, execCommand{
  1798  								Delay:    1,
  1799  								ExitCode: 1,
  1800  							}),
  1801  						},
  1802  					},
  1803  					Containers: []v1.Container{
  1804  						{
  1805  							Name:  regular1,
  1806  							Image: busyboxImage,
  1807  							Command: ExecCommand(regular1, execCommand{
  1808  								Delay:    600,
  1809  								ExitCode: 0,
  1810  							}),
  1811  						},
  1812  					},
  1813  				},
  1814  			}
  1815  
  1816  			preparePod(podSpec)
  1817  			var results containerOutputList
  1818  
  1819  			ginkgo.It("should continuously run Pod keeping it Pending", func() {
  1820  				client := e2epod.NewPodClient(f)
  1821  				podSpec = client.Create(context.TODO(), podSpec)
  1822  
  1823  				err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) {
  1824  					if pod.Status.Phase != v1.PodPending {
  1825  						return false, fmt.Errorf("pod should be in pending phase")
  1826  					}
  1827  					if len(pod.Status.InitContainerStatuses) < 1 {
  1828  						return false, nil
  1829  					}
  1830  					containerStatus := pod.Status.InitContainerStatuses[0]
  1831  					return containerStatus.RestartCount >= 3, nil
  1832  				})
  1833  				framework.ExpectNoError(err)
  1834  
  1835  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1836  				framework.ExpectNoError(err)
  1837  				results = parseOutput(context.TODO(), f, podSpec)
  1838  			})
  1839  			ginkgo.It("should have Init container restartCount greater than 0", func() {
  1840  				framework.ExpectNoError(results.HasRestarted(init1))
  1841  			})
  1842  			// TODO: how will we be able to test it if restartable init container will never fail and there will be no termination log? Or will be?
  1843  			ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() {
  1844  				framework.ExpectNoError(results.RunTogether(restartableInit1, init1))
  1845  			})
  1846  		})
  1847  	})
  1848  
  1849  	ginkgo.When("using a restartable init container in a Pod with restartPolicy=Always", ginkgo.Ordered, func() {
  1850  		ginkgo.When("a restartable init container runs continuously", func() {
  1851  
  1852  			restartableInit1 := "restartable-init-1"
  1853  			regular1 := "regular-1"
  1854  
  1855  			podSpec := &v1.Pod{
  1856  				ObjectMeta: metav1.ObjectMeta{
  1857  					Name: "restartable-init-container-run-continuously",
  1858  				},
  1859  				Spec: v1.PodSpec{
  1860  					RestartPolicy: v1.RestartPolicyAlways,
  1861  					InitContainers: []v1.Container{
  1862  						{
  1863  							Name:  restartableInit1,
  1864  							Image: busyboxImage,
  1865  							Command: ExecCommand(restartableInit1, execCommand{
  1866  								Delay:    600,
  1867  								ExitCode: 0,
  1868  							}),
  1869  							RestartPolicy: &containerRestartPolicyAlways,
  1870  						},
  1871  					},
  1872  					Containers: []v1.Container{
  1873  						{
  1874  							Name:  regular1,
  1875  							Image: busyboxImage,
  1876  							Command: ExecCommand(regular1, execCommand{
  1877  								Delay:    1,
  1878  								ExitCode: 0,
  1879  							}),
  1880  						},
  1881  					},
  1882  				},
  1883  			}
  1884  
  1885  			preparePod(podSpec)
  1886  			var results containerOutputList
  1887  
  1888  			// regular container should exit at least once so we can get it's termination log
  1889  			// this test case is different from restartPolicy=Never
  1890  			ginkgo.It("should keep running a Pod continuously and produce log", func() { /* check the regular container restartCount > 0 */
  1891  				client := e2epod.NewPodClient(f)
  1892  				podSpec = client.Create(context.TODO(), podSpec)
  1893  
  1894  				err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 2, 2*time.Minute)
  1895  				framework.ExpectNoError(err)
  1896  
  1897  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  1898  				framework.ExpectNoError(err)
  1899  				results = parseOutput(context.TODO(), f, podSpec)
  1900  			})
  1901  
  1902  			ginkgo.It("should not restart a restartable init container", func() {
  1903  				framework.ExpectNoError(results.DoesntStartAfter(restartableInit1, regular1))
  1904  			})
  1905  			// this test case is different from restartPolicy=Never
  1906  			ginkgo.It("should start a regular container", func() {
  1907  				framework.ExpectNoError(results.HasRestarted(regular1))
  1908  			})
  1909  		})
  1910  
  1911  		ginkgo.When("a restartable init container fails to start because of a bad image", ginkgo.Ordered, func() {
  1912  
  1913  			restartableInit1 := "restartable-init-1"
  1914  			regular1 := "regular-1"
  1915  
  1916  			podSpec := &v1.Pod{
  1917  				ObjectMeta: metav1.ObjectMeta{
  1918  					Name: "restartable-init-runs-with-pod",
  1919  				},
  1920  				Spec: v1.PodSpec{
  1921  					RestartPolicy: v1.RestartPolicyNever,
  1922  					InitContainers: []v1.Container{
  1923  						{
  1924  							Name:  restartableInit1,
  1925  							Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage),
  1926  							Command: ExecCommand(restartableInit1, execCommand{
  1927  								Delay:    600,
  1928  								ExitCode: 0,
  1929  							}),
  1930  							RestartPolicy: &containerRestartPolicyAlways,
  1931  						},
  1932  					},
  1933  					Containers: []v1.Container{
  1934  						{
  1935  							Name:  regular1,
  1936  							Image: busyboxImage,
  1937  							Command: ExecCommand(regular1, execCommand{
  1938  								Delay:    1,
  1939  								ExitCode: 0,
  1940  							}),
  1941  						},
  1942  					},
  1943  				},
  1944  			}
  1945  
  1946  			preparePod(podSpec)
  1947  			var results containerOutputList
  1948  
  1949  			ginkgo.It("should continuously run Pod keeping it Pending and produce log", func() {
  1950  				client := e2epod.NewPodClient(f)
  1951  				podSpec = client.Create(context.TODO(), podSpec)
  1952  
  1953  				// restartable init container should be in image pull backoff
  1954  				err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart)
  1955  				framework.ExpectNoError(err)
  1956  
  1957  				podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{})
  1958  				framework.ExpectNoError(err)
  1959  				results = parseOutput(context.TODO(), f, podSpec)
  1960  			})
  1961  			ginkgo.It("should not start a restartable init container", func() {
  1962  				framework.ExpectNoError(results.DoesntStart(restartableInit1))
  1963  			})
  1964  			ginkgo.It("should not start a regular container", func() {
  1965  				framework.ExpectNoError(results.DoesntStart(regular1))
  1966  			})
  1967  		})
  1968  
  1969  		// TODO: add a test case similar to one above, but with startup probe never succeeding
  1970  
  1971  		ginkgo.When("a restartable init container starts and exits with exit code 0 continuously", ginkgo.Ordered, func() {
  1972  
  1973  			restartableInit1 := "restartable-init-1"
  1974  			init1 := "init-1"
  1975  			regular1 := "regular-1"
  1976  
  1977  			podSpec := &v1.Pod{
  1978  				ObjectMeta: metav1.ObjectMeta{
  1979  					Name: "restartable-init-container-exit-0-continuously",
  1980  				},
  1981  				Spec: v1.PodSpec{
  1982  					RestartPolicy: v1.RestartPolicyAlways,
  1983  					InitContainers: []v1.Container{
  1984  						{
  1985  							Name:  restartableInit1,
  1986  							Image: busyboxImage,
  1987  							Command: ExecCommand(restartableInit1, execCommand{
  1988  								Delay:    1,
  1989  								ExitCode: 0,
  1990  							}),
  1991  							RestartPolicy: &containerRestartPolicyAlways,
  1992  						},
  1993  						{
  1994  							Name:  init1,
  1995  							Image: busyboxImage,
  1996  							Command: ExecCommand(init1, execCommand{
  1997  								Delay:    5,
  1998  								ExitCode: 0,
  1999  							}),
  2000  						},
  2001  					},
  2002  					Containers: []v1.Container{
  2003  						{
  2004  							Name:  regular1,
  2005  							Image: busyboxImage,
  2006  							Command: ExecCommand(regular1, execCommand{
  2007  								Delay:    60,
  2008  								ExitCode: 0,
  2009  							}),
  2010  						},
  2011  					},
  2012  				},
  2013  			}
  2014  
  2015  			preparePod(podSpec)
  2016  			var results containerOutputList
  2017  
  2018  			ginkgo.It("should keep running a Pod continuously and produce log", func() { /* check the regular container restartCount > 0 */
  2019  				client := e2epod.NewPodClient(f)
  2020  				podSpec = client.Create(context.TODO(), podSpec)
  2021  
  2022  				err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 1, 2*time.Minute)
  2023  				framework.ExpectNoError(err)
  2024  
  2025  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  2026  				framework.ExpectNoError(err)
  2027  				results = parseOutput(context.TODO(), f, podSpec)
  2028  			})
  2029  			ginkgo.It("should restart a restartable init container before the regular container started", func() {
  2030  				framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1))
  2031  			})
  2032  			ginkgo.It("should restart a restartable init container after the regular container started", func() {
  2033  				framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1))
  2034  			})
  2035  			ginkgo.It("should start a regular container", func() {
  2036  				framework.ExpectNoError(results.Starts(regular1))
  2037  			})
  2038  		})
  2039  
  2040  		// this test case the same as for restartPolicy=Never
  2041  		ginkgo.When("a restartable init container starts and exits with exit code 1 continuously", ginkgo.Ordered, func() {
  2042  
  2043  			restartableInit1 := "restartable-init-1"
  2044  			init1 := "init-1"
  2045  			regular1 := "regular-1"
  2046  
  2047  			podSpec := &v1.Pod{
  2048  				ObjectMeta: metav1.ObjectMeta{
  2049  					Name: "restartable-init-container-exit-1-continuously",
  2050  				},
  2051  				Spec: v1.PodSpec{
  2052  					RestartPolicy: v1.RestartPolicyAlways,
  2053  					InitContainers: []v1.Container{
  2054  						{
  2055  							Name:  restartableInit1,
  2056  							Image: busyboxImage,
  2057  							Command: ExecCommand(restartableInit1, execCommand{
  2058  								Delay:    1,
  2059  								ExitCode: 1,
  2060  							}),
  2061  							RestartPolicy: &containerRestartPolicyAlways,
  2062  						},
  2063  						{
  2064  							Name:  init1,
  2065  							Image: busyboxImage,
  2066  							Command: ExecCommand(init1, execCommand{
  2067  								Delay:    5,
  2068  								ExitCode: 0,
  2069  							}),
  2070  						},
  2071  					},
  2072  					Containers: []v1.Container{
  2073  						{
  2074  							Name:  regular1,
  2075  							Image: busyboxImage,
  2076  							Command: ExecCommand(regular1, execCommand{
  2077  								Delay:    60,
  2078  								ExitCode: 0,
  2079  							}),
  2080  						},
  2081  					},
  2082  				},
  2083  			}
  2084  
  2085  			preparePod(podSpec)
  2086  			var results containerOutputList
  2087  
  2088  			ginkgo.It("should keep running a Pod continuously and produce log", func() { /* check the regular container restartCount > 0 */
  2089  				client := e2epod.NewPodClient(f)
  2090  				podSpec = client.Create(context.TODO(), podSpec)
  2091  
  2092  				err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 1, 2*time.Minute)
  2093  				framework.ExpectNoError(err)
  2094  
  2095  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  2096  				framework.ExpectNoError(err)
  2097  				results = parseOutput(context.TODO(), f, podSpec)
  2098  			})
  2099  			ginkgo.It("should restart a restartable init container before the regular container started", func() {
  2100  				framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1))
  2101  			})
  2102  			ginkgo.It("should restart a restartable init container after the regular container started", func() {
  2103  				framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1))
  2104  			})
  2105  			ginkgo.It("should start a regular container", func() {
  2106  				framework.ExpectNoError(results.Starts(regular1))
  2107  			})
  2108  		})
  2109  
  2110  		ginkgo.When("an Init container before restartable init container continuously fails", ginkgo.Ordered, func() {
  2111  
  2112  			init1 := "init-1"
  2113  			restartableInit1 := "restartable-init-1"
  2114  			regular1 := "regular-1"
  2115  
  2116  			podSpec := &v1.Pod{
  2117  				ObjectMeta: metav1.ObjectMeta{
  2118  					Name: "init-container-fails-before-restartable-init-starts",
  2119  				},
  2120  				Spec: v1.PodSpec{
  2121  					RestartPolicy: v1.RestartPolicyAlways,
  2122  					InitContainers: []v1.Container{
  2123  						{
  2124  							Name:  init1,
  2125  							Image: busyboxImage,
  2126  							Command: ExecCommand(init1, execCommand{
  2127  								Delay:    1,
  2128  								ExitCode: 1,
  2129  							}),
  2130  						},
  2131  						{
  2132  							Name:  restartableInit1,
  2133  							Image: busyboxImage,
  2134  							Command: ExecCommand(restartableInit1, execCommand{
  2135  								Delay:    600,
  2136  								ExitCode: 0,
  2137  							}),
  2138  							RestartPolicy: &containerRestartPolicyAlways,
  2139  						},
  2140  					},
  2141  					Containers: []v1.Container{
  2142  						{
  2143  							Name:  regular1,
  2144  							Image: busyboxImage,
  2145  							Command: ExecCommand(regular1, execCommand{
  2146  								Delay:    600,
  2147  								ExitCode: 0,
  2148  							}),
  2149  						},
  2150  					},
  2151  				},
  2152  			}
  2153  
  2154  			preparePod(podSpec)
  2155  			var results containerOutputList
  2156  
  2157  			ginkgo.It("should continuously run Pod keeping it Pending", func() {
  2158  				client := e2epod.NewPodClient(f)
  2159  				podSpec = client.Create(context.TODO(), podSpec)
  2160  
  2161  				err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) {
  2162  					if pod.Status.Phase != v1.PodPending {
  2163  						return false, fmt.Errorf("pod should be in pending phase")
  2164  					}
  2165  					if len(pod.Status.InitContainerStatuses) < 1 {
  2166  						return false, nil
  2167  					}
  2168  					containerStatus := pod.Status.InitContainerStatuses[0]
  2169  					return containerStatus.RestartCount >= 3, nil
  2170  				})
  2171  				framework.ExpectNoError(err)
  2172  
  2173  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  2174  				framework.ExpectNoError(err)
  2175  				results = parseOutput(context.TODO(), f, podSpec)
  2176  			})
  2177  			ginkgo.It("should have Init container restartCount greater than 0", func() {
  2178  				framework.ExpectNoError(results.HasRestarted(init1))
  2179  			})
  2180  			ginkgo.It("should not start restartable init container", func() {
  2181  				framework.ExpectNoError(results.DoesntStart(restartableInit1))
  2182  			})
  2183  		})
  2184  
  2185  		ginkgo.When("an Init container after restartable init container fails", ginkgo.Ordered, func() {
  2186  
  2187  			init1 := "init-1"
  2188  			restartableInit1 := "restartable-init-1"
  2189  			regular1 := "regular-1"
  2190  
  2191  			podSpec := &v1.Pod{
  2192  				ObjectMeta: metav1.ObjectMeta{
  2193  					Name: "restartable-init-container-fails-before-init-container",
  2194  				},
  2195  				Spec: v1.PodSpec{
  2196  					RestartPolicy: v1.RestartPolicyAlways,
  2197  					InitContainers: []v1.Container{
  2198  						{
  2199  							Name:  restartableInit1,
  2200  							Image: busyboxImage,
  2201  							Command: ExecCommand(restartableInit1, execCommand{
  2202  								Delay:    1,
  2203  								ExitCode: 1,
  2204  							}),
  2205  							RestartPolicy: &containerRestartPolicyAlways,
  2206  						},
  2207  						{
  2208  							Name:  init1,
  2209  							Image: busyboxImage,
  2210  							Command: ExecCommand(init1, execCommand{
  2211  								Delay:    1,
  2212  								ExitCode: 1,
  2213  							}),
  2214  						},
  2215  					},
  2216  					Containers: []v1.Container{
  2217  						{
  2218  							Name:  regular1,
  2219  							Image: busyboxImage,
  2220  							Command: ExecCommand(regular1, execCommand{
  2221  								Delay:    600,
  2222  								ExitCode: 0,
  2223  							}),
  2224  						},
  2225  					},
  2226  				},
  2227  			}
  2228  
  2229  			preparePod(podSpec)
  2230  			var results containerOutputList
  2231  
  2232  			ginkgo.It("should continuously run Pod keeping it Pending", func() {
  2233  				client := e2epod.NewPodClient(f)
  2234  				podSpec = client.Create(context.TODO(), podSpec)
  2235  
  2236  				err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) {
  2237  					if pod.Status.Phase != v1.PodPending {
  2238  						return false, fmt.Errorf("pod should be in pending phase")
  2239  					}
  2240  					if len(pod.Status.InitContainerStatuses) < 1 {
  2241  						return false, nil
  2242  					}
  2243  					containerStatus := pod.Status.InitContainerStatuses[0]
  2244  					return containerStatus.RestartCount >= 3, nil
  2245  				})
  2246  				framework.ExpectNoError(err)
  2247  
  2248  				podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
  2249  				framework.ExpectNoError(err)
  2250  				results = parseOutput(context.TODO(), f, podSpec)
  2251  			})
  2252  			ginkgo.It("should have Init container restartCount greater than 0", func() {
  2253  				framework.ExpectNoError(results.HasRestarted(init1))
  2254  			})
  2255  			// TODO: how will we be able to test it if restartable init container will never fail and there will be no termination log? Or will be?
  2256  			ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() {
  2257  				framework.ExpectNoError(results.RunTogether(restartableInit1, init1))
  2258  			})
  2259  		})
  2260  	})
  2261  
  2262  	ginkgo.It("should launch restartable init containers serially considering the startup probe", func() {
  2263  
  2264  		restartableInit1 := "restartable-init-1"
  2265  		restartableInit2 := "restartable-init-2"
  2266  		regular1 := "regular-1"
  2267  
  2268  		pod := &v1.Pod{
  2269  			ObjectMeta: metav1.ObjectMeta{
  2270  				Name: "restartable-init-containers-start-serially",
  2271  			},
  2272  			Spec: v1.PodSpec{
  2273  				RestartPolicy: v1.RestartPolicyNever,
  2274  				InitContainers: []v1.Container{
  2275  					{
  2276  						Name:  restartableInit1,
  2277  						Image: busyboxImage,
  2278  						Command: ExecCommand(restartableInit1, execCommand{
  2279  							StartDelay: 10,
  2280  							Delay:      600,
  2281  							ExitCode:   0,
  2282  						}),
  2283  						StartupProbe: &v1.Probe{
  2284  							ProbeHandler: v1.ProbeHandler{
  2285  								Exec: &v1.ExecAction{
  2286  									Command: []string{"test", "-f", "started"},
  2287  								},
  2288  							},
  2289  						},
  2290  						RestartPolicy: &containerRestartPolicyAlways,
  2291  					},
  2292  					{
  2293  						Name:  restartableInit2,
  2294  						Image: busyboxImage,
  2295  						Command: ExecCommand(restartableInit2, execCommand{
  2296  							StartDelay: 10,
  2297  							Delay:      600,
  2298  							ExitCode:   0,
  2299  						}),
  2300  						StartupProbe: &v1.Probe{
  2301  							ProbeHandler: v1.ProbeHandler{
  2302  								Exec: &v1.ExecAction{
  2303  									Command: []string{"test", "-f", "started"},
  2304  								},
  2305  							},
  2306  						},
  2307  						RestartPolicy: &containerRestartPolicyAlways,
  2308  					},
  2309  				},
  2310  				Containers: []v1.Container{
  2311  					{
  2312  						Name:  regular1,
  2313  						Image: busyboxImage,
  2314  						Command: ExecCommand(regular1, execCommand{
  2315  							Delay:    1,
  2316  							ExitCode: 0,
  2317  						}),
  2318  					},
  2319  				},
  2320  			},
  2321  		}
  2322  
  2323  		preparePod(pod)
  2324  
  2325  		client := e2epod.NewPodClient(f)
  2326  		pod = client.Create(context.TODO(), pod)
  2327  
  2328  		ginkgo.By("Waiting for the pod to finish")
  2329  		err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 5*time.Minute)
  2330  		framework.ExpectNoError(err)
  2331  
  2332  		pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{})
  2333  		framework.ExpectNoError(err)
  2334  		results := parseOutput(context.TODO(), f, pod)
  2335  
  2336  		ginkgo.By("Analyzing results")
  2337  		framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit2))
  2338  		framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1))
  2339  	})
  2340  
  2341  	ginkgo.It("should call the container's preStop hook and not launch next container if the restartable init container's startup probe fails", func() {
  2342  
  2343  		restartableInit1 := "restartable-init-1"
  2344  		regular1 := "regular-1"
  2345  
  2346  		pod := &v1.Pod{
  2347  			ObjectMeta: metav1.ObjectMeta{
  2348  				Name: "restartable-init-container-failed-startup",
  2349  			},
  2350  			Spec: v1.PodSpec{
  2351  				RestartPolicy: v1.RestartPolicyAlways,
  2352  				InitContainers: []v1.Container{
  2353  					{
  2354  						Name:  restartableInit1,
  2355  						Image: busyboxImage,
  2356  						Command: ExecCommand(restartableInit1, execCommand{
  2357  							Delay:              600,
  2358  							TerminationSeconds: 15,
  2359  							ExitCode:           0,
  2360  						}),
  2361  						StartupProbe: &v1.Probe{
  2362  							InitialDelaySeconds: 5,
  2363  							FailureThreshold:    1,
  2364  							ProbeHandler: v1.ProbeHandler{
  2365  								Exec: &v1.ExecAction{
  2366  									Command: []string{
  2367  										"sh",
  2368  										"-c",
  2369  										"exit 1",
  2370  									},
  2371  								},
  2372  							},
  2373  						},
  2374  						Lifecycle: &v1.Lifecycle{
  2375  							PreStop: &v1.LifecycleHandler{
  2376  								Exec: &v1.ExecAction{
  2377  									Command: ExecCommand(prefixedName(PreStopPrefix, restartableInit1), execCommand{
  2378  										Delay:         1,
  2379  										ExitCode:      0,
  2380  										ContainerName: restartableInit1,
  2381  									}),
  2382  								},
  2383  							},
  2384  						},
  2385  						RestartPolicy: &containerRestartPolicyAlways,
  2386  					},
  2387  				},
  2388  				Containers: []v1.Container{
  2389  					{
  2390  						Name:  regular1,
  2391  						Image: busyboxImage,
  2392  						Command: ExecCommand(regular1, execCommand{
  2393  							Delay:    1,
  2394  							ExitCode: 0,
  2395  						}),
  2396  					},
  2397  				},
  2398  			},
  2399  		}
  2400  
  2401  		preparePod(pod)
  2402  
  2403  		client := e2epod.NewPodClient(f)
  2404  		pod = client.Create(context.TODO(), pod)
  2405  
  2406  		ginkgo.By("Waiting for the restartable init container to restart")
  2407  		err := WaitForPodInitContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 2, 2*time.Minute)
  2408  		framework.ExpectNoError(err)
  2409  
  2410  		pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{})
  2411  		framework.ExpectNoError(err)
  2412  
  2413  		if pod.Status.Phase != v1.PodPending {
  2414  			framework.Failf("pod %q is not pending, it's %q", pod.Name, pod.Status.Phase)
  2415  		}
  2416  
  2417  		results := parseOutput(context.TODO(), f, pod)
  2418  
  2419  		ginkgo.By("Analyzing results")
  2420  		framework.ExpectNoError(results.RunTogether(restartableInit1, prefixedName(PreStopPrefix, restartableInit1)))
  2421  		framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, restartableInit1)))
  2422  		framework.ExpectNoError(results.Exits(restartableInit1))
  2423  		framework.ExpectNoError(results.DoesntStart(regular1))
  2424  	})
  2425  
  2426  	ginkgo.It("should call the container's preStop hook and start the next container if the restartable init container's liveness probe fails", func() {
  2427  
  2428  		restartableInit1 := "restartable-init-1"
  2429  		regular1 := "regular-1"
  2430  
  2431  		pod := &v1.Pod{
  2432  			ObjectMeta: metav1.ObjectMeta{
  2433  				Name: "restartable-init-container-failed-startup",
  2434  			},
  2435  			Spec: v1.PodSpec{
  2436  				RestartPolicy: v1.RestartPolicyAlways,
  2437  				InitContainers: []v1.Container{
  2438  					{
  2439  						Name:  restartableInit1,
  2440  						Image: busyboxImage,
  2441  						Command: ExecCommand(restartableInit1, execCommand{
  2442  							Delay:              600,
  2443  							TerminationSeconds: 15,
  2444  							ExitCode:           0,
  2445  						}),
  2446  						LivenessProbe: &v1.Probe{
  2447  							InitialDelaySeconds: 5,
  2448  							FailureThreshold:    1,
  2449  							ProbeHandler: v1.ProbeHandler{
  2450  								Exec: &v1.ExecAction{
  2451  									Command: []string{
  2452  										"sh",
  2453  										"-c",
  2454  										"exit 1",
  2455  									},
  2456  								},
  2457  							},
  2458  						},
  2459  						Lifecycle: &v1.Lifecycle{
  2460  							PreStop: &v1.LifecycleHandler{
  2461  								Exec: &v1.ExecAction{
  2462  									Command: ExecCommand(prefixedName(PreStopPrefix, restartableInit1), execCommand{
  2463  										Delay:         1,
  2464  										ExitCode:      0,
  2465  										ContainerName: restartableInit1,
  2466  									}),
  2467  								},
  2468  							},
  2469  						},
  2470  						RestartPolicy: &containerRestartPolicyAlways,
  2471  					},
  2472  				},
  2473  				Containers: []v1.Container{
  2474  					{
  2475  						Name:  regular1,
  2476  						Image: busyboxImage,
  2477  						Command: ExecCommand(regular1, execCommand{
  2478  							Delay:    1,
  2479  							ExitCode: 0,
  2480  						}),
  2481  					},
  2482  				},
  2483  			},
  2484  		}
  2485  
  2486  		preparePod(pod)
  2487  
  2488  		client := e2epod.NewPodClient(f)
  2489  		pod = client.Create(context.TODO(), pod)
  2490  
  2491  		ginkgo.By("Waiting for the restartable init container to restart")
  2492  		err := WaitForPodInitContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 2, 2*time.Minute)
  2493  		framework.ExpectNoError(err)
  2494  
  2495  		err = WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 1, 2*time.Minute)
  2496  		framework.ExpectNoError(err)
  2497  
  2498  		pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{})
  2499  		framework.ExpectNoError(err)
  2500  
  2501  		results := parseOutput(context.TODO(), f, pod)
  2502  
  2503  		ginkgo.By("Analyzing results")
  2504  		framework.ExpectNoError(results.RunTogether(restartableInit1, prefixedName(PreStopPrefix, restartableInit1)))
  2505  		framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, restartableInit1)))
  2506  		framework.ExpectNoError(results.Exits(restartableInit1))
  2507  		framework.ExpectNoError(results.Starts(regular1))
  2508  	})
  2509  
  2510  	ginkgo.It("should terminate sidecars in reverse order after all main containers have exited", func() {
  2511  		restartableInit1 := "restartable-init-1"
  2512  		restartableInit2 := "restartable-init-2"
  2513  		restartableInit3 := "restartable-init-3"
  2514  		regular1 := "regular-1"
  2515  
  2516  		pod := &v1.Pod{
  2517  			ObjectMeta: metav1.ObjectMeta{
  2518  				Name: "serialize-termination",
  2519  			},
  2520  			Spec: v1.PodSpec{
  2521  				RestartPolicy: v1.RestartPolicyNever,
  2522  				InitContainers: []v1.Container{
  2523  					{
  2524  						Name:          restartableInit1,
  2525  						Image:         busyboxImage,
  2526  						RestartPolicy: &containerRestartPolicyAlways,
  2527  						Command: ExecCommand(restartableInit1, execCommand{
  2528  							Delay:              60,
  2529  							TerminationSeconds: 5,
  2530  							ExitCode:           0,
  2531  						}),
  2532  					},
  2533  					{
  2534  						Name:          restartableInit2,
  2535  						Image:         busyboxImage,
  2536  						RestartPolicy: &containerRestartPolicyAlways,
  2537  						Command: ExecCommand(restartableInit2, execCommand{
  2538  							Delay:              60,
  2539  							TerminationSeconds: 5,
  2540  							ExitCode:           0,
  2541  						}),
  2542  					},
  2543  					{
  2544  						Name:          restartableInit3,
  2545  						Image:         busyboxImage,
  2546  						RestartPolicy: &containerRestartPolicyAlways,
  2547  						Command: ExecCommand(restartableInit3, execCommand{
  2548  							Delay:              60,
  2549  							TerminationSeconds: 5,
  2550  							ExitCode:           0,
  2551  						}),
  2552  					},
  2553  				},
  2554  				Containers: []v1.Container{
  2555  					{
  2556  						Name:  regular1,
  2557  						Image: busyboxImage,
  2558  						Command: ExecCommand(regular1, execCommand{
  2559  							Delay:    5,
  2560  							ExitCode: 0,
  2561  						}),
  2562  					},
  2563  				},
  2564  			},
  2565  		}
  2566  
  2567  		preparePod(pod)
  2568  
  2569  		client := e2epod.NewPodClient(f)
  2570  		pod = client.Create(context.TODO(), pod)
  2571  
  2572  		err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 5*time.Minute)
  2573  		framework.ExpectNoError(err)
  2574  
  2575  		pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{})
  2576  		framework.ExpectNoError(err)
  2577  
  2578  		results := parseOutput(context.TODO(), f, pod)
  2579  
  2580  		ginkgo.By("Analyzing results")
  2581  		framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit2))
  2582  		framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit3))
  2583  		framework.ExpectNoError(results.StartsBefore(restartableInit2, restartableInit3))
  2584  		framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1))
  2585  		framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1))
  2586  		framework.ExpectNoError(results.StartsBefore(restartableInit3, regular1))
  2587  
  2588  		// main containers exit first
  2589  		framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit1))
  2590  		framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit2))
  2591  		framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit3))
  2592  		// followed by sidecars in reverse order
  2593  		framework.ExpectNoError(results.ExitsBefore(restartableInit3, restartableInit2))
  2594  		framework.ExpectNoError(results.ExitsBefore(restartableInit2, restartableInit1))
  2595  	})
  2596  
  2597  	ginkgo.It("should terminate sidecars simultaneously if prestop doesn't exit", func() {
  2598  		restartableInit1 := "restartable-init-1"
  2599  		restartableInit2 := "restartable-init-2"
  2600  		restartableInit3 := "restartable-init-3"
  2601  		regular1 := "regular-1"
  2602  
  2603  		makePrestop := func(containerName string) *v1.Lifecycle {
  2604  			return &v1.Lifecycle{
  2605  				PreStop: &v1.LifecycleHandler{
  2606  					Exec: &v1.ExecAction{
  2607  						Command: ExecCommand(prefixedName(PreStopPrefix, containerName), execCommand{
  2608  							ExitCode:      0,
  2609  							ContainerName: containerName,
  2610  							LoopForever:   true,
  2611  						}),
  2612  					},
  2613  				},
  2614  			}
  2615  		}
  2616  
  2617  		pod := &v1.Pod{
  2618  			ObjectMeta: metav1.ObjectMeta{
  2619  				Name: "serialize-termination",
  2620  			},
  2621  			Spec: v1.PodSpec{
  2622  				RestartPolicy: v1.RestartPolicyNever,
  2623  				InitContainers: []v1.Container{
  2624  					{
  2625  						Name:          restartableInit1,
  2626  						Image:         busyboxImage,
  2627  						RestartPolicy: &containerRestartPolicyAlways,
  2628  						Command: ExecCommand(restartableInit1, execCommand{
  2629  							Delay:              60,
  2630  							TerminationSeconds: 5,
  2631  							ExitCode:           0,
  2632  						}),
  2633  						Lifecycle: makePrestop(restartableInit1),
  2634  					},
  2635  					{
  2636  						Name:          restartableInit2,
  2637  						Image:         busyboxImage,
  2638  						RestartPolicy: &containerRestartPolicyAlways,
  2639  						Command: ExecCommand(restartableInit2, execCommand{
  2640  							Delay:              60,
  2641  							TerminationSeconds: 5,
  2642  							ExitCode:           0,
  2643  						}),
  2644  						Lifecycle: makePrestop(restartableInit2),
  2645  					},
  2646  					{
  2647  						Name:          restartableInit3,
  2648  						Image:         busyboxImage,
  2649  						RestartPolicy: &containerRestartPolicyAlways,
  2650  						Command: ExecCommand(restartableInit3, execCommand{
  2651  							Delay:              60,
  2652  							TerminationSeconds: 5,
  2653  							ExitCode:           0,
  2654  						}),
  2655  						Lifecycle: makePrestop(restartableInit3),
  2656  					},
  2657  				},
  2658  				Containers: []v1.Container{
  2659  					{
  2660  						Name:  regular1,
  2661  						Image: busyboxImage,
  2662  						Command: ExecCommand(regular1, execCommand{
  2663  							Delay:    5,
  2664  							ExitCode: 0,
  2665  						}),
  2666  					},
  2667  				},
  2668  			},
  2669  		}
  2670  
  2671  		preparePod(pod)
  2672  
  2673  		client := e2epod.NewPodClient(f)
  2674  		pod = client.Create(context.TODO(), pod)
  2675  
  2676  		err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 5*time.Minute)
  2677  		framework.ExpectNoError(err)
  2678  
  2679  		pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{})
  2680  		framework.ExpectNoError(err)
  2681  
  2682  		results := parseOutput(context.TODO(), f, pod)
  2683  
  2684  		ginkgo.By("Analyzing results")
  2685  		framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit2))
  2686  		framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit3))
  2687  		framework.ExpectNoError(results.StartsBefore(restartableInit2, restartableInit3))
  2688  		framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1))
  2689  		framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1))
  2690  		framework.ExpectNoError(results.StartsBefore(restartableInit3, regular1))
  2691  
  2692  		ps1, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit1))
  2693  		framework.ExpectNoError(err)
  2694  		ps2, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit2))
  2695  		framework.ExpectNoError(err)
  2696  		ps3, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit3))
  2697  		framework.ExpectNoError(err)
  2698  
  2699  		ps1Last, err := results.TimeOfLastLoop(prefixedName(PreStopPrefix, restartableInit1))
  2700  		framework.ExpectNoError(err)
  2701  		ps2Last, err := results.TimeOfLastLoop(prefixedName(PreStopPrefix, restartableInit2))
  2702  		framework.ExpectNoError(err)
  2703  		ps3Last, err := results.TimeOfLastLoop(prefixedName(PreStopPrefix, restartableInit3))
  2704  		framework.ExpectNoError(err)
  2705  
  2706  		const simulToleration = 0.5
  2707  		// should all end together since they loop infinitely and exceed their grace period
  2708  		gomega.Expect(ps1Last-ps2Last).To(gomega.BeNumerically("~", 0, simulToleration),
  2709  			fmt.Sprintf("expected PostStart 1 & PostStart 2 to be killed at the same time, got %s", results))
  2710  		gomega.Expect(ps1Last-ps3Last).To(gomega.BeNumerically("~", 0, simulToleration),
  2711  			fmt.Sprintf("expected PostStart 1 & PostStart 3 to be killed at the same time, got %s", results))
  2712  		gomega.Expect(ps2Last-ps3Last).To(gomega.BeNumerically("~", 0, simulToleration),
  2713  			fmt.Sprintf("expected PostStart 2 & PostStart 3 to be killed at the same time, got %s", results))
  2714  
  2715  		// 30 seconds + 2 second minimum grace for the SIGKILL
  2716  		const lifetimeToleration = 1
  2717  		gomega.Expect(ps1Last-ps1).To(gomega.BeNumerically("~", 32, lifetimeToleration),
  2718  			fmt.Sprintf("expected PostStart 1 to live for ~32 seconds, got %s", results))
  2719  		gomega.Expect(ps2Last-ps2).To(gomega.BeNumerically("~", 32, lifetimeToleration),
  2720  			fmt.Sprintf("expected PostStart 2 to live for ~32 seconds, got %s", results))
  2721  		gomega.Expect(ps3Last-ps3).To(gomega.BeNumerically("~", 32, lifetimeToleration),
  2722  			fmt.Sprintf("expected PostStart 3 to live for ~32 seconds, got %s", results))
  2723  
  2724  	})
  2725  
  2726  	ginkgo.It("should call sidecar container PreStop hook simultaneously", func() {
  2727  		restartableInit1 := "restartable-init-1"
  2728  		restartableInit2 := "restartable-init-2"
  2729  		restartableInit3 := "restartable-init-3"
  2730  		regular1 := "regular-1"
  2731  
  2732  		makePrestop := func(containerName string) *v1.Lifecycle {
  2733  			return &v1.Lifecycle{
  2734  				PreStop: &v1.LifecycleHandler{
  2735  					Exec: &v1.ExecAction{
  2736  						Command: ExecCommand(prefixedName(PreStopPrefix, containerName), execCommand{
  2737  							Delay:         1,
  2738  							ExitCode:      0,
  2739  							ContainerName: containerName,
  2740  						}),
  2741  					},
  2742  				},
  2743  			}
  2744  		}
  2745  
  2746  		pod := &v1.Pod{
  2747  			ObjectMeta: metav1.ObjectMeta{
  2748  				Name: "serialize-termination-simul-prestop",
  2749  			},
  2750  			Spec: v1.PodSpec{
  2751  				RestartPolicy: v1.RestartPolicyNever,
  2752  				InitContainers: []v1.Container{
  2753  					{
  2754  						Name:          restartableInit1,
  2755  						Image:         busyboxImage,
  2756  						RestartPolicy: &containerRestartPolicyAlways,
  2757  						Command: ExecCommand(restartableInit1, execCommand{
  2758  							Delay:              60,
  2759  							TerminationSeconds: 5,
  2760  							ExitCode:           0,
  2761  						}),
  2762  						Lifecycle: makePrestop(restartableInit1),
  2763  					},
  2764  					{
  2765  						Name:          restartableInit2,
  2766  						Image:         busyboxImage,
  2767  						RestartPolicy: &containerRestartPolicyAlways,
  2768  						Command: ExecCommand(restartableInit2, execCommand{
  2769  							Delay:              60,
  2770  							TerminationSeconds: 5,
  2771  							ExitCode:           0,
  2772  						}),
  2773  						Lifecycle: makePrestop(restartableInit2),
  2774  					},
  2775  					{
  2776  						Name:          restartableInit3,
  2777  						Image:         busyboxImage,
  2778  						RestartPolicy: &containerRestartPolicyAlways,
  2779  						Command: ExecCommand(restartableInit3, execCommand{
  2780  							Delay:              60,
  2781  							TerminationSeconds: 5,
  2782  							ExitCode:           0,
  2783  						}),
  2784  						Lifecycle: makePrestop(restartableInit3),
  2785  					},
  2786  				},
  2787  				Containers: []v1.Container{
  2788  					{
  2789  						Name:  regular1,
  2790  						Image: busyboxImage,
  2791  						Command: ExecCommand(regular1, execCommand{
  2792  							Delay:    5,
  2793  							ExitCode: 0,
  2794  						}),
  2795  					},
  2796  				},
  2797  			},
  2798  		}
  2799  
  2800  		preparePod(pod)
  2801  
  2802  		client := e2epod.NewPodClient(f)
  2803  		pod = client.Create(context.TODO(), pod)
  2804  
  2805  		err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 5*time.Minute)
  2806  		framework.ExpectNoError(err)
  2807  
  2808  		pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{})
  2809  		framework.ExpectNoError(err)
  2810  
  2811  		results := parseOutput(context.TODO(), f, pod)
  2812  
  2813  		ginkgo.By("Analyzing results")
  2814  		framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit2))
  2815  		framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit3))
  2816  		framework.ExpectNoError(results.StartsBefore(restartableInit2, restartableInit3))
  2817  		framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1))
  2818  		framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1))
  2819  		framework.ExpectNoError(results.StartsBefore(restartableInit3, regular1))
  2820  
  2821  		// main containers exit first
  2822  		framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit1))
  2823  		framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit2))
  2824  		framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit3))
  2825  
  2826  		// followed by sidecars in reverse order
  2827  		framework.ExpectNoError(results.ExitsBefore(restartableInit3, restartableInit2))
  2828  		framework.ExpectNoError(results.ExitsBefore(restartableInit2, restartableInit1))
  2829  
  2830  		// and the pre-stop hooks should have been called simultaneously
  2831  		ps1, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit1))
  2832  		framework.ExpectNoError(err)
  2833  		ps2, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit2))
  2834  		framework.ExpectNoError(err)
  2835  		ps3, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit3))
  2836  		framework.ExpectNoError(err)
  2837  
  2838  		const toleration = 0.5
  2839  		gomega.Expect(ps1-ps2).To(gomega.BeNumerically("~", 0, toleration),
  2840  			fmt.Sprintf("expected PostStart 1 & PostStart 2 to start at the same time, got %s", results))
  2841  		gomega.Expect(ps1-ps3).To(gomega.BeNumerically("~", 0, toleration),
  2842  			fmt.Sprintf("expected PostStart 1 & PostStart 3 to start at the same time, got %s", results))
  2843  		gomega.Expect(ps2-ps3).To(gomega.BeNumerically("~", 0, toleration),
  2844  			fmt.Sprintf("expected PostStart 2 & PostStart 3 to start at the same time, got %s", results))
  2845  	})
  2846  
  2847  	ginkgo.It("should not hang in termination if terminated during initialization", func() {
  2848  		startInit := "start-init"
  2849  		restartableInit1 := "restartable-init-1"
  2850  		restartableInit2 := "restartable-init-2"
  2851  		restartableInit3 := "restartable-init-3"
  2852  		regular1 := "regular-1"
  2853  
  2854  		makePrestop := func(containerName string) *v1.Lifecycle {
  2855  			return &v1.Lifecycle{
  2856  				PreStop: &v1.LifecycleHandler{
  2857  					Exec: &v1.ExecAction{
  2858  						Command: ExecCommand(prefixedName(PreStopPrefix, containerName), execCommand{
  2859  							Delay:         1,
  2860  							ExitCode:      0,
  2861  							ContainerName: containerName,
  2862  						}),
  2863  					},
  2864  				},
  2865  			}
  2866  		}
  2867  
  2868  		pod := &v1.Pod{
  2869  			ObjectMeta: metav1.ObjectMeta{
  2870  				Name: "dont-hang-if-terminated-in-init",
  2871  			},
  2872  			Spec: v1.PodSpec{
  2873  				RestartPolicy: v1.RestartPolicyNever,
  2874  				InitContainers: []v1.Container{
  2875  					{
  2876  						Name:  startInit,
  2877  						Image: busyboxImage,
  2878  						Command: ExecCommand(startInit, execCommand{
  2879  							Delay:              300,
  2880  							TerminationSeconds: 0,
  2881  							ExitCode:           0,
  2882  						}),
  2883  					},
  2884  					{
  2885  						Name:          restartableInit1,
  2886  						Image:         busyboxImage,
  2887  						RestartPolicy: &containerRestartPolicyAlways,
  2888  						Command: ExecCommand(restartableInit1, execCommand{
  2889  							Delay:              60,
  2890  							TerminationSeconds: 5,
  2891  							ExitCode:           0,
  2892  						}),
  2893  						Lifecycle: makePrestop(restartableInit1),
  2894  					},
  2895  					{
  2896  						Name:          restartableInit2,
  2897  						Image:         busyboxImage,
  2898  						RestartPolicy: &containerRestartPolicyAlways,
  2899  						Command: ExecCommand(restartableInit2, execCommand{
  2900  							Delay:              60,
  2901  							TerminationSeconds: 5,
  2902  							ExitCode:           0,
  2903  						}),
  2904  						Lifecycle: makePrestop(restartableInit2),
  2905  					},
  2906  					{
  2907  						Name:          restartableInit3,
  2908  						Image:         busyboxImage,
  2909  						RestartPolicy: &containerRestartPolicyAlways,
  2910  						Command: ExecCommand(restartableInit3, execCommand{
  2911  							Delay:              60,
  2912  							TerminationSeconds: 5,
  2913  							ExitCode:           0,
  2914  						}),
  2915  						Lifecycle: makePrestop(restartableInit3),
  2916  					},
  2917  				},
  2918  				Containers: []v1.Container{
  2919  					{
  2920  						Name:  regular1,
  2921  						Image: busyboxImage,
  2922  						Command: ExecCommand(regular1, execCommand{
  2923  							Delay:    5,
  2924  							ExitCode: 0,
  2925  						}),
  2926  					},
  2927  				},
  2928  			},
  2929  		}
  2930  
  2931  		preparePod(pod)
  2932  
  2933  		client := e2epod.NewPodClient(f)
  2934  		pod = client.Create(context.TODO(), pod)
  2935  
  2936  		err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, "pod pending and init running", 2*time.Minute, func(pod *v1.Pod) (bool, error) {
  2937  			if pod.Status.Phase != v1.PodPending {
  2938  				return false, fmt.Errorf("pod should be in pending phase")
  2939  			}
  2940  			if len(pod.Status.InitContainerStatuses) < 1 {
  2941  				return false, nil
  2942  			}
  2943  			containerStatus := pod.Status.InitContainerStatuses[0]
  2944  			return *containerStatus.Started && containerStatus.State.Running != nil, nil
  2945  		})
  2946  		framework.ExpectNoError(err)
  2947  
  2948  		// the init container is running, so we stop the pod before the sidecars even start
  2949  		start := time.Now()
  2950  		grace := int64(3)
  2951  		ginkgo.By("deleting the pod")
  2952  		err = client.Delete(context.TODO(), pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &grace})
  2953  		framework.ExpectNoError(err)
  2954  		ginkgo.By("waiting for the pod to disappear")
  2955  		err = e2epod.WaitForPodNotFoundInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 120*time.Second)
  2956  		framework.ExpectNoError(err)
  2957  
  2958  		buffer := int64(2)
  2959  		deleteTime := time.Since(start).Seconds()
  2960  		// should delete quickly and not try to start/wait on any sidecars since they never started
  2961  		gomega.Expect(deleteTime).To(gomega.BeNumerically("<", grace+buffer), fmt.Sprintf("should delete in < %d seconds, took %f", grace+buffer, deleteTime))
  2962  	})
  2963  })
  2964  
  2965  var _ = SIGDescribe(nodefeature.SidecarContainers, framework.WithSerial(), "Containers Lifecycle", func() {
  2966  	f := framework.NewDefaultFramework("containers-lifecycle-test-serial")
  2967  	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
  2968  
  2969  	ginkgo.It("should restart the containers in right order after the node reboot", func(ctx context.Context) {
  2970  		init1 := "init-1"
  2971  		restartableInit2 := "restartable-init-2"
  2972  		init3 := "init-3"
  2973  		regular1 := "regular-1"
  2974  
  2975  		podLabels := map[string]string{
  2976  			"test":      "containers-lifecycle-test-serial",
  2977  			"namespace": f.Namespace.Name,
  2978  		}
  2979  		pod := &v1.Pod{
  2980  			ObjectMeta: metav1.ObjectMeta{
  2981  				Name:   "initialized-pod",
  2982  				Labels: podLabels,
  2983  			},
  2984  			Spec: v1.PodSpec{
  2985  				RestartPolicy: v1.RestartPolicyAlways,
  2986  				InitContainers: []v1.Container{
  2987  					{
  2988  						Name:  init1,
  2989  						Image: busyboxImage,
  2990  						Command: ExecCommand(init1, execCommand{
  2991  							Delay:    5,
  2992  							ExitCode: 0,
  2993  						}),
  2994  					},
  2995  					{
  2996  						Name:  restartableInit2,
  2997  						Image: busyboxImage,
  2998  						Command: ExecCommand(restartableInit2, execCommand{
  2999  							Delay:    300,
  3000  							ExitCode: 0,
  3001  						}),
  3002  						RestartPolicy: &containerRestartPolicyAlways,
  3003  					},
  3004  					{
  3005  						Name:  init3,
  3006  						Image: busyboxImage,
  3007  						Command: ExecCommand(init3, execCommand{
  3008  							Delay:    5,
  3009  							ExitCode: 0,
  3010  						}),
  3011  					},
  3012  				},
  3013  				Containers: []v1.Container{
  3014  					{
  3015  						Name:  regular1,
  3016  						Image: busyboxImage,
  3017  						Command: ExecCommand(regular1, execCommand{
  3018  							Delay:    300,
  3019  							ExitCode: 0,
  3020  						}),
  3021  					},
  3022  				},
  3023  			},
  3024  		}
  3025  		preparePod(pod)
  3026  
  3027  		client := e2epod.NewPodClient(f)
  3028  		pod = client.Create(ctx, pod)
  3029  		ginkgo.By("Waiting for the pod to be initialized and run")
  3030  		err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
  3031  		framework.ExpectNoError(err)
  3032  
  3033  		ginkgo.By("Getting the current pod sandbox ID")
  3034  		rs, _, err := getCRIClient()
  3035  		framework.ExpectNoError(err)
  3036  
  3037  		sandboxes, err := rs.ListPodSandbox(ctx, &runtimeapi.PodSandboxFilter{
  3038  			LabelSelector: podLabels,
  3039  		})
  3040  		framework.ExpectNoError(err)
  3041  		gomega.Expect(sandboxes).To(gomega.HaveLen(1))
  3042  		podSandboxID := sandboxes[0].Id
  3043  
  3044  		ginkgo.By("Stopping the kubelet")
  3045  		restartKubelet := stopKubelet()
  3046  		gomega.Eventually(ctx, func() bool {
  3047  			return kubeletHealthCheck(kubeletHealthCheckURL)
  3048  		}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse())
  3049  
  3050  		ginkgo.By("Stopping the pod sandbox to simulate the node reboot")
  3051  		err = rs.StopPodSandbox(ctx, podSandboxID)
  3052  		framework.ExpectNoError(err)
  3053  
  3054  		ginkgo.By("Restarting the kubelet")
  3055  		restartKubelet()
  3056  		gomega.Eventually(ctx, func() bool {
  3057  			return kubeletHealthCheck(kubeletHealthCheckURL)
  3058  		}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue())
  3059  
  3060  		ginkgo.By("Waiting for the pod to be re-initialized and run")
  3061  		err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "re-initialized", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) {
  3062  			if pod.Status.ContainerStatuses[0].RestartCount < 1 {
  3063  				return false, nil
  3064  			}
  3065  			if pod.Status.Phase != v1.PodRunning {
  3066  				return false, nil
  3067  			}
  3068  			return true, nil
  3069  		})
  3070  		framework.ExpectNoError(err)
  3071  
  3072  		ginkgo.By("Parsing results")
  3073  		pod, err = client.Get(ctx, pod.Name, metav1.GetOptions{})
  3074  		framework.ExpectNoError(err)
  3075  		results := parseOutput(context.TODO(), f, pod)
  3076  
  3077  		ginkgo.By("Analyzing results")
  3078  		init1Started, err := results.FindIndex(init1, "Started", 0)
  3079  		framework.ExpectNoError(err)
  3080  		restartableInit2Started, err := results.FindIndex(restartableInit2, "Started", 0)
  3081  		framework.ExpectNoError(err)
  3082  		init3Started, err := results.FindIndex(init3, "Started", 0)
  3083  		framework.ExpectNoError(err)
  3084  		regular1Started, err := results.FindIndex(regular1, "Started", 0)
  3085  		framework.ExpectNoError(err)
  3086  
  3087  		init1Restarted, err := results.FindIndex(init1, "Started", init1Started+1)
  3088  		framework.ExpectNoError(err)
  3089  		restartableInit2Restarted, err := results.FindIndex(restartableInit2, "Started", restartableInit2Started+1)
  3090  		framework.ExpectNoError(err)
  3091  		init3Restarted, err := results.FindIndex(init3, "Started", init3Started+1)
  3092  		framework.ExpectNoError(err)
  3093  		regular1Restarted, err := results.FindIndex(regular1, "Started", regular1Started+1)
  3094  		framework.ExpectNoError(err)
  3095  
  3096  		framework.ExpectNoError(init1Started.IsBefore(restartableInit2Started))
  3097  		framework.ExpectNoError(restartableInit2Started.IsBefore(init3Started))
  3098  		framework.ExpectNoError(init3Started.IsBefore(regular1Started))
  3099  
  3100  		framework.ExpectNoError(init1Restarted.IsBefore(restartableInit2Restarted))
  3101  		framework.ExpectNoError(restartableInit2Restarted.IsBefore(init3Restarted))
  3102  		framework.ExpectNoError(init3Restarted.IsBefore(regular1Restarted))
  3103  	})
  3104  })
  3105  

View as plain text