pod_conditions_test.go

Documentation: k8s.io/kubernetes/test/e2e_node

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package e2enode
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strconv"
    23  	"strings"
    24  	"time"
    25  
    26  	v1 "k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	admissionapi "k8s.io/pod-security-admission/api"
    29  
    30  	"k8s.io/apimachinery/pkg/fields"
    31  	"k8s.io/apimachinery/pkg/util/uuid"
    32  	"k8s.io/kubernetes/pkg/kubelet/events"
    33  	"k8s.io/kubernetes/test/e2e/feature"
    34  	"k8s.io/kubernetes/test/e2e/framework"
    35  	e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
    36  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    37  	testutils "k8s.io/kubernetes/test/utils"
    38  	imageutils "k8s.io/kubernetes/test/utils/image"
    39  
    40  	"k8s.io/kubernetes/pkg/features"
    41  	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
    42  
    43  	"github.com/onsi/ginkgo/v2"
    44  	"github.com/onsi/gomega"
    45  )
    46  
    47  var _ = SIGDescribe("Pod conditions managed by Kubelet", func() {
    48  	f := framework.NewDefaultFramework("pod-conditions")
    49  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
    50  
    51  	f.Context("including PodReadyToStartContainers condition", f.WithSerial(), feature.PodReadyToStartContainersCondition, func() {
    52  		tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
    53  			initialConfig.FeatureGates = map[string]bool{
    54  				string(features.PodReadyToStartContainersCondition): true,
    55  			}
    56  		})
    57  		ginkgo.It("a pod without init containers should report all conditions set in expected order after the pod is up", runPodReadyConditionsTest(f, false, true))
    58  		ginkgo.It("a pod with init containers should report all conditions set in expected order after the pod is up", runPodReadyConditionsTest(f, true, true))
    59  		ginkgo.It("a pod failing to mount volumes and without init containers should report scheduled and initialized conditions set", runPodFailingConditionsTest(f, false, true))
    60  		ginkgo.It("a pod failing to mount volumes and with init containers should report just the scheduled condition set", runPodFailingConditionsTest(f, true, true))
    61  		cleanupPods(f)
    62  	})
    63  
    64  	ginkgo.Context("without PodReadyToStartContainersCondition condition", func() {
    65  		ginkgo.It("a pod without init containers should report all conditions set in expected order after the pod is up", runPodReadyConditionsTest(f, false, false))
    66  		ginkgo.It("a pod with init containers should report all conditions set in expected order after the pod is up", runPodReadyConditionsTest(f, true, false))
    67  		ginkgo.It("a pod failing to mount volumes and without init containers should report scheduled and initialized conditions set", runPodFailingConditionsTest(f, false, false))
    68  		ginkgo.It("a pod failing to mount volumes and with init containers should report just the scheduled condition set", runPodFailingConditionsTest(f, true, false))
    69  		cleanupPods(f)
    70  	})
    71  })
    72  
    73  func runPodFailingConditionsTest(f *framework.Framework, hasInitContainers, checkPodReadyToStart bool) func(ctx context.Context) {
    74  	return func(ctx context.Context) {
    75  		ginkgo.By("creating a pod whose sandbox creation is blocked due to a missing volume")
    76  
    77  		p := webserverPodSpec("pod-"+string(uuid.NewUUID()), "web1", "init1", hasInitContainers)
    78  		p.Spec.Volumes = []v1.Volume{
    79  			{
    80  				Name: "cm",
    81  				VolumeSource: v1.VolumeSource{
    82  					ConfigMap: &v1.ConfigMapVolumeSource{
    83  						LocalObjectReference: v1.LocalObjectReference{Name: "does-not-exist"},
    84  					},
    85  				},
    86  			},
    87  		}
    88  		p.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{
    89  			{
    90  				Name:      "cm",
    91  				MountPath: "/config",
    92  			},
    93  		}
    94  
    95  		p = e2epod.NewPodClient(f).Create(ctx, p)
    96  
    97  		ginkgo.By("waiting until kubelet has started trying to set up the pod and started to fail")
    98  
    99  		eventSelector := fields.Set{
   100  			"involvedObject.kind":      "Pod",
   101  			"involvedObject.name":      p.Name,
   102  			"involvedObject.namespace": f.Namespace.Name,
   103  			"reason":                   events.FailedMountVolume,
   104  		}.AsSelector().String()
   105  		framework.ExpectNoError(e2eevents.WaitTimeoutForEvent(ctx, f.ClientSet, f.Namespace.Name, eventSelector, "MountVolume.SetUp failed for volume", framework.PodEventTimeout))
   106  
   107  		p, err := e2epod.NewPodClient(f).Get(ctx, p.Name, metav1.GetOptions{})
   108  		framework.ExpectNoError(err)
   109  
   110  		ginkgo.By("checking pod condition for a pod whose sandbox creation is blocked")
   111  
   112  		scheduledTime, err := getTransitionTimeForPodConditionWithStatus(p, v1.PodScheduled, true)
   113  		framework.ExpectNoError(err)
   114  
   115  		// Verify PodReadyToStartContainers is not set (since sandboxcreation is blocked)
   116  		if checkPodReadyToStart {
   117  			_, err := getTransitionTimeForPodConditionWithStatus(p, v1.PodReadyToStartContainers, false)
   118  			framework.ExpectNoError(err)
   119  		}
   120  
   121  		if hasInitContainers {
   122  			// Verify PodInitialized is not set if init containers are present (since sandboxcreation is blocked)
   123  			_, err := getTransitionTimeForPodConditionWithStatus(p, v1.PodInitialized, false)
   124  			framework.ExpectNoError(err)
   125  		} else {
   126  			// Verify PodInitialized is set if init containers are not present (since without init containers, it gets set very early)
   127  			initializedTime, err := getTransitionTimeForPodConditionWithStatus(p, v1.PodInitialized, true)
   128  			framework.ExpectNoError(err)
   129  			gomega.Expect(initializedTime.Before(scheduledTime)).NotTo(gomega.BeTrue(), fmt.Sprintf("pod without init containers is initialized at: %v which is before pod scheduled at: %v", initializedTime, scheduledTime))
   130  		}
   131  
   132  		// Verify ContainersReady is not set (since sandboxcreation is blocked)
   133  		_, err = getTransitionTimeForPodConditionWithStatus(p, v1.ContainersReady, false)
   134  		framework.ExpectNoError(err)
   135  		// Verify PodReady is not set (since sandboxcreation is blocked)
   136  		_, err = getTransitionTimeForPodConditionWithStatus(p, v1.PodReady, false)
   137  		framework.ExpectNoError(err)
   138  
   139  		// this testcase is creating the missing volume that unblock the pod above,
   140  		// and check PodReadyToStartContainer is setting correctly.
   141  		ginkgo.By("checking pod condition for a pod when volumes source is created")
   142  
   143  		configmap := v1.ConfigMap{
   144  			ObjectMeta: metav1.ObjectMeta{
   145  				Name: "cm-that-unblock-pod-condition",
   146  			},
   147  			Data: map[string]string{
   148  				"key": "value",
   149  			},
   150  			BinaryData: map[string][]byte{
   151  				"binaryKey": []byte("value"),
   152  			},
   153  		}
   154  
   155  		_, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, &configmap, metav1.CreateOptions{})
   156  		framework.ExpectNoError(err)
   157  
   158  		defer func() {
   159  			err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, "cm-that-unblock-pod-condition", metav1.DeleteOptions{})
   160  			framework.ExpectNoError(err, "unable to delete configmap")
   161  		}()
   162  
   163  		p2 := webserverPodSpec("pod2-"+string(uuid.NewUUID()), "web2", "init2", hasInitContainers)
   164  		p2.Spec.Volumes = []v1.Volume{
   165  			{
   166  				Name: "cm-2",
   167  				VolumeSource: v1.VolumeSource{
   168  					ConfigMap: &v1.ConfigMapVolumeSource{
   169  						LocalObjectReference: v1.LocalObjectReference{Name: "cm-that-unblock-pod-condition"},
   170  					},
   171  				},
   172  			},
   173  		}
   174  		p2.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{
   175  			{
   176  				Name:      "cm-2",
   177  				MountPath: "/config",
   178  			},
   179  		}
   180  
   181  		p2 = e2epod.NewPodClient(f).Create(ctx, p2)
   182  		framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(ctx, f.ClientSet, p2.Name, p2.Namespace, framework.PodStartTimeout))
   183  
   184  		p2, err = e2epod.NewPodClient(f).Get(ctx, p2.Name, metav1.GetOptions{})
   185  		framework.ExpectNoError(err)
   186  
   187  		_, err = getTransitionTimeForPodConditionWithStatus(p2, v1.PodScheduled, true)
   188  		framework.ExpectNoError(err)
   189  
   190  		_, err = getTransitionTimeForPodConditionWithStatus(p2, v1.PodInitialized, true)
   191  		framework.ExpectNoError(err)
   192  
   193  		// Verify PodReadyToStartContainers is set (since sandboxcreation is unblocked)
   194  		if checkPodReadyToStart {
   195  			_, err = getTransitionTimeForPodConditionWithStatus(p2, v1.PodReadyToStartContainers, true)
   196  			framework.ExpectNoError(err)
   197  		}
   198  	}
   199  }
   200  
   201  func runPodReadyConditionsTest(f *framework.Framework, hasInitContainers, checkPodReadyToStart bool) func(ctx context.Context) {
   202  	return func(ctx context.Context) {
   203  		ginkgo.By("creating a pod that successfully comes up in a ready/running state")
   204  
   205  		p := e2epod.NewPodClient(f).Create(ctx, webserverPodSpec("pod-"+string(uuid.NewUUID()), "web1", "init1", hasInitContainers))
   206  		framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(ctx, f.ClientSet, p.Name, f.Namespace.Name, framework.PodStartTimeout))
   207  
   208  		p, err := e2epod.NewPodClient(f).Get(ctx, p.Name, metav1.GetOptions{})
   209  		framework.ExpectNoError(err)
   210  		isReady, err := testutils.PodRunningReady(p)
   211  		framework.ExpectNoError(err)
   212  		if !isReady {
   213  			framework.Failf("pod %q should be ready", p.Name)
   214  		}
   215  
   216  		ginkgo.By("checking order of pod condition transitions for a pod with no container/sandbox restarts")
   217  
   218  		scheduledTime, err := getTransitionTimeForPodConditionWithStatus(p, v1.PodScheduled, true)
   219  		framework.ExpectNoError(err)
   220  		initializedTime, err := getTransitionTimeForPodConditionWithStatus(p, v1.PodInitialized, true)
   221  		framework.ExpectNoError(err)
   222  
   223  		condBeforeContainersReadyTransitionTime := initializedTime
   224  		errSubstrIfContainersReadyTooEarly := "is initialized"
   225  		if checkPodReadyToStart {
   226  			readyToStartContainersTime, err := getTransitionTimeForPodConditionWithStatus(p, v1.PodReadyToStartContainers, true)
   227  			framework.ExpectNoError(err)
   228  
   229  			if hasInitContainers {
   230  				// With init containers, verify the sequence of conditions is: Scheduled => PodReadyToStartContainers => Initialized
   231  				gomega.Expect(readyToStartContainersTime.Before(scheduledTime)).ToNot(gomega.BeTrue(), fmt.Sprintf("pod with init containers is initialized at: %v which is before pod has ready to start at: %v", initializedTime, readyToStartContainersTime))
   232  				gomega.Expect(initializedTime.Before(readyToStartContainersTime)).ToNot(gomega.BeTrue(), fmt.Sprintf("pod with init containers is initialized at: %v which is before pod has ready to start at: %v", initializedTime, readyToStartContainersTime))
   233  			} else {
   234  				// Without init containers, verify the sequence of conditions is: Scheduled => Initialized => PodReadyToStartContainers
   235  				condBeforeContainersReadyTransitionTime = readyToStartContainersTime
   236  				errSubstrIfContainersReadyTooEarly = "ready to start"
   237  				gomega.Expect(initializedTime.Before(scheduledTime)).NotTo(gomega.BeTrue(), fmt.Sprintf("pod without init containers initialized at: %v which is before pod scheduled at: %v", initializedTime, scheduledTime))
   238  				gomega.Expect(readyToStartContainersTime.Before(initializedTime)).NotTo(gomega.BeTrue(), fmt.Sprintf("pod without init containers has ready to start at: %v which is before pod is initialized at: %v", readyToStartContainersTime, initializedTime))
   239  			}
   240  		} else {
   241  			// In the absence of PodHasReadyToStartContainers feature disabled, verify the sequence is: Scheduled => Initialized
   242  			gomega.Expect(initializedTime.Before(scheduledTime)).NotTo(gomega.BeTrue(), fmt.Sprintf("pod initialized at: %v which is before pod scheduled at: %v", initializedTime, scheduledTime))
   243  		}
   244  		// Verify the next condition to get set is ContainersReady
   245  		containersReadyTime, err := getTransitionTimeForPodConditionWithStatus(p, v1.ContainersReady, true)
   246  		framework.ExpectNoError(err)
   247  		gomega.Expect(containersReadyTime.Before(condBeforeContainersReadyTransitionTime)).NotTo(gomega.BeTrue(), fmt.Sprintf("containers ready at: %v which is before pod %s: %v", containersReadyTime, errSubstrIfContainersReadyTooEarly, initializedTime))
   248  
   249  		// Verify ContainersReady => PodReady
   250  		podReadyTime, err := getTransitionTimeForPodConditionWithStatus(p, v1.PodReady, true)
   251  		framework.ExpectNoError(err)
   252  		gomega.Expect(podReadyTime.Before(containersReadyTime)).NotTo(gomega.BeTrue(), fmt.Sprintf("pod ready at: %v which is before pod containers ready at: %v", podReadyTime, containersReadyTime))
   253  	}
   254  }
   255  
   256  func getTransitionTimeForPodConditionWithStatus(pod *v1.Pod, condType v1.PodConditionType, expectedStatus bool) (time.Time, error) {
   257  	for _, cond := range pod.Status.Conditions {
   258  		if cond.Type == condType {
   259  			if strings.EqualFold(string(cond.Status), strconv.FormatBool(expectedStatus)) {
   260  				return cond.LastTransitionTime.Time, nil
   261  			}
   262  			return time.Time{}, fmt.Errorf("condition: %s found for pod but status: %s did not match expected status: %s", condType, cond.Status, strconv.FormatBool(expectedStatus))
   263  		}
   264  	}
   265  	return time.Time{}, fmt.Errorf("condition: %s not found for pod", condType)
   266  }
   267  
   268  func webserverPodSpec(podName, containerName, initContainerName string, addInitContainer bool) *v1.Pod {
   269  	p := &v1.Pod{
   270  		ObjectMeta: metav1.ObjectMeta{
   271  			Name: podName,
   272  		},
   273  		Spec: v1.PodSpec{
   274  			Containers: []v1.Container{
   275  				{
   276  					Name:  containerName,
   277  					Image: imageutils.GetE2EImage(imageutils.Agnhost),
   278  					Args:  []string{"test-webserver"},
   279  				},
   280  			},
   281  		},
   282  	}
   283  	if addInitContainer {
   284  		p.Spec.InitContainers = []v1.Container{
   285  			{
   286  				Name:    initContainerName,
   287  				Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   288  				Command: []string{"sh", "-c", "sleep 5s"},
   289  			},
   290  		}
   291  	}
   292  	return p
   293  }
   294
View as plain text