node_lease.go

Documentation: k8s.io/kubernetes/test/e2e/common/node

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package node
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	coordinationv1 "k8s.io/api/coordination/v1"
    25  	v1 "k8s.io/api/core/v1"
    26  	apiequality "k8s.io/apimachinery/pkg/api/equality"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/apimachinery/pkg/util/wait"
    29  	clientset "k8s.io/client-go/kubernetes"
    30  	admissionapi "k8s.io/pod-security-admission/api"
    31  
    32  	"k8s.io/kubernetes/test/e2e/framework"
    33  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    34  	testutils "k8s.io/kubernetes/test/utils"
    35  
    36  	"github.com/google/go-cmp/cmp"
    37  	"github.com/onsi/ginkgo/v2"
    38  	"github.com/onsi/gomega"
    39  )
    40  
    41  var _ = SIGDescribe("NodeLease", func() {
    42  	var nodeName string
    43  	f := framework.NewDefaultFramework("node-lease-test")
    44  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    45  
    46  	ginkgo.BeforeEach(func(ctx context.Context) {
    47  		node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
    48  		framework.ExpectNoError(err)
    49  		nodeName = node.Name
    50  	})
    51  
    52  	ginkgo.Context("NodeLease", func() {
    53  		ginkgo.It("the kubelet should create and update a lease in the kube-node-lease namespace", func(ctx context.Context) {
    54  			leaseClient := f.ClientSet.CoordinationV1().Leases(v1.NamespaceNodeLease)
    55  			var (
    56  				err   error
    57  				lease *coordinationv1.Lease
    58  			)
    59  			ginkgo.By("check that lease for this Kubelet exists in the kube-node-lease namespace")
    60  			gomega.Eventually(ctx, func() error {
    61  				lease, err = leaseClient.Get(ctx, nodeName, metav1.GetOptions{})
    62  				if err != nil {
    63  					return err
    64  				}
    65  				return nil
    66  			}, 5*time.Minute, 5*time.Second).Should(gomega.BeNil())
    67  			// check basic expectations for the lease
    68  			framework.ExpectNoError(expectLease(lease, nodeName))
    69  
    70  			ginkgo.By("check that node lease is updated at least once within the lease duration")
    71  			gomega.Eventually(ctx, func() error {
    72  				newLease, err := leaseClient.Get(ctx, nodeName, metav1.GetOptions{})
    73  				if err != nil {
    74  					return err
    75  				}
    76  				// check basic expectations for the latest lease
    77  				if err := expectLease(newLease, nodeName); err != nil {
    78  					return err
    79  				}
    80  				// check that RenewTime has been updated on the latest lease
    81  				newTime := (*newLease.Spec.RenewTime).Time
    82  				oldTime := (*lease.Spec.RenewTime).Time
    83  				if !newTime.After(oldTime) {
    84  					return fmt.Errorf("new lease has time %v, which is not after old lease time %v", newTime, oldTime)
    85  				}
    86  				return nil
    87  			}, time.Duration(*lease.Spec.LeaseDurationSeconds)*time.Second,
    88  				time.Duration(*lease.Spec.LeaseDurationSeconds/4)*time.Second).Should(gomega.Succeed())
    89  		})
    90  
    91  		ginkgo.It("should have OwnerReferences set", func(ctx context.Context) {
    92  			leaseClient := f.ClientSet.CoordinationV1().Leases(v1.NamespaceNodeLease)
    93  			var (
    94  				err       error
    95  				leaseList *coordinationv1.LeaseList
    96  			)
    97  			gomega.Eventually(ctx, func() error {
    98  				leaseList, err = leaseClient.List(ctx, metav1.ListOptions{})
    99  				if err != nil {
   100  					return err
   101  				}
   102  				return nil
   103  			}, 5*time.Minute, 5*time.Second).Should(gomega.BeNil())
   104  			// All the leases should have OwnerReferences set to their corresponding
   105  			// Node object.
   106  			for i := range leaseList.Items {
   107  				lease := &leaseList.Items[i]
   108  				ownerRefs := lease.ObjectMeta.OwnerReferences
   109  				gomega.Expect(ownerRefs).To(gomega.HaveLen(1))
   110  				gomega.Expect(ownerRefs[0].Kind).To(gomega.Equal(v1.SchemeGroupVersion.WithKind("Node").Kind))
   111  				gomega.Expect(ownerRefs[0].APIVersion).To(gomega.Equal(v1.SchemeGroupVersion.WithKind("Node").Version))
   112  			}
   113  		})
   114  
   115  		ginkgo.It("the kubelet should report node status infrequently", func(ctx context.Context) {
   116  			ginkgo.By("wait until node is ready")
   117  			e2enode.WaitForNodeToBeReady(ctx, f.ClientSet, nodeName, 5*time.Minute)
   118  
   119  			ginkgo.By("wait until there is node lease")
   120  			var err error
   121  			var lease *coordinationv1.Lease
   122  			gomega.Eventually(ctx, func() error {
   123  				lease, err = f.ClientSet.CoordinationV1().Leases(v1.NamespaceNodeLease).Get(ctx, nodeName, metav1.GetOptions{})
   124  				if err != nil {
   125  					return err
   126  				}
   127  				return nil
   128  			}, 5*time.Minute, 5*time.Second).Should(gomega.BeNil())
   129  			// check basic expectations for the lease
   130  			framework.ExpectNoError(expectLease(lease, nodeName))
   131  			leaseDuration := time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second
   132  
   133  			ginkgo.By("verify NodeStatus report period is longer than lease duration")
   134  			// NodeStatus is reported from node to master when there is some change or
   135  			// enough time has passed. So for here, keep checking the time diff
   136  			// between 2 NodeStatus report, until it is longer than lease duration
   137  			// (the same as nodeMonitorGracePeriod), or it doesn't change for at least leaseDuration
   138  			lastHeartbeatTime, lastStatus := getHeartbeatTimeAndStatus(ctx, f.ClientSet, nodeName)
   139  			lastObserved := time.Now()
   140  			err = wait.Poll(time.Second, 5*time.Minute, func() (bool, error) {
   141  				currentHeartbeatTime, currentStatus := getHeartbeatTimeAndStatus(ctx, f.ClientSet, nodeName)
   142  				currentObserved := time.Now()
   143  
   144  				if currentHeartbeatTime == lastHeartbeatTime {
   145  					if currentObserved.Sub(lastObserved) > 2*leaseDuration {
   146  						// heartbeat hasn't changed while watching for at least 2*leaseDuration, success!
   147  						framework.Logf("node status heartbeat is unchanged for %s, was waiting for at least %s, success!", currentObserved.Sub(lastObserved), 2*leaseDuration)
   148  						return true, nil
   149  					}
   150  					framework.Logf("node status heartbeat is unchanged for %s, waiting for %s", currentObserved.Sub(lastObserved), 2*leaseDuration)
   151  					return false, nil
   152  				}
   153  
   154  				if currentHeartbeatTime.Sub(lastHeartbeatTime) >= leaseDuration {
   155  					// heartbeat time changed, but the diff was greater than leaseDuration, success!
   156  					framework.Logf("node status heartbeat changed in %s, was waiting for at least %s, success!", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration)
   157  					return true, nil
   158  				}
   159  
   160  				if !apiequality.Semantic.DeepEqual(lastStatus, currentStatus) {
   161  					// heartbeat time changed, but there were relevant changes in the status, keep waiting
   162  					framework.Logf("node status heartbeat changed in %s (with other status changes), waiting for %s", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration)
   163  					framework.Logf("%s", cmp.Diff(lastStatus, currentStatus))
   164  					lastHeartbeatTime = currentHeartbeatTime
   165  					lastObserved = currentObserved
   166  					lastStatus = currentStatus
   167  					return false, nil
   168  				}
   169  
   170  				// heartbeat time changed, with no other status changes, in less time than we expected, so fail.
   171  				return false, fmt.Errorf("node status heartbeat changed in %s (with no other status changes), was waiting for %s", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration)
   172  			})
   173  			// a timeout is acceptable, since it means we waited 5 minutes and didn't see any unwarranted node status updates
   174  			if !wait.Interrupted(err) {
   175  				framework.ExpectNoError(err, "error waiting for infrequent nodestatus update")
   176  			}
   177  
   178  			ginkgo.By("verify node is still in ready status even though node status report is infrequent")
   179  			// This check on node status is only meaningful when this e2e test is
   180  			// running as cluster e2e test, because node e2e test does not create and
   181  			// run controller manager, i.e., no node lifecycle controller.
   182  			node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
   183  			framework.ExpectNoError(err)
   184  			_, readyCondition := testutils.GetNodeCondition(&node.Status, v1.NodeReady)
   185  			gomega.Expect(readyCondition.Status).To(gomega.Equal(v1.ConditionTrue))
   186  		})
   187  	})
   188  })
   189  
   190  func getHeartbeatTimeAndStatus(ctx context.Context, clientSet clientset.Interface, nodeName string) (time.Time, v1.NodeStatus) {
   191  	node, err := clientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
   192  	framework.ExpectNoError(err)
   193  	_, readyCondition := testutils.GetNodeCondition(&node.Status, v1.NodeReady)
   194  	gomega.Expect(readyCondition.Status).To(gomega.Equal(v1.ConditionTrue))
   195  	heartbeatTime := readyCondition.LastHeartbeatTime.Time
   196  	readyCondition.LastHeartbeatTime = metav1.Time{}
   197  	return heartbeatTime, node.Status
   198  }
   199  
   200  func expectLease(lease *coordinationv1.Lease, nodeName string) error {
   201  	// expect values for HolderIdentity, LeaseDurationSeconds, and RenewTime
   202  	if lease.Spec.HolderIdentity == nil {
   203  		return fmt.Errorf("Spec.HolderIdentity should not be nil")
   204  	}
   205  	if lease.Spec.LeaseDurationSeconds == nil {
   206  		return fmt.Errorf("Spec.LeaseDurationSeconds should not be nil")
   207  	}
   208  	if lease.Spec.RenewTime == nil {
   209  		return fmt.Errorf("Spec.RenewTime should not be nil")
   210  	}
   211  	// ensure that the HolderIdentity matches the node name
   212  	if *lease.Spec.HolderIdentity != nodeName {
   213  		return fmt.Errorf("Spec.HolderIdentity (%v) should match the node name (%v)", *lease.Spec.HolderIdentity, nodeName)
   214  	}
   215  	return nil
   216  }
   217
View as plain text