node_lease.go

Documentation: k8s.io/kubernetes/test/e2e/cloud/gcp

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package gcp
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"time"
    24  
    25  	v1 "k8s.io/api/core/v1"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	clientset "k8s.io/client-go/kubernetes"
    28  	"k8s.io/kubernetes/test/e2e/framework"
    29  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    30  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    31  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    32  	admissionapi "k8s.io/pod-security-admission/api"
    33  
    34  	"github.com/onsi/ginkgo/v2"
    35  	"github.com/onsi/gomega"
    36  )
    37  
    38  var _ = SIGDescribe(framework.WithDisruptive(), "NodeLease", func() {
    39  	f := framework.NewDefaultFramework("node-lease-test")
    40  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    41  	var systemPodsNo int32
    42  	var c clientset.Interface
    43  	var ns string
    44  	var group string
    45  
    46  	ginkgo.BeforeEach(func(ctx context.Context) {
    47  		c = f.ClientSet
    48  		ns = f.Namespace.Name
    49  		systemPods, err := e2epod.GetPodsInNamespace(ctx, c, ns, map[string]string{})
    50  		framework.ExpectNoError(err)
    51  		systemPodsNo = int32(len(systemPods))
    52  		if strings.Contains(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") {
    53  			framework.Failf("Test dose not support cluster setup with more than one MIG: %s", framework.TestContext.CloudConfig.NodeInstanceGroup)
    54  		} else {
    55  			group = framework.TestContext.CloudConfig.NodeInstanceGroup
    56  		}
    57  	})
    58  
    59  	ginkgo.Describe("NodeLease deletion", func() {
    60  		var skipped bool
    61  
    62  		ginkgo.BeforeEach(func() {
    63  			skipped = true
    64  			e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
    65  			e2eskipper.SkipUnlessNodeCountIsAtLeast(2)
    66  			skipped = false
    67  		})
    68  
    69  		ginkgo.AfterEach(func(ctx context.Context) {
    70  			if skipped {
    71  				return
    72  			}
    73  
    74  			ginkgo.By("restoring the original node instance group size")
    75  			if err := framework.ResizeGroup(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil {
    76  				framework.Failf("Couldn't restore the original node instance group size: %v", err)
    77  			}
    78  			// In GKE, our current tunneling setup has the potential to hold on to a broken tunnel (from a
    79  			// rebooted/deleted node) for up to 5 minutes before all tunnels are dropped and recreated.
    80  			// Most tests make use of some proxy feature to verify functionality. So, if a reboot test runs
    81  			// right before a test that tries to get logs, for example, we may get unlucky and try to use a
    82  			// closed tunnel to a node that was recently rebooted. There's no good way to framework.Poll for proxies
    83  			// being closed, so we sleep.
    84  			//
    85  			// TODO(cjcullen) reduce this sleep (#19314)
    86  			if framework.ProviderIs("gke") {
    87  				ginkgo.By("waiting 5 minutes for all dead tunnels to be dropped")
    88  				time.Sleep(5 * time.Minute)
    89  			}
    90  			if err := framework.WaitForGroupSize(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil {
    91  				framework.Failf("Couldn't restore the original node instance group size: %v", err)
    92  			}
    93  
    94  			if err := e2enode.WaitForReadyNodes(ctx, c, framework.TestContext.CloudConfig.NumNodes, 10*time.Minute); err != nil {
    95  				framework.Failf("Couldn't restore the original cluster size: %v", err)
    96  			}
    97  			// Many e2e tests assume that the cluster is fully healthy before they start.  Wait until
    98  			// the cluster is restored to health.
    99  			ginkgo.By("waiting for system pods to successfully restart")
   100  			err := e2epod.WaitForPodsRunningReady(ctx, c, metav1.NamespaceSystem, systemPodsNo, 0, framework.PodReadyBeforeTimeout)
   101  			framework.ExpectNoError(err)
   102  		})
   103  
   104  		ginkgo.It("node lease should be deleted when corresponding node is deleted", func(ctx context.Context) {
   105  			leaseClient := c.CoordinationV1().Leases(v1.NamespaceNodeLease)
   106  			err := e2enode.WaitForReadyNodes(ctx, c, framework.TestContext.CloudConfig.NumNodes, 10*time.Minute)
   107  			framework.ExpectNoError(err)
   108  
   109  			ginkgo.By("verify node lease exists for every nodes")
   110  			originalNodes, err := e2enode.GetReadySchedulableNodes(ctx, c)
   111  			framework.ExpectNoError(err)
   112  			gomega.Expect(originalNodes.Items).To(gomega.HaveLen(framework.TestContext.CloudConfig.NumNodes))
   113  
   114  			gomega.Eventually(ctx, func() error {
   115  				pass := true
   116  				for _, node := range originalNodes.Items {
   117  					if _, err := leaseClient.Get(ctx, node.ObjectMeta.Name, metav1.GetOptions{}); err != nil {
   118  						framework.Logf("Try to get lease of node %s, but got error: %v", node.ObjectMeta.Name, err)
   119  						pass = false
   120  					}
   121  				}
   122  				if pass {
   123  					return nil
   124  				}
   125  				return fmt.Errorf("some node lease is not ready")
   126  			}, 1*time.Minute, 5*time.Second).Should(gomega.BeNil())
   127  
   128  			targetNumNodes := int32(framework.TestContext.CloudConfig.NumNodes - 1)
   129  			ginkgo.By(fmt.Sprintf("decreasing cluster size to %d", targetNumNodes))
   130  			err = framework.ResizeGroup(group, targetNumNodes)
   131  			framework.ExpectNoError(err)
   132  			err = framework.WaitForGroupSize(group, targetNumNodes)
   133  			framework.ExpectNoError(err)
   134  			err = e2enode.WaitForReadyNodes(ctx, c, framework.TestContext.CloudConfig.NumNodes-1, 10*time.Minute)
   135  			framework.ExpectNoError(err)
   136  			targetNodes, err := e2enode.GetReadySchedulableNodes(ctx, c)
   137  			framework.ExpectNoError(err)
   138  			gomega.Expect(targetNodes.Items).To(gomega.HaveLen(int(targetNumNodes)))
   139  
   140  			ginkgo.By("verify node lease is deleted for the deleted node")
   141  			var deletedNodeName string
   142  			for _, originalNode := range originalNodes.Items {
   143  				originalNodeName := originalNode.ObjectMeta.Name
   144  				for _, targetNode := range targetNodes.Items {
   145  					if originalNodeName == targetNode.ObjectMeta.Name {
   146  						continue
   147  					}
   148  				}
   149  				deletedNodeName = originalNodeName
   150  				break
   151  			}
   152  			gomega.Expect(deletedNodeName).NotTo(gomega.BeEmpty())
   153  			gomega.Eventually(ctx, func() error {
   154  				if _, err := leaseClient.Get(ctx, deletedNodeName, metav1.GetOptions{}); err == nil {
   155  					return fmt.Errorf("node lease is not deleted yet for node %q", deletedNodeName)
   156  				}
   157  				return nil
   158  			}, 1*time.Minute, 5*time.Second).Should(gomega.BeNil())
   159  
   160  			ginkgo.By("verify node leases still exist for remaining nodes")
   161  			gomega.Eventually(ctx, func() error {
   162  				for _, node := range targetNodes.Items {
   163  					if _, err := leaseClient.Get(ctx, node.ObjectMeta.Name, metav1.GetOptions{}); err != nil {
   164  						return err
   165  					}
   166  				}
   167  				return nil
   168  			}, 1*time.Minute, 5*time.Second).Should(gomega.BeNil())
   169  		})
   170  	})
   171  })
   172
View as plain text