recreate_node.go

Documentation: k8s.io/kubernetes/test/e2e/cloud/gcp

     1  //go:build !providerless
     2  // +build !providerless
     3  
     4  /*
     5  Copyright 2019 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package gcp
    21  
    22  import (
    23  	"context"
    24  	"fmt"
    25  	"time"
    26  
    27  	"github.com/onsi/ginkgo/v2"
    28  	v1 "k8s.io/api/core/v1"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/fields"
    31  	"k8s.io/apimachinery/pkg/labels"
    32  	clientset "k8s.io/client-go/kubernetes"
    33  	"k8s.io/kubernetes/test/e2e/feature"
    34  	"k8s.io/kubernetes/test/e2e/framework"
    35  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    36  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    37  	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
    38  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    39  	testutils "k8s.io/kubernetes/test/utils"
    40  	admissionapi "k8s.io/pod-security-admission/api"
    41  )
    42  
    43  const (
    44  	// recreateNodeReadyAgainTimeout is how long a node is allowed to become "Ready" after it is recreated before
    45  	// the test is considered failed.
    46  	recreateNodeReadyAgainTimeout = 10 * time.Minute
    47  )
    48  
    49  var _ = SIGDescribe("Recreate", feature.Recreate, func() {
    50  	f := framework.NewDefaultFramework("recreate")
    51  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    52  	var originalNodes []v1.Node
    53  	var originalPodNames []string
    54  	var ps *testutils.PodStore
    55  	systemNamespace := metav1.NamespaceSystem
    56  	ginkgo.BeforeEach(func(ctx context.Context) {
    57  		e2eskipper.SkipUnlessProviderIs("gce", "gke")
    58  		var err error
    59  		numNodes, err := e2enode.TotalRegistered(ctx, f.ClientSet)
    60  		framework.ExpectNoError(err)
    61  		originalNodes, err = e2enode.CheckReady(ctx, f.ClientSet, numNodes, framework.NodeReadyInitialTimeout)
    62  		framework.ExpectNoError(err)
    63  
    64  		framework.Logf("Got the following nodes before recreate %v", nodeNames(originalNodes))
    65  
    66  		ps, err = testutils.NewPodStore(f.ClientSet, systemNamespace, labels.Everything(), fields.Everything())
    67  		framework.ExpectNoError(err)
    68  		allPods := ps.List()
    69  		originalPods := e2epod.FilterNonRestartablePods(allPods)
    70  		originalPodNames = make([]string, len(originalPods))
    71  		for i, p := range originalPods {
    72  			originalPodNames[i] = p.ObjectMeta.Name
    73  		}
    74  
    75  		if !e2epod.CheckPodsRunningReadyOrSucceeded(ctx, f.ClientSet, systemNamespace, originalPodNames, framework.PodReadyBeforeTimeout) {
    76  			framework.Failf("At least one pod wasn't running and ready or succeeded at test start.")
    77  		}
    78  
    79  	})
    80  
    81  	ginkgo.AfterEach(func(ctx context.Context) {
    82  		if ginkgo.CurrentSpecReport().Failed() {
    83  			// Make sure that addon/system pods are running, so dump
    84  			// events for the kube-system namespace on failures
    85  			ginkgo.By(fmt.Sprintf("Collecting events from namespace %q.", systemNamespace))
    86  			events, err := f.ClientSet.CoreV1().Events(systemNamespace).List(ctx, metav1.ListOptions{})
    87  			framework.ExpectNoError(err)
    88  
    89  			for _, e := range events.Items {
    90  				framework.Logf("event for %v: %v %v: %v", e.InvolvedObject.Name, e.Source, e.Reason, e.Message)
    91  			}
    92  		}
    93  		if ps != nil {
    94  			ps.Stop()
    95  		}
    96  	})
    97  
    98  	ginkgo.It("recreate nodes and ensure they function upon restart", func(ctx context.Context) {
    99  		testRecreate(ctx, f.ClientSet, ps, systemNamespace, originalNodes, originalPodNames)
   100  	})
   101  })
   102  
   103  // Recreate all the nodes in the test instance group
   104  func testRecreate(ctx context.Context, c clientset.Interface, ps *testutils.PodStore, systemNamespace string, nodes []v1.Node, podNames []string) {
   105  	err := gce.RecreateNodes(c, nodes)
   106  	if err != nil {
   107  		framework.Failf("Test failed; failed to start the restart instance group command.")
   108  	}
   109  
   110  	err = gce.WaitForNodeBootIdsToChange(ctx, c, nodes, recreateNodeReadyAgainTimeout)
   111  	if err != nil {
   112  		framework.Failf("Test failed; failed to recreate at least one node in %v.", recreateNodeReadyAgainTimeout)
   113  	}
   114  
   115  	nodesAfter, err := e2enode.CheckReady(ctx, c, len(nodes), framework.RestartNodeReadyAgainTimeout)
   116  	framework.ExpectNoError(err)
   117  	framework.Logf("Got the following nodes after recreate: %v", nodeNames(nodesAfter))
   118  
   119  	if len(nodes) != len(nodesAfter) {
   120  		framework.Failf("Had %d nodes before nodes were recreated, but now only have %d",
   121  			len(nodes), len(nodesAfter))
   122  	}
   123  
   124  	// Make sure the pods from before node recreation are running/completed
   125  	podCheckStart := time.Now()
   126  	podNamesAfter, err := e2epod.WaitForNRestartablePods(ctx, ps, len(podNames), framework.RestartPodReadyAgainTimeout)
   127  	framework.ExpectNoError(err)
   128  	remaining := framework.RestartPodReadyAgainTimeout - time.Since(podCheckStart)
   129  	if !e2epod.CheckPodsRunningReadyOrSucceeded(ctx, c, systemNamespace, podNamesAfter, remaining) {
   130  		framework.Failf("At least one pod wasn't running and ready after the restart.")
   131  	}
   132  }
   133
View as plain text