static_pods.go

Documentation: k8s.io/kubernetes/test/e2e/storage

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package storage
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"os"
    23  	"os/exec"
    24  	"strings"
    25  	"time"
    26  
    27  	"github.com/onsi/ginkgo/v2"
    28  	"github.com/onsi/gomega"
    29  	v1 "k8s.io/api/core/v1"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/kubernetes/test/e2e/feature"
    32  	"k8s.io/kubernetes/test/e2e/framework"
    33  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    34  	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
    35  	"k8s.io/kubernetes/test/e2e/storage/drivers"
    36  	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
    37  	"k8s.io/kubernetes/test/e2e/storage/utils"
    38  	admissionapi "k8s.io/pod-security-admission/api"
    39  )
    40  
    41  // Tests for static Pods + CSI volume reconstruction.
    42  //
    43  // NOTE: these tests *require* the cluster under test to be Kubernetes In Docker (kind)!
    44  // Kind runs its API server as a static Pod, and we leverage it here
    45  // to test kubelet starting without the API server.
    46  var _ = utils.SIGDescribe("StaticPods", feature.Kind, func() {
    47  	f := framework.NewDefaultFramework("static-pods-csi")
    48  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    49  
    50  	// global vars for the ginkgo.BeforeEach() / It()'s below
    51  	var (
    52  		driver     storageframework.DynamicPVTestDriver
    53  		testConfig *storageframework.PerTestConfig
    54  	)
    55  
    56  	ginkgo.BeforeEach(func(ctx context.Context) {
    57  		// Install a CSI driver. Using csi-driver-hospath here.
    58  		driver = drivers.InitHostPathCSIDriver().(storageframework.DynamicPVTestDriver)
    59  		testConfig = driver.PrepareTest(ctx, f)
    60  	})
    61  
    62  	// Test https://github.com/kubernetes/kubernetes/issues/117745
    63  	// I.e. kubelet starts and it must start the API server as a static pod,
    64  	// while there is a CSI volume mounted by the previous kubelet.
    65  	f.It("should run after kubelet stopped with CSI volume mounted", f.WithDisruptive(), f.WithSerial(), func(ctx context.Context) {
    66  		var timeout int64 = 5
    67  
    68  		ginkgo.By("Provision a new CSI volume")
    69  		res := storageframework.CreateVolumeResource(ctx, driver, testConfig, storageframework.DefaultFsDynamicPV, e2evolume.SizeRange{Min: "1", Max: "1Gi"})
    70  		ginkgo.DeferCleanup(func(ctx context.Context) {
    71  			res.CleanupResource(ctx)
    72  		})
    73  
    74  		ginkgo.By("Run a pod with the volume")
    75  		podConfig := &e2epod.Config{
    76  			NS:            f.Namespace.Name,
    77  			PVCs:          []*v1.PersistentVolumeClaim{res.Pvc},
    78  			NodeSelection: testConfig.ClientNodeSelection,
    79  		}
    80  		pod, err := e2epod.CreateSecPod(ctx, f.ClientSet, podConfig, f.Timeouts.PodStart)
    81  		framework.ExpectNoError(err, "Creating a pod with a CSI volume")
    82  		ginkgo.DeferCleanup(f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete, pod.Name, metav1.DeleteOptions{})
    83  
    84  		ginkgo.By("Stop the control plane by removing the static pod manifests")
    85  		err = stopControlPlane(ctx)
    86  		framework.ExpectNoError(err, "Stopping the control plane")
    87  
    88  		ginkgo.By("Wait for the API server to disappear")
    89  		gomega.Eventually(ctx, func(ctx context.Context) error {
    90  			_, err := f.ClientSet.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{TimeoutSeconds: &timeout})
    91  			return err
    92  		}).WithTimeout(time.Minute).Should(gomega.Not(gomega.Succeed()))
    93  
    94  		ginkgo.By("Stop kubelet")
    95  		err = stopKindKubelet(ctx)
    96  		framework.ExpectNoError(err, "Stopping kubelet")
    97  
    98  		ginkgo.By("Re-enable control plane")
    99  		err = enableControlPlane(ctx)
   100  		framework.ExpectNoError(err, "Re-enabling control plane by restoring the static pod manifests")
   101  
   102  		ginkgo.By("Start kubelet")
   103  		err = startKindKubelet(ctx)
   104  		framework.ExpectNoError(err, "Starting kubelet")
   105  
   106  		ginkgo.By("Wait for a static pod with the API server to start")
   107  		const apiServerStartTimeout = 10 * time.Minute // It takes up to 2 minutes on my machine
   108  		gomega.Eventually(ctx, func(ctx context.Context) error {
   109  			_, err := f.ClientSet.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{TimeoutSeconds: &timeout})
   110  			return err
   111  		}).WithTimeout(apiServerStartTimeout).Should(gomega.Succeed())
   112  	})
   113  })
   114  
   115  func stopKindKubelet(ctx context.Context) error {
   116  	return kubeletExec("systemctl", "stop", "kubelet")
   117  }
   118  
   119  func startKindKubelet(ctx context.Context) error {
   120  	return kubeletExec("systemctl", "start", "kubelet")
   121  }
   122  
   123  func stopControlPlane(ctx context.Context) error {
   124  	// Move all static pod manifests to a backup dir (incl. KCM, scheduler, API server and etcd)
   125  	return kubeletExec("sh", "-c", "mkdir -p /etc/kubernetes/manifests-backup && mv /etc/kubernetes/manifests/* /etc/kubernetes/manifests-backup/")
   126  }
   127  
   128  func enableControlPlane(ctx context.Context) error {
   129  	// Restore the static pod manifests from the backup dir.
   130  	// Using `sh` to get `*` expansion.
   131  	return kubeletExec("sh", "-c", "mv /etc/kubernetes/manifests-backup/* /etc/kubernetes/manifests/")
   132  }
   133  
   134  // Run a command in container with kubelet (and the whole control plane as containers)
   135  func kubeletExec(command ...string) error {
   136  	containerName := getKindContainerName()
   137  	args := []string{"exec", containerName}
   138  	args = append(args, command...)
   139  	cmd := exec.Command("docker", args...)
   140  
   141  	out, err := cmd.CombinedOutput()
   142  	if err != nil {
   143  		return fmt.Errorf("command %q failed: %v\noutput:%s", prettyCmd(cmd), err, string(out))
   144  	}
   145  
   146  	framework.Logf("command %q succeeded:\n%s", prettyCmd(cmd), string(out))
   147  	return nil
   148  }
   149  
   150  func getKindContainerName() string {
   151  	clusterName := os.Getenv("KIND_CLUSTER_NAME")
   152  	if clusterName == "" {
   153  		clusterName = "kind"
   154  	}
   155  	return clusterName + "-control-plane"
   156  }
   157  
   158  func prettyCmd(cmd *exec.Cmd) string {
   159  	return fmt.Sprintf("%s %s", cmd.Path, strings.Join(cmd.Args, " "))
   160  }
   161
View as plain text