kubelet_stats.go

Documentation: k8s.io/kubernetes/test/e2e/windows

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package windows
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	v1 "k8s.io/api/core/v1"
    25  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    26  	"k8s.io/apimachinery/pkg/labels"
    27  	"k8s.io/apimachinery/pkg/util/uuid"
    28  	"k8s.io/kubernetes/test/e2e/feature"
    29  	"k8s.io/kubernetes/test/e2e/framework"
    30  	e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet"
    31  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    32  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    33  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    34  	imageutils "k8s.io/kubernetes/test/utils/image"
    35  	admissionapi "k8s.io/pod-security-admission/api"
    36  
    37  	"github.com/onsi/ginkgo/v2"
    38  	"github.com/onsi/gomega"
    39  )
    40  
    41  var _ = sigDescribe(feature.Windows, "Kubelet-Stats", framework.WithSerial(), skipUnlessWindows(func() {
    42  	f := framework.NewDefaultFramework("kubelet-stats-test-windows-serial")
    43  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    44  
    45  	ginkgo.Describe("Kubelet stats collection for Windows nodes", func() {
    46  
    47  		ginkgo.Context("when running 10 pods", func() {
    48  			// 10 seconds is the default scrape timeout for metrics-server and kube-prometheus
    49  			ginkgo.It("should return within 10 seconds", func(ctx context.Context) {
    50  
    51  				ginkgo.By("Selecting a Windows node")
    52  				targetNode, err := findWindowsNode(ctx, f)
    53  				framework.ExpectNoError(err, "Error finding Windows node")
    54  				framework.Logf("Using node: %v", targetNode.Name)
    55  
    56  				ginkgo.By("Scheduling 10 pods")
    57  				powershellImage := imageutils.GetConfig(imageutils.BusyBox)
    58  				pods := newKubeletStatsTestPods(10, powershellImage, targetNode.Name)
    59  				e2epod.NewPodClient(f).CreateBatch(ctx, pods)
    60  
    61  				ginkgo.By("Waiting up to 3 minutes for pods to be running")
    62  				timeout := 3 * time.Minute
    63  				err = e2epod.WaitForPodsRunningReady(ctx, f.ClientSet, f.Namespace.Name, 10, 0, timeout)
    64  				framework.ExpectNoError(err)
    65  
    66  				ginkgo.By("Getting kubelet stats 5 times and checking average duration")
    67  				iterations := 5
    68  				var totalDurationMs int64
    69  
    70  				for i := 0; i < iterations; i++ {
    71  					start := time.Now()
    72  					nodeStats, err := e2ekubelet.GetStatsSummary(ctx, f.ClientSet, targetNode.Name)
    73  					duration := time.Since(start)
    74  					totalDurationMs += duration.Milliseconds()
    75  
    76  					framework.ExpectNoError(err, "Error getting kubelet stats")
    77  
    78  					// Perform some basic sanity checks on retrieved stats for pods in this test's namespace
    79  					statsChecked := 0
    80  					for _, podStats := range nodeStats.Pods {
    81  						if podStats.PodRef.Namespace != f.Namespace.Name {
    82  							continue
    83  						}
    84  						statsChecked = statsChecked + 1
    85  
    86  						if *podStats.CPU.UsageCoreNanoSeconds <= 0 {
    87  							framework.Failf("Pod %s/%s stats report cpu usage equal to %v but it should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds)
    88  						}
    89  						if *podStats.Memory.WorkingSetBytes <= 0 {
    90  							framework.Failf("Pod %s/%s stats report bytes for memory working set equal to %v but it should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.Memory.WorkingSetBytes)
    91  						}
    92  
    93  						for _, containerStats := range podStats.Containers {
    94  							if containerStats.Logs == nil {
    95  								framework.Failf("Pod %s/%s stats should have container log stats, but it is nil", podStats.PodRef.Namespace, podStats.PodRef.Name)
    96  							}
    97  							if *containerStats.Logs.AvailableBytes <= 0 {
    98  								framework.Failf("Pod %s/%s container log stats report %v bytes for AvailableBytes, but it should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *containerStats.Logs.AvailableBytes)
    99  							}
   100  						}
   101  					}
   102  					gomega.Expect(statsChecked).To(gomega.Equal(10), "Should find stats for 10 pods in kubelet stats")
   103  
   104  					time.Sleep(5 * time.Second)
   105  				}
   106  
   107  				avgDurationMs := totalDurationMs / int64(iterations)
   108  
   109  				durationMatch := avgDurationMs <= time.Duration(10*time.Second).Milliseconds()
   110  				framework.Logf("Getting kubelet stats for node %v took an average of %v milliseconds over %v iterations", targetNode.Name, avgDurationMs, iterations)
   111  				if !durationMatch {
   112  					framework.Failf("Collecting kubelet stats took %v seconds but it should not take longer than 10 seconds", durationMatch)
   113  				}
   114  			})
   115  		})
   116  	})
   117  }))
   118  
   119  var _ = sigDescribe(feature.Windows, "Kubelet-Stats", skipUnlessWindows(func() {
   120  	f := framework.NewDefaultFramework("kubelet-stats-test-windows")
   121  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
   122  
   123  	ginkgo.Describe("Kubelet stats collection for Windows nodes", func() {
   124  
   125  		ginkgo.Context("when windows is booted", func() {
   126  			ginkgo.It("should return bootid within 10 seconds", func(ctx context.Context) {
   127  				ginkgo.By("Selecting a Windows node")
   128  				targetNode, err := findWindowsNode(ctx, f)
   129  				framework.ExpectNoError(err, "Error finding Windows node")
   130  				framework.Logf("Using node: %v", targetNode.Name)
   131  
   132  				ginkgo.By("Getting bootid")
   133  				if len(targetNode.Status.NodeInfo.BootID) == 0 {
   134  					framework.Failf("expected bootId in kubelet stats, got none")
   135  				}
   136  			})
   137  		})
   138  
   139  		ginkgo.Context("when running 3 pods", func() {
   140  			// 10 seconds is the default scrape timeout for metrics-server and kube-prometheus
   141  			ginkgo.It("should return within 10 seconds", func(ctx context.Context) {
   142  
   143  				ginkgo.By("Selecting a Windows node")
   144  				targetNode, err := findWindowsNode(ctx, f)
   145  				framework.ExpectNoError(err, "Error finding Windows node")
   146  				framework.Logf("Using node: %v", targetNode.Name)
   147  
   148  				ginkgo.By("Scheduling 3 pods")
   149  				powershellImage := imageutils.GetConfig(imageutils.BusyBox)
   150  				pods := newKubeletStatsTestPods(3, powershellImage, targetNode.Name)
   151  				e2epod.NewPodClient(f).CreateBatch(ctx, pods)
   152  
   153  				ginkgo.By("Waiting up to 3 minutes for pods to be running")
   154  				timeout := 3 * time.Minute
   155  				err = e2epod.WaitForPodsRunningReady(ctx, f.ClientSet, f.Namespace.Name, 3, 0, timeout)
   156  				framework.ExpectNoError(err)
   157  
   158  				ginkgo.By("Getting kubelet stats 1 time")
   159  				iterations := 1
   160  				var totalDurationMs int64
   161  
   162  				for i := 0; i < iterations; i++ {
   163  					start := time.Now()
   164  					nodeStats, err := e2ekubelet.GetStatsSummary(ctx, f.ClientSet, targetNode.Name)
   165  					duration := time.Since(start)
   166  					totalDurationMs += duration.Milliseconds()
   167  
   168  					framework.ExpectNoError(err, "Error getting kubelet stats")
   169  
   170  					// Perform some basic sanity checks on retrieved stats for pods in this test's namespace
   171  					statsChecked := 0
   172  					for _, podStats := range nodeStats.Pods {
   173  						if podStats.PodRef.Namespace != f.Namespace.Name {
   174  							continue
   175  						}
   176  						statsChecked = statsChecked + 1
   177  
   178  						if *podStats.CPU.UsageCoreNanoSeconds <= 0 {
   179  							framework.Failf("Pod %s/%s stats report cpu usage equal to %v but it should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds)
   180  						}
   181  						if *podStats.Memory.WorkingSetBytes <= 0 {
   182  							framework.Failf("Pod %s/%s stats report bytes for memory working set equal to %v but it should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.Memory.WorkingSetBytes)
   183  						}
   184  
   185  						for _, containerStats := range podStats.Containers {
   186  							if containerStats.Logs == nil {
   187  								framework.Failf("Pod %s/%s stats should have container log stats, but it is nil", podStats.PodRef.Namespace, podStats.PodRef.Name)
   188  							}
   189  							if *containerStats.Logs.AvailableBytes <= 0 {
   190  								framework.Failf("Pod %s/%s container log stats report %v bytes for AvailableBytes, but it should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *containerStats.Logs.AvailableBytes)
   191  							}
   192  						}
   193  					}
   194  					gomega.Expect(statsChecked).To(gomega.Equal(3), "Should find stats for 3 pods in kubelet stats")
   195  
   196  					time.Sleep(5 * time.Second)
   197  				}
   198  
   199  				avgDurationMs := totalDurationMs / int64(iterations)
   200  
   201  				durationMatch := avgDurationMs <= time.Duration(10*time.Second).Milliseconds()
   202  				framework.Logf("Getting kubelet stats for node %v took an average of %v milliseconds over %v iterations", targetNode.Name, avgDurationMs, iterations)
   203  				if !durationMatch {
   204  					framework.Failf("Collecting kubelet stats took %v seconds but it should not take longer than 10 seconds", durationMatch)
   205  				}
   206  			})
   207  		})
   208  	})
   209  }))
   210  
   211  // findWindowsNode finds a Windows node that is Ready and Schedulable
   212  func findWindowsNode(ctx context.Context, f *framework.Framework) (v1.Node, error) {
   213  	selector := labels.Set{"kubernetes.io/os": "windows"}.AsSelector()
   214  	nodeList, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
   215  
   216  	if err != nil {
   217  		return v1.Node{}, err
   218  	}
   219  
   220  	var targetNode v1.Node
   221  	foundNode := false
   222  	for _, n := range nodeList.Items {
   223  		if e2enode.IsNodeReady(&n) && e2enode.IsNodeSchedulable(&n) {
   224  			targetNode = n
   225  			foundNode = true
   226  			break
   227  		}
   228  	}
   229  
   230  	if !foundNode {
   231  		e2eskipper.Skipf("Could not find and ready and schedulable Windows nodes")
   232  	}
   233  
   234  	return targetNode, nil
   235  }
   236  
   237  // newKubeletStatsTestPods creates a list of pods (specification) for test.
   238  func newKubeletStatsTestPods(numPods int, image imageutils.Config, nodeName string) []*v1.Pod {
   239  	var pods []*v1.Pod
   240  
   241  	for i := 0; i < numPods; i++ {
   242  		podName := "statscollectiontest-" + string(uuid.NewUUID())
   243  		pod := v1.Pod{
   244  			ObjectMeta: metav1.ObjectMeta{
   245  				Name: fmt.Sprintf("%s-%d", podName, i),
   246  				Labels: map[string]string{
   247  					"name":    podName,
   248  					"testapp": "stats-collection",
   249  				},
   250  			},
   251  			Spec: v1.PodSpec{
   252  				Containers: []v1.Container{
   253  					{
   254  						Image: image.GetE2EImage(),
   255  						Name:  "stat-container",
   256  						Command: []string{
   257  							"powershell.exe",
   258  							"-Command",
   259  							"sleep -Seconds 600",
   260  						},
   261  					},
   262  				},
   263  				InitContainers: []v1.Container{
   264  					{
   265  						Image: image.GetE2EImage(),
   266  						Name:  "init-container",
   267  						Command: []string{
   268  							"powershell.exe",
   269  							"-Command",
   270  							"sleep -Seconds 1",
   271  						},
   272  					},
   273  				},
   274  				NodeName: nodeName,
   275  			},
   276  		}
   277  
   278  		pods = append(pods, &pod)
   279  	}
   280  
   281  	return pods
   282  }
   283
View as plain text