kubelet_perf.go

Documentation: k8s.io/kubernetes/test/e2e/node

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package node
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"time"
    24  
    25  	"k8s.io/apimachinery/pkg/util/sets"
    26  	"k8s.io/apimachinery/pkg/util/uuid"
    27  	clientset "k8s.io/client-go/kubernetes"
    28  	kubeletstatsv1alpha1 "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
    29  	"k8s.io/kubernetes/test/e2e/feature"
    30  	"k8s.io/kubernetes/test/e2e/framework"
    31  	e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet"
    32  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    33  	e2eperf "k8s.io/kubernetes/test/e2e/framework/perf"
    34  	e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
    35  	"k8s.io/kubernetes/test/e2e/perftype"
    36  	testutils "k8s.io/kubernetes/test/utils"
    37  	imageutils "k8s.io/kubernetes/test/utils/image"
    38  	admissionapi "k8s.io/pod-security-admission/api"
    39  
    40  	"github.com/onsi/ginkgo/v2"
    41  )
    42  
    43  const (
    44  	// Interval to poll /stats/container on a node
    45  	containerStatsPollingPeriod = 10 * time.Second
    46  	// The monitoring time for one test.
    47  	monitoringTime = 20 * time.Minute
    48  	// The periodic reporting period.
    49  	reportingPeriod = 5 * time.Minute
    50  )
    51  
    52  type resourceTest struct {
    53  	podsPerNode int
    54  	cpuLimits   e2ekubelet.ContainersCPUSummary
    55  	memLimits   e2ekubelet.ResourceUsagePerContainer
    56  }
    57  
    58  func logPodsOnNodes(ctx context.Context, c clientset.Interface, nodeNames []string) {
    59  	for _, n := range nodeNames {
    60  		podList, err := e2ekubelet.GetKubeletRunningPods(ctx, c, n)
    61  		if err != nil {
    62  			framework.Logf("Unable to retrieve kubelet pods for node %v", n)
    63  			continue
    64  		}
    65  		framework.Logf("%d pods are running on node %v", len(podList.Items), n)
    66  	}
    67  }
    68  
    69  func runResourceTrackingTest(ctx context.Context, f *framework.Framework, podsPerNode int, nodeNames sets.String, rm *e2ekubelet.ResourceMonitor,
    70  	expectedCPU map[string]map[float64]float64, expectedMemory e2ekubelet.ResourceUsagePerContainer) {
    71  	numNodes := nodeNames.Len()
    72  	totalPods := podsPerNode * numNodes
    73  	ginkgo.By(fmt.Sprintf("Creating a RC of %d pods and wait until all pods of this RC are running", totalPods))
    74  	rcName := fmt.Sprintf("resource%d-%s", totalPods, string(uuid.NewUUID()))
    75  
    76  	// TODO: Use a more realistic workload
    77  	err := e2erc.RunRC(ctx, testutils.RCConfig{
    78  		Client:    f.ClientSet,
    79  		Name:      rcName,
    80  		Namespace: f.Namespace.Name,
    81  		Image:     imageutils.GetPauseImageName(),
    82  		Replicas:  totalPods,
    83  	})
    84  	framework.ExpectNoError(err)
    85  
    86  	// Log once and flush the stats.
    87  	rm.LogLatest()
    88  	rm.Reset()
    89  
    90  	ginkgo.By("Start monitoring resource usage")
    91  	// Periodically dump the cpu summary until the deadline is met.
    92  	// Note that without calling e2ekubelet.ResourceMonitor.Reset(), the stats
    93  	// would occupy increasingly more memory. This should be fine
    94  	// for the current test duration, but we should reclaim the
    95  	// entries if we plan to monitor longer (e.g., 8 hours).
    96  	deadline := time.Now().Add(monitoringTime)
    97  	for time.Now().Before(deadline) && ctx.Err() == nil {
    98  		timeLeft := time.Until(deadline)
    99  		framework.Logf("Still running...%v left", timeLeft)
   100  		if timeLeft < reportingPeriod {
   101  			time.Sleep(timeLeft)
   102  		} else {
   103  			time.Sleep(reportingPeriod)
   104  		}
   105  		logPodsOnNodes(ctx, f.ClientSet, nodeNames.List())
   106  	}
   107  
   108  	ginkgo.By("Reporting overall resource usage")
   109  	logPodsOnNodes(ctx, f.ClientSet, nodeNames.List())
   110  	usageSummary, err := rm.GetLatest()
   111  	framework.ExpectNoError(err)
   112  	// TODO(random-liu): Remove the original log when we migrate to new perfdash
   113  	framework.Logf("%s", rm.FormatResourceUsage(usageSummary))
   114  	// Log perf result
   115  	printPerfData(e2eperf.ResourceUsageToPerfData(rm.GetMasterNodeLatest(usageSummary)))
   116  	verifyMemoryLimits(ctx, f.ClientSet, expectedMemory, usageSummary)
   117  
   118  	cpuSummary := rm.GetCPUSummary()
   119  	framework.Logf("%s", rm.FormatCPUSummary(cpuSummary))
   120  	// Log perf result
   121  	printPerfData(e2eperf.CPUUsageToPerfData(rm.GetMasterNodeCPUSummary(cpuSummary)))
   122  	verifyCPULimits(expectedCPU, cpuSummary)
   123  
   124  	ginkgo.By("Deleting the RC")
   125  	e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, rcName)
   126  }
   127  
   128  func verifyMemoryLimits(ctx context.Context, c clientset.Interface, expected e2ekubelet.ResourceUsagePerContainer, actual e2ekubelet.ResourceUsagePerNode) {
   129  	if expected == nil {
   130  		return
   131  	}
   132  	var errList []string
   133  	for nodeName, nodeSummary := range actual {
   134  		var nodeErrs []string
   135  		for cName, expectedResult := range expected {
   136  			container, ok := nodeSummary[cName]
   137  			if !ok {
   138  				nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: missing", cName))
   139  				continue
   140  			}
   141  
   142  			expectedValue := expectedResult.MemoryRSSInBytes
   143  			actualValue := container.MemoryRSSInBytes
   144  			if expectedValue != 0 && actualValue > expectedValue {
   145  				nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: expected RSS memory (MB) < %d; got %d",
   146  					cName, expectedValue, actualValue))
   147  			}
   148  		}
   149  		if len(nodeErrs) > 0 {
   150  			errList = append(errList, fmt.Sprintf("node %v:\n %s", nodeName, strings.Join(nodeErrs, ", ")))
   151  			heapStats, err := e2ekubelet.GetKubeletHeapStats(ctx, c, nodeName)
   152  			if err != nil {
   153  				framework.Logf("Unable to get heap stats from %q", nodeName)
   154  			} else {
   155  				framework.Logf("Heap stats on %q\n:%v", nodeName, heapStats)
   156  			}
   157  		}
   158  	}
   159  	if len(errList) > 0 {
   160  		framework.Failf("Memory usage exceeding limits:\n %s", strings.Join(errList, "\n"))
   161  	}
   162  }
   163  
   164  func verifyCPULimits(expected e2ekubelet.ContainersCPUSummary, actual e2ekubelet.NodesCPUSummary) {
   165  	if expected == nil {
   166  		return
   167  	}
   168  	var errList []string
   169  	for nodeName, perNodeSummary := range actual {
   170  		var nodeErrs []string
   171  		for cName, expectedResult := range expected {
   172  			perContainerSummary, ok := perNodeSummary[cName]
   173  			if !ok {
   174  				nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: missing", cName))
   175  				continue
   176  			}
   177  			for p, expectedValue := range expectedResult {
   178  				actualValue, ok := perContainerSummary[p]
   179  				if !ok {
   180  					nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: missing percentile %v", cName, p))
   181  					continue
   182  				}
   183  				if actualValue > expectedValue {
   184  					nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: expected %.0fth%% usage < %.3f; got %.3f",
   185  						cName, p*100, expectedValue, actualValue))
   186  				}
   187  			}
   188  		}
   189  		if len(nodeErrs) > 0 {
   190  			errList = append(errList, fmt.Sprintf("node %v:\n %s", nodeName, strings.Join(nodeErrs, ", ")))
   191  		}
   192  	}
   193  	if len(errList) > 0 {
   194  		framework.Failf("CPU usage exceeding limits:\n %s", strings.Join(errList, "\n"))
   195  	}
   196  }
   197  
   198  // Slow by design (1 hour)
   199  var _ = SIGDescribe("Kubelet", framework.WithSerial(), framework.WithSlow(), func() {
   200  	var nodeNames sets.String
   201  	f := framework.NewDefaultFramework("kubelet-perf")
   202  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
   203  	var om *e2ekubelet.RuntimeOperationMonitor
   204  	var rm *e2ekubelet.ResourceMonitor
   205  
   206  	ginkgo.BeforeEach(func(ctx context.Context) {
   207  		nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet)
   208  		framework.ExpectNoError(err)
   209  		nodeNames = sets.NewString()
   210  		for _, node := range nodes.Items {
   211  			nodeNames.Insert(node.Name)
   212  		}
   213  		om = e2ekubelet.NewRuntimeOperationMonitor(ctx, f.ClientSet)
   214  		rm = e2ekubelet.NewResourceMonitor(f.ClientSet, e2ekubelet.TargetContainers(), containerStatsPollingPeriod)
   215  		rm.Start(ctx)
   216  	})
   217  
   218  	ginkgo.AfterEach(func(ctx context.Context) {
   219  		rm.Stop()
   220  		result := om.GetLatestRuntimeOperationErrorRate(ctx)
   221  		framework.Logf("runtime operation error metrics:\n%s", e2ekubelet.FormatRuntimeOperationErrorRate(result))
   222  	})
   223  	f.Describe("regular resource usage tracking", feature.RegularResourceUsageTracking, func() {
   224  		// We assume that the scheduler will make reasonable scheduling choices
   225  		// and assign ~N pods on the node.
   226  		// Although we want to track N pods per node, there are N + add-on pods
   227  		// in the cluster. The cluster add-on pods can be distributed unevenly
   228  		// among the nodes because they are created during the cluster
   229  		// initialization. This *noise* is obvious when N is small. We
   230  		// deliberately set higher resource usage limits to account for the
   231  		// noise.
   232  		//
   233  		// We set all resource limits generously because this test is mainly
   234  		// used to catch resource leaks in the soak cluster. For tracking
   235  		// kubelet/runtime resource usage, please see the node e2e benchmark
   236  		// dashboard. http://node-perf-dash.k8s.io/
   237  		//
   238  		// TODO(#36621): Deprecate this test once we have a node e2e soak
   239  		// cluster.
   240  		rTests := []resourceTest{
   241  			{
   242  				podsPerNode: 0,
   243  				cpuLimits: e2ekubelet.ContainersCPUSummary{
   244  					kubeletstatsv1alpha1.SystemContainerKubelet: {0.50: 0.10, 0.95: 0.20},
   245  					kubeletstatsv1alpha1.SystemContainerRuntime: {0.50: 0.10, 0.95: 0.20},
   246  				},
   247  				memLimits: e2ekubelet.ResourceUsagePerContainer{
   248  					kubeletstatsv1alpha1.SystemContainerKubelet: &e2ekubelet.ContainerResourceUsage{MemoryRSSInBytes: 200 * 1024 * 1024},
   249  					// The detail can be found at https://github.com/kubernetes/kubernetes/issues/28384#issuecomment-244158892
   250  					kubeletstatsv1alpha1.SystemContainerRuntime: &e2ekubelet.ContainerResourceUsage{MemoryRSSInBytes: 125 * 1024 * 1024},
   251  				},
   252  			},
   253  			{
   254  				cpuLimits: e2ekubelet.ContainersCPUSummary{
   255  					kubeletstatsv1alpha1.SystemContainerKubelet: {0.50: 0.35, 0.95: 0.50},
   256  					kubeletstatsv1alpha1.SystemContainerRuntime: {0.50: 0.10, 0.95: 0.50},
   257  				},
   258  				podsPerNode: 100,
   259  				memLimits: e2ekubelet.ResourceUsagePerContainer{
   260  					kubeletstatsv1alpha1.SystemContainerKubelet: &e2ekubelet.ContainerResourceUsage{MemoryRSSInBytes: 300 * 1024 * 1024},
   261  					kubeletstatsv1alpha1.SystemContainerRuntime: &e2ekubelet.ContainerResourceUsage{MemoryRSSInBytes: 350 * 1024 * 1024},
   262  				},
   263  			},
   264  		}
   265  		for _, testArg := range rTests {
   266  			itArg := testArg
   267  			podsPerNode := itArg.podsPerNode
   268  			name := fmt.Sprintf(
   269  				"resource tracking for %d pods per node", podsPerNode)
   270  			ginkgo.It(name, func(ctx context.Context) {
   271  				runResourceTrackingTest(ctx, f, podsPerNode, nodeNames, rm, itArg.cpuLimits, itArg.memLimits)
   272  			})
   273  		}
   274  	})
   275  	f.Describe("experimental resource usage tracking", feature.ExperimentalResourceUsageTracking, func() {
   276  		density := []int{100}
   277  		for i := range density {
   278  			podsPerNode := density[i]
   279  			name := fmt.Sprintf(
   280  				"resource tracking for %d pods per node", podsPerNode)
   281  			ginkgo.It(name, func(ctx context.Context) {
   282  				runResourceTrackingTest(ctx, f, podsPerNode, nodeNames, rm, nil, nil)
   283  			})
   284  		}
   285  	})
   286  })
   287  
   288  // printPerfData prints the perfdata in json format with PerfResultTag prefix.
   289  // If an error occurs, nothing will be printed.
   290  func printPerfData(p *perftype.PerfData) {
   291  	// Notice that we must make sure the perftype.PerfResultEnd is in a new line.
   292  	if str := framework.PrettyPrintJSON(p); str != "" {
   293  		framework.Logf("%s %s\n%s", perftype.PerfResultTag, str, perftype.PerfResultEnd)
   294  	}
   295  }
   296
View as plain text