metrics_test.go

Documentation: k8s.io/kubernetes/test/integration/client/metrics

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package metrics
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strconv"
    23  	"strings"
    24  	"testing"
    25  
    26  	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    27  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/apiserver/pkg/util/feature"
    30  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    31  	"k8s.io/component-base/metrics/legacyregistry"
    32  	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
    33  	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
    34  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    35  	"k8s.io/kubernetes/test/integration/framework"
    36  
    37  	// the metrics are loaded on cmd/kube-apiserver/apiserver.go
    38  	// so we need to load them here to be available for the test
    39  	_ "k8s.io/component-base/metrics/prometheus/restclient"
    40  )
    41  
    42  // IMPORTANT: metrics are stored globally so all the test must run serially
    43  // and reset the metrics.
    44  
    45  // regression test for https://issues.k8s.io/117258
    46  func TestAPIServerTransportMetrics(t *testing.T) {
    47  	defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)()
    48  	defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)()
    49  
    50  	// reset default registry metrics
    51  	legacyregistry.Reset()
    52  
    53  	result := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins", "ServiceAccount"}, framework.SharedEtcd())
    54  	defer result.TearDownFn()
    55  
    56  	client := clientset.NewForConfigOrDie(result.ClientConfig)
    57  
    58  	// IMPORTANT: reflect the current values if the test changes
    59  	//     client_test.go:1407: metric rest_client_transport_cache_entries 3
    60  	//     client_test.go:1407: metric rest_client_transport_create_calls_total{result="hit"} 61
    61  	//     client_test.go:1407: metric rest_client_transport_create_calls_total{result="miss"} 3
    62  	hits1, misses1, entries1 := checkTransportMetrics(t, client)
    63  	// hit ratio at startup depends on multiple factors
    64  	if (hits1*100)/(hits1+misses1) < 90 {
    65  		t.Fatalf("transport cache hit ratio %d lower than 90 percent", (hits1*100)/(hits1+misses1))
    66  	}
    67  
    68  	aggregatorClient := aggregatorclient.NewForConfigOrDie(result.ClientConfig)
    69  	aggregatedAPI := &apiregistrationv1.APIService{
    70  		ObjectMeta: metav1.ObjectMeta{Name: "v1alpha1.wardle.example.com"},
    71  		Spec: apiregistrationv1.APIServiceSpec{
    72  			Service: &apiregistrationv1.ServiceReference{
    73  				Namespace: "kube-wardle",
    74  				Name:      "api",
    75  			},
    76  			Group:                "wardle.example.com",
    77  			Version:              "v1alpha1",
    78  			GroupPriorityMinimum: 200,
    79  			VersionPriority:      200,
    80  		},
    81  	}
    82  	_, err := aggregatorClient.ApiregistrationV1().APIServices().Create(context.Background(), aggregatedAPI, metav1.CreateOptions{})
    83  	if err != nil {
    84  		t.Fatal(err)
    85  	}
    86  
    87  	requests := 30
    88  	errors := 0
    89  	for i := 0; i < requests; i++ {
    90  		apiService, err := aggregatorClient.ApiregistrationV1().APIServices().Get(context.Background(), "v1alpha1.wardle.example.com", metav1.GetOptions{})
    91  		if err != nil {
    92  			t.Fatal(err)
    93  		}
    94  		// mutate the object
    95  		apiService.Labels = map[string]string{"key": fmt.Sprintf("val%d", i)}
    96  		_, err = aggregatorClient.ApiregistrationV1().APIServices().Update(context.Background(), apiService, metav1.UpdateOptions{})
    97  		if err != nil && !apierrors.IsConflict(err) {
    98  			t.Logf("unexpected error: %v", err)
    99  			errors++
   100  		}
   101  	}
   102  
   103  	if (errors*100)/requests > 20 {
   104  		t.Fatalf("high number of errors during the test %d out of %d", errors, requests)
   105  	}
   106  
   107  	// IMPORTANT: reflect the current values if the test changes
   108  	//     client_test.go:1407: metric rest_client_transport_cache_entries 4
   109  	//     client_test.go:1407: metric rest_client_transport_create_calls_total{result="hit"} 120
   110  	//     client_test.go:1407: metric rest_client_transport_create_calls_total{result="miss"} 4
   111  	hits2, misses2, entries2 := checkTransportMetrics(t, client)
   112  	if entries2-entries1 > 10 {
   113  		t.Fatalf("possible transport leak, number of new cache entries increased by %d", entries2-entries1)
   114  	}
   115  
   116  	// hit ratio after startup should grow since no new transports are expected
   117  	if (hits2*100)/(hits2+misses2) < 95 {
   118  		t.Fatalf("transport cache hit ratio %d lower than 95 percent", (hits2*100)/(hits2+misses2))
   119  	}
   120  }
   121  
   122  func checkTransportMetrics(t *testing.T, client *clientset.Clientset) (hits int, misses int, entries int) {
   123  	t.Helper()
   124  	body, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.Background())
   125  	if err != nil {
   126  		t.Fatal(err)
   127  	}
   128  
   129  	// TODO: this can be much better if there is some library that parse prometheus metrics
   130  	// the existing one in "k8s.io/component-base/metrics/testutil" uses the global variable
   131  	// but we want to parse the ones returned by the endpoint to be sure the metrics are
   132  	// exposed correctly
   133  	for _, line := range strings.Split(string(body), "\n") {
   134  		if !strings.HasPrefix(line, "rest_client_transport") {
   135  			continue
   136  		}
   137  		if strings.Contains(line, "uncacheable") {
   138  			t.Fatalf("detected transport that is not cacheable, please check https://issues.k8s.io/112017")
   139  		}
   140  
   141  		output := strings.Split(line, " ")
   142  		if len(output) != 2 {
   143  			t.Fatalf("expected metrics to be in the format name value, got %v", output)
   144  		}
   145  		name := output[0]
   146  		value, err := strconv.Atoi(output[1])
   147  		if err != nil {
   148  			t.Fatalf("metric value can not be converted to integer %v", err)
   149  		}
   150  		switch name {
   151  		case "rest_client_transport_cache_entries":
   152  			entries = value
   153  		case `rest_client_transport_create_calls_total{result="hit"}`:
   154  			hits = value
   155  		case `rest_client_transport_create_calls_total{result="miss"}`:
   156  			misses = value
   157  		}
   158  		t.Logf("metric %s", line)
   159  	}
   160  
   161  	if misses != entries || misses == 0 {
   162  		t.Errorf("expected as many entries %d in the cache as misses, got %d", entries, misses)
   163  	}
   164  
   165  	if hits < misses {
   166  		t.Errorf("expected more hits %d in the cache than misses %d", hits, misses)
   167  	}
   168  	return
   169  }
   170
View as plain text