Source file src/k8s.io/kubernetes/test/integration/metrics/metrics_test.go

Documentation: k8s.io/kubernetes/test/integration/metrics

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package metrics
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"runtime"
    24  	"testing"
    25  
    26  	"github.com/prometheus/common/model"
    27  	v1 "k8s.io/api/core/v1"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	clientset "k8s.io/client-go/kubernetes"
    30  	restclient "k8s.io/client-go/rest"
    31  	"k8s.io/component-base/metrics/testutil"
    32  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    33  	"k8s.io/kubernetes/test/integration/framework"
    34  )
    35  
    36  func scrapeMetrics(s *kubeapiservertesting.TestServer) (testutil.Metrics, error) {
    37  	client, err := clientset.NewForConfig(s.ClientConfig)
    38  	if err != nil {
    39  		return nil, fmt.Errorf("couldn't create client")
    40  	}
    41  
    42  	body, err := client.RESTClient().Get().AbsPath("metrics").DoRaw(context.TODO())
    43  	if err != nil {
    44  		return nil, fmt.Errorf("request failed: %v", err)
    45  	}
    46  	metrics := testutil.NewMetrics()
    47  	err = testutil.ParseMetrics(string(body), &metrics)
    48  	return metrics, err
    49  }
    50  
    51  func checkForExpectedMetrics(t *testing.T, metrics testutil.Metrics, expectedMetrics []string) {
    52  	for _, expected := range expectedMetrics {
    53  		if _, found := metrics[expected]; !found {
    54  			t.Errorf("API server metrics did not include expected metric %q", expected)
    55  		}
    56  	}
    57  }
    58  
    59  func TestAPIServerProcessMetrics(t *testing.T) {
    60  	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
    61  		t.Skipf("not supported on GOOS=%s", runtime.GOOS)
    62  	}
    63  
    64  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
    65  	defer s.TearDownFn()
    66  
    67  	metrics, err := scrapeMetrics(s)
    68  	if err != nil {
    69  		t.Fatal(err)
    70  	}
    71  	checkForExpectedMetrics(t, metrics, []string{
    72  		"process_start_time_seconds",
    73  		"process_cpu_seconds_total",
    74  		"process_open_fds",
    75  		"process_resident_memory_bytes",
    76  	})
    77  }
    78  
    79  func TestAPIServerStorageMetrics(t *testing.T) {
    80  	config := framework.SharedEtcd()
    81  	config.Transport.ServerList = []string{config.Transport.ServerList[0], config.Transport.ServerList[0]}
    82  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, config)
    83  	defer s.TearDownFn()
    84  
    85  	metrics, err := scrapeMetrics(s)
    86  	if err != nil {
    87  		t.Fatal(err)
    88  	}
    89  
    90  	samples, ok := metrics["apiserver_storage_size_bytes"]
    91  	if !ok {
    92  		t.Fatalf("apiserver_storage_size_bytes metric not exposed")
    93  	}
    94  	if len(samples) != 1 {
    95  		t.Fatalf("Unexpected number of samples in apiserver_storage_size_bytes")
    96  	}
    97  
    98  	if samples[0].Value == -1 {
    99  		t.Errorf("Unexpected non-zero apiserver_storage_size_bytes, got: %s", samples[0].Value)
   100  	}
   101  }
   102  
   103  func TestAPIServerMetrics(t *testing.T) {
   104  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
   105  	defer s.TearDownFn()
   106  
   107  	// Make a request to the apiserver to ensure there's at least one data point
   108  	// for the metrics we're expecting -- otherwise, they won't be exported.
   109  	client := clientset.NewForConfigOrDie(s.ClientConfig)
   110  	if _, err := client.CoreV1().Pods(metav1.NamespaceDefault).List(context.TODO(), metav1.ListOptions{}); err != nil {
   111  		t.Fatalf("unexpected error getting pods: %v", err)
   112  	}
   113  
   114  	// Make a request to a deprecated API to ensure there's at least one data point
   115  	if _, err := client.FlowcontrolV1beta3().FlowSchemas().List(context.TODO(), metav1.ListOptions{}); err != nil {
   116  		t.Fatalf("unexpected error: %v", err)
   117  	}
   118  
   119  	metrics, err := scrapeMetrics(s)
   120  	if err != nil {
   121  		t.Fatal(err)
   122  	}
   123  	checkForExpectedMetrics(t, metrics, []string{
   124  		"apiserver_requested_deprecated_apis",
   125  		"apiserver_request_total",
   126  		"apiserver_request_duration_seconds_sum",
   127  		"etcd_request_duration_seconds_sum",
   128  	})
   129  }
   130  
   131  func TestAPIServerMetricsLabels(t *testing.T) {
   132  	// Disable ServiceAccount admission plugin as we don't have service account controller running.
   133  	s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd())
   134  	defer s.TearDownFn()
   135  
   136  	clientConfig := restclient.CopyConfig(s.ClientConfig)
   137  	clientConfig.QPS = -1
   138  	client, err := clientset.NewForConfig(clientConfig)
   139  	if err != nil {
   140  		t.Fatalf("Error in create clientset: %v", err)
   141  	}
   142  
   143  	expectedMetrics := []model.Metric{}
   144  
   145  	metricLabels := func(group, version, resource, subresource, scope, verb string) model.Metric {
   146  		return map[model.LabelName]model.LabelValue{
   147  			model.LabelName("group"):       model.LabelValue(group),
   148  			model.LabelName("version"):     model.LabelValue(version),
   149  			model.LabelName("resource"):    model.LabelValue(resource),
   150  			model.LabelName("subresource"): model.LabelValue(subresource),
   151  			model.LabelName("scope"):       model.LabelValue(scope),
   152  			model.LabelName("verb"):        model.LabelValue(verb),
   153  		}
   154  	}
   155  
   156  	callOrDie := func(_ interface{}, err error) {
   157  		if err != nil {
   158  			t.Fatalf("unexpected error: %v", err)
   159  		}
   160  	}
   161  
   162  	appendExpectedMetric := func(metric model.Metric) {
   163  		expectedMetrics = append(expectedMetrics, metric)
   164  	}
   165  
   166  	// Call appropriate endpoints to ensure particular metrics will be exposed
   167  
   168  	// Namespace-scoped resource
   169  	c := client.CoreV1().Pods(metav1.NamespaceDefault)
   170  	makePod := func(labelValue string) *v1.Pod {
   171  		return &v1.Pod{
   172  			ObjectMeta: metav1.ObjectMeta{
   173  				Name:   "foo",
   174  				Labels: map[string]string{"foo": labelValue},
   175  			},
   176  			Spec: v1.PodSpec{
   177  				Containers: []v1.Container{
   178  					{
   179  						Name:  "container",
   180  						Image: "image",
   181  					},
   182  				},
   183  			},
   184  		}
   185  	}
   186  
   187  	callOrDie(c.Create(context.TODO(), makePod("foo"), metav1.CreateOptions{}))
   188  	appendExpectedMetric(metricLabels("", "v1", "pods", "", "resource", "POST"))
   189  	callOrDie(c.Update(context.TODO(), makePod("bar"), metav1.UpdateOptions{}))
   190  	appendExpectedMetric(metricLabels("", "v1", "pods", "", "resource", "PUT"))
   191  	callOrDie(c.UpdateStatus(context.TODO(), makePod("bar"), metav1.UpdateOptions{}))
   192  	appendExpectedMetric(metricLabels("", "v1", "pods", "status", "resource", "PUT"))
   193  	callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{}))
   194  	appendExpectedMetric(metricLabels("", "v1", "pods", "", "resource", "GET"))
   195  	callOrDie(c.List(context.TODO(), metav1.ListOptions{}))
   196  	appendExpectedMetric(metricLabels("", "v1", "pods", "", "namespace", "LIST"))
   197  	callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{}))
   198  	appendExpectedMetric(metricLabels("", "v1", "pods", "", "resource", "DELETE"))
   199  	// cluster-scoped LIST of namespace-scoped resources
   200  	callOrDie(client.CoreV1().Pods(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}))
   201  	appendExpectedMetric(metricLabels("", "v1", "pods", "", "cluster", "LIST"))
   202  
   203  	// Cluster-scoped resource
   204  	cn := client.CoreV1().Namespaces()
   205  	makeNamespace := func(labelValue string) *v1.Namespace {
   206  		return &v1.Namespace{
   207  			ObjectMeta: metav1.ObjectMeta{
   208  				Name:   "foo",
   209  				Labels: map[string]string{"foo": labelValue},
   210  			},
   211  		}
   212  	}
   213  
   214  	callOrDie(cn.Create(context.TODO(), makeNamespace("foo"), metav1.CreateOptions{}))
   215  	appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "resource", "POST"))
   216  	callOrDie(cn.Update(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{}))
   217  	appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "resource", "PUT"))
   218  	callOrDie(cn.UpdateStatus(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{}))
   219  	appendExpectedMetric(metricLabels("", "v1", "namespaces", "status", "resource", "PUT"))
   220  	callOrDie(cn.Get(context.TODO(), "foo", metav1.GetOptions{}))
   221  	appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "resource", "GET"))
   222  	callOrDie(cn.List(context.TODO(), metav1.ListOptions{}))
   223  	appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "cluster", "LIST"))
   224  	callOrDie(nil, cn.Delete(context.TODO(), "foo", metav1.DeleteOptions{}))
   225  	appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "resource", "DELETE"))
   226  
   227  	// Verify if all metrics were properly exported.
   228  	metrics, err := scrapeMetrics(s)
   229  	if err != nil {
   230  		t.Fatal(err)
   231  	}
   232  
   233  	samples, ok := metrics["apiserver_request_total"]
   234  	if !ok {
   235  		t.Fatalf("apiserver_request_total metric not exposed")
   236  	}
   237  
   238  	hasLabels := func(current, expected model.Metric) bool {
   239  		for key, value := range expected {
   240  			if current[key] != value {
   241  				return false
   242  			}
   243  		}
   244  		return true
   245  	}
   246  
   247  	for _, expectedMetric := range expectedMetrics {
   248  		found := false
   249  		for _, sample := range samples {
   250  			if hasLabels(sample.Metric, expectedMetric) {
   251  				found = true
   252  				break
   253  			}
   254  		}
   255  		if !found {
   256  			t.Errorf("No sample found for %#v", expectedMetric)
   257  		}
   258  	}
   259  }
   260  
   261  func TestAPIServerMetricsPods(t *testing.T) {
   262  	callOrDie := func(_ interface{}, err error) {
   263  		if err != nil {
   264  			t.Fatalf("unexpected error: %v", err)
   265  		}
   266  	}
   267  
   268  	makePod := func(labelValue string) *v1.Pod {
   269  		return &v1.Pod{
   270  			ObjectMeta: metav1.ObjectMeta{
   271  				Name:   "foo",
   272  				Labels: map[string]string{"foo": labelValue},
   273  			},
   274  			Spec: v1.PodSpec{
   275  				Containers: []v1.Container{
   276  					{
   277  						Name:  "container",
   278  						Image: "image",
   279  					},
   280  				},
   281  			},
   282  		}
   283  	}
   284  
   285  	// Disable ServiceAccount admission plugin as we don't have service account controller running.
   286  	server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd())
   287  	defer server.TearDownFn()
   288  
   289  	clientConfig := restclient.CopyConfig(server.ClientConfig)
   290  	clientConfig.QPS = -1
   291  	client, err := clientset.NewForConfig(clientConfig)
   292  	if err != nil {
   293  		t.Fatalf("Error in create clientset: %v", err)
   294  	}
   295  
   296  	c := client.CoreV1().Pods(metav1.NamespaceDefault)
   297  
   298  	for _, tc := range []struct {
   299  		name     string
   300  		executor func()
   301  
   302  		want string
   303  	}{
   304  		{
   305  			name: "create pod",
   306  			executor: func() {
   307  				callOrDie(c.Create(context.TODO(), makePod("foo"), metav1.CreateOptions{}))
   308  			},
   309  			want: `apiserver_request_total{code="201", component="apiserver", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="POST", version="v1"}`,
   310  		},
   311  		{
   312  			name: "update pod",
   313  			executor: func() {
   314  				callOrDie(c.Update(context.TODO(), makePod("bar"), metav1.UpdateOptions{}))
   315  			},
   316  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="PUT", version="v1"}`,
   317  		},
   318  		{
   319  			name: "update pod status",
   320  			executor: func() {
   321  				callOrDie(c.UpdateStatus(context.TODO(), makePod("bar"), metav1.UpdateOptions{}))
   322  			},
   323  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="pods", scope="resource", subresource="status", verb="PUT", version="v1"}`,
   324  		},
   325  		{
   326  			name: "get pod",
   327  			executor: func() {
   328  				callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{}))
   329  			},
   330  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="GET", version="v1"}`,
   331  		},
   332  		{
   333  			name: "list pod",
   334  			executor: func() {
   335  				callOrDie(c.List(context.TODO(), metav1.ListOptions{}))
   336  			},
   337  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="pods", scope="namespace", subresource="", verb="LIST", version="v1"}`,
   338  		},
   339  		{
   340  			name: "delete pod",
   341  			executor: func() {
   342  				callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{}))
   343  			},
   344  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="DELETE", version="v1"}`,
   345  		},
   346  	} {
   347  		t.Run(tc.name, func(t *testing.T) {
   348  
   349  			baseSamples, err := getSamples(server)
   350  			if err != nil {
   351  				t.Fatal(err)
   352  			}
   353  
   354  			tc.executor()
   355  
   356  			updatedSamples, err := getSamples(server)
   357  			if err != nil {
   358  				t.Fatal(err)
   359  			}
   360  
   361  			newSamples := diffMetrics(updatedSamples, baseSamples)
   362  			found := false
   363  
   364  			for _, sample := range newSamples {
   365  				if sample.Metric.String() == tc.want {
   366  					found = true
   367  					break
   368  				}
   369  			}
   370  
   371  			if !found {
   372  				t.Fatalf("could not find metric for API call >%s< among samples >%+v<", tc.name, newSamples)
   373  			}
   374  		})
   375  	}
   376  }
   377  
   378  func TestAPIServerMetricsNamespaces(t *testing.T) {
   379  	callOrDie := func(_ interface{}, err error) {
   380  		if err != nil {
   381  			t.Fatalf("unexpected error: %v", err)
   382  		}
   383  	}
   384  
   385  	makeNamespace := func(labelValue string) *v1.Namespace {
   386  		return &v1.Namespace{
   387  			ObjectMeta: metav1.ObjectMeta{
   388  				Name:   "foo",
   389  				Labels: map[string]string{"foo": labelValue},
   390  			},
   391  		}
   392  	}
   393  
   394  	server := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
   395  	defer server.TearDownFn()
   396  
   397  	clientConfig := restclient.CopyConfig(server.ClientConfig)
   398  	clientConfig.QPS = -1
   399  	client, err := clientset.NewForConfig(clientConfig)
   400  	if err != nil {
   401  		t.Fatalf("Error in create clientset: %v", err)
   402  	}
   403  
   404  	c := client.CoreV1().Namespaces()
   405  
   406  	for _, tc := range []struct {
   407  		name     string
   408  		executor func()
   409  
   410  		want string
   411  	}{
   412  		{
   413  			name: "create namespace",
   414  			executor: func() {
   415  				callOrDie(c.Create(context.TODO(), makeNamespace("foo"), metav1.CreateOptions{}))
   416  			},
   417  			want: `apiserver_request_total{code="201", component="apiserver", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="POST", version="v1"}`,
   418  		},
   419  		{
   420  			name: "update namespace",
   421  			executor: func() {
   422  				callOrDie(c.Update(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{}))
   423  			},
   424  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="PUT", version="v1"}`,
   425  		},
   426  		{
   427  			name: "update namespace status",
   428  			executor: func() {
   429  				callOrDie(c.UpdateStatus(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{}))
   430  			},
   431  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="namespaces", scope="resource", subresource="status", verb="PUT", version="v1"}`,
   432  		},
   433  		{
   434  			name: "get namespace",
   435  			executor: func() {
   436  				callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{}))
   437  			},
   438  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="GET", version="v1"}`,
   439  		},
   440  		{
   441  			name: "list namespace",
   442  			executor: func() {
   443  				callOrDie(c.List(context.TODO(), metav1.ListOptions{}))
   444  			},
   445  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="namespaces", scope="cluster", subresource="", verb="LIST", version="v1"}`,
   446  		},
   447  		{
   448  			name: "delete namespace",
   449  			executor: func() {
   450  				callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{}))
   451  			},
   452  			want: `apiserver_request_total{code="200", component="apiserver", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="DELETE", version="v1"}`,
   453  		},
   454  	} {
   455  		t.Run(tc.name, func(t *testing.T) {
   456  
   457  			baseSamples, err := getSamples(server)
   458  			if err != nil {
   459  				t.Fatal(err)
   460  			}
   461  
   462  			tc.executor()
   463  
   464  			updatedSamples, err := getSamples(server)
   465  			if err != nil {
   466  				t.Fatal(err)
   467  			}
   468  
   469  			newSamples := diffMetrics(updatedSamples, baseSamples)
   470  			found := false
   471  
   472  			for _, sample := range newSamples {
   473  				if sample.Metric.String() == tc.want {
   474  					found = true
   475  					break
   476  				}
   477  			}
   478  
   479  			if !found {
   480  				t.Fatalf("could not find metric for API call >%s< among samples >%+v<", tc.name, newSamples)
   481  			}
   482  		})
   483  	}
   484  }
   485  
   486  func getSamples(s *kubeapiservertesting.TestServer) (model.Samples, error) {
   487  	metrics, err := scrapeMetrics(s)
   488  	if err != nil {
   489  		return nil, err
   490  	}
   491  
   492  	samples, ok := metrics["apiserver_request_total"]
   493  	if !ok {
   494  		return nil, errors.New("apiserver_request_total doesn't exist")
   495  	}
   496  	return samples, nil
   497  }
   498  
   499  func diffMetrics(newSamples model.Samples, oldSamples model.Samples) model.Samples {
   500  	samplesDiff := model.Samples{}
   501  	for _, sample := range newSamples {
   502  		if !sampleExistsInSamples(sample, oldSamples) {
   503  			samplesDiff = append(samplesDiff, sample)
   504  		}
   505  	}
   506  	return samplesDiff
   507  }
   508  
   509  func sampleExistsInSamples(s *model.Sample, samples model.Samples) bool {
   510  	for _, sample := range samples {
   511  		if sample.Equal(s) {
   512  			return true
   513  		}
   514  	}
   515  	return false
   516  }
   517  

View as plain text