1
16
17 package metrics
18
19 import (
20 "context"
21 "fmt"
22 "testing"
23 "time"
24
25 autoscalingapi "k8s.io/api/autoscaling/v2"
26 v1 "k8s.io/api/core/v1"
27 "k8s.io/apimachinery/pkg/api/meta/testrestmapper"
28 "k8s.io/apimachinery/pkg/api/resource"
29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30 "k8s.io/apimachinery/pkg/labels"
31 "k8s.io/apimachinery/pkg/runtime"
32 "k8s.io/apimachinery/pkg/runtime/schema"
33 core "k8s.io/client-go/testing"
34 "k8s.io/kubernetes/pkg/api/legacyscheme"
35 _ "k8s.io/kubernetes/pkg/apis/apps/install"
36 cmapi "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2"
37 emapi "k8s.io/metrics/pkg/apis/external_metrics/v1beta1"
38 metricsapi "k8s.io/metrics/pkg/apis/metrics/v1beta1"
39 metricsfake "k8s.io/metrics/pkg/client/clientset/versioned/fake"
40 cmfake "k8s.io/metrics/pkg/client/custom_metrics/fake"
41 emfake "k8s.io/metrics/pkg/client/external_metrics/fake"
42
43 "github.com/stretchr/testify/assert"
44 )
45
46 var fixedTimestamp = time.Date(2015, time.November, 10, 12, 30, 0, 0, time.UTC)
47
48
49 type metricPoint struct {
50 level uint64
51 timestamp int
52 }
53
54 type restClientTestCase struct {
55 desiredMetricValues PodMetricsInfo
56 desiredError error
57
58
59 targetTimestamp int
60 window time.Duration
61 reportedMetricPoints []metricPoint
62 reportedPodMetrics []map[string]int64
63 singleObject *autoscalingapi.CrossVersionObjectReference
64
65 namespace string
66 selector labels.Selector
67 resourceName v1.ResourceName
68 container string
69 metricName string
70 metricSelector *metav1.LabelSelector
71 metricLabelSelector labels.Selector
72 }
73
74 func (tc *restClientTestCase) prepareTestClient(t *testing.T) (*metricsfake.Clientset, *cmfake.FakeCustomMetricsClient, *emfake.FakeExternalMetricsClient) {
75 namespace := "test-namespace"
76 tc.namespace = namespace
77 podNamePrefix := "test-pod"
78 podLabels := map[string]string{"name": podNamePrefix}
79 tc.selector = labels.SelectorFromSet(podLabels)
80
81
82 isResource := len(tc.resourceName) > 0
83
84 isExternal := tc.metricSelector != nil
85
86 fakeMetricsClient := &metricsfake.Clientset{}
87 fakeCMClient := &cmfake.FakeCustomMetricsClient{}
88 fakeEMClient := &emfake.FakeExternalMetricsClient{}
89
90 if isResource {
91 fakeMetricsClient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
92 metrics := &metricsapi.PodMetricsList{}
93 for i, containers := range tc.reportedPodMetrics {
94 metric := metricsapi.PodMetrics{
95 ObjectMeta: metav1.ObjectMeta{
96 Name: fmt.Sprintf("%s-%d", podNamePrefix, i),
97 Namespace: namespace,
98 Labels: podLabels,
99 },
100 Timestamp: metav1.Time{Time: offsetTimestampBy(tc.targetTimestamp)},
101 Window: metav1.Duration{Duration: tc.window},
102 Containers: []metricsapi.ContainerMetrics{},
103 }
104 for containerName, cpu := range containers {
105 cm := metricsapi.ContainerMetrics{
106 Name: containerName,
107 Usage: v1.ResourceList{
108 v1.ResourceCPU: *resource.NewMilliQuantity(
109 cpu,
110 resource.DecimalSI),
111 v1.ResourceMemory: *resource.NewQuantity(
112 int64(1024*1024),
113 resource.BinarySI),
114 },
115 }
116 metric.Containers = append(metric.Containers, cm)
117 }
118 metrics.Items = append(metrics.Items, metric)
119 }
120 return true, metrics, nil
121 })
122 } else if isExternal {
123 fakeEMClient.AddReactor("list", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
124 listAction := action.(core.ListAction)
125 assert.Equal(t, tc.metricName, listAction.GetResource().Resource, "the metric requested should have matched the one specified.")
126 assert.Equal(t, tc.metricLabelSelector, listAction.GetListRestrictions().Labels, "the metric selector should have matched the one specified")
127
128 metrics := emapi.ExternalMetricValueList{}
129 for _, metricPoint := range tc.reportedMetricPoints {
130 timestamp := offsetTimestampBy(metricPoint.timestamp)
131 metric := emapi.ExternalMetricValue{
132 Value: *resource.NewMilliQuantity(int64(metricPoint.level), resource.DecimalSI),
133 Timestamp: metav1.Time{Time: timestamp},
134 MetricName: tc.metricName,
135 }
136 metrics.Items = append(metrics.Items, metric)
137 }
138 return true, &metrics, nil
139 })
140 } else {
141 fakeCMClient.AddReactor("get", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
142 getForAction := action.(cmfake.GetForAction)
143 assert.Equal(t, tc.metricName, getForAction.GetMetricName(), "the metric requested should have matched the one specified")
144
145 if getForAction.GetName() == "*" {
146
147 metrics := cmapi.MetricValueList{}
148 assert.Equal(t, "pods", getForAction.GetResource().Resource, "type of object that we requested multiple metrics for should have been pods")
149
150 for i, metricPoint := range tc.reportedMetricPoints {
151 timestamp := offsetTimestampBy(metricPoint.timestamp)
152 metric := cmapi.MetricValue{
153 DescribedObject: v1.ObjectReference{
154 Kind: "Pod",
155 APIVersion: "v1",
156 Name: fmt.Sprintf("%s-%d", podNamePrefix, i),
157 },
158 Value: *resource.NewMilliQuantity(int64(metricPoint.level), resource.DecimalSI),
159 Timestamp: metav1.Time{Time: timestamp},
160 Metric: cmapi.MetricIdentifier{
161 Name: tc.metricName,
162 },
163 }
164
165 metrics.Items = append(metrics.Items, metric)
166 }
167
168 return true, &metrics, nil
169 } else {
170 name := getForAction.GetName()
171 mapper := testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme)
172 assert.NotNil(t, tc.singleObject, "should have only requested a single-object metric when we asked for metrics for a single object")
173 gk := schema.FromAPIVersionAndKind(tc.singleObject.APIVersion, tc.singleObject.Kind).GroupKind()
174 mapping, err := mapper.RESTMapping(gk)
175 if err != nil {
176 return true, nil, fmt.Errorf("unable to get mapping for %s: %v", gk.String(), err)
177 }
178 groupResource := mapping.Resource.GroupResource()
179
180 assert.Equal(t, groupResource.String(), getForAction.GetResource().Resource, "should have requested metrics for the resource matching the GroupKind passed in")
181 assert.Equal(t, tc.singleObject.Name, name, "should have requested metrics for the object matching the name passed in")
182 metricPoint := tc.reportedMetricPoints[0]
183 timestamp := offsetTimestampBy(metricPoint.timestamp)
184
185 metrics := &cmapi.MetricValueList{
186 Items: []cmapi.MetricValue{
187 {
188 DescribedObject: v1.ObjectReference{
189 Kind: tc.singleObject.Kind,
190 APIVersion: tc.singleObject.APIVersion,
191 Name: tc.singleObject.Name,
192 },
193 Timestamp: metav1.Time{Time: timestamp},
194 Metric: cmapi.MetricIdentifier{
195 Name: tc.metricName,
196 },
197 Value: *resource.NewMilliQuantity(int64(metricPoint.level), resource.DecimalSI),
198 },
199 },
200 }
201
202 return true, metrics, nil
203 }
204 })
205 }
206
207 return fakeMetricsClient, fakeCMClient, fakeEMClient
208 }
209
210 func (tc *restClientTestCase) verifyResults(t *testing.T, metrics PodMetricsInfo, timestamp time.Time, err error) {
211 if tc.desiredError != nil {
212 assert.Error(t, err, "there should be an error retrieving the metrics")
213 assert.Contains(t, fmt.Sprintf("%v", err), fmt.Sprintf("%v", tc.desiredError), "the error message should be as expected")
214 return
215 }
216 assert.NoError(t, err, "there should be no error retrieving the metrics")
217 assert.NotNil(t, metrics, "there should be metrics returned")
218
219 if len(metrics) != len(tc.desiredMetricValues) {
220 t.Errorf("Not equal:\nexpected: %v\nactual: %v", tc.desiredMetricValues, metrics)
221 } else {
222 for k, m := range metrics {
223 if !m.Timestamp.Equal(tc.desiredMetricValues[k].Timestamp) ||
224 m.Window != tc.desiredMetricValues[k].Window ||
225 m.Value != tc.desiredMetricValues[k].Value {
226 t.Errorf("Not equal:\nexpected: %v\nactual: %v", tc.desiredMetricValues, metrics)
227 break
228 }
229 }
230 }
231
232 targetTimestamp := offsetTimestampBy(tc.targetTimestamp)
233 assert.True(t, targetTimestamp.Equal(timestamp), fmt.Sprintf("the timestamp should be as expected (%s) but was %s", targetTimestamp, timestamp))
234 }
235
236 func (tc *restClientTestCase) runTest(t *testing.T) {
237 var err error
238 testMetricsClient, testCMClient, testEMClient := tc.prepareTestClient(t)
239 metricsClient := NewRESTMetricsClient(testMetricsClient.MetricsV1beta1(), testCMClient, testEMClient)
240 isResource := len(tc.resourceName) > 0
241 isExternal := tc.metricSelector != nil
242 if isResource {
243 info, timestamp, err := metricsClient.GetResourceMetric(context.TODO(), v1.ResourceName(tc.resourceName), tc.namespace, tc.selector, tc.container)
244 tc.verifyResults(t, info, timestamp, err)
245 } else if isExternal {
246 tc.metricLabelSelector, err = metav1.LabelSelectorAsSelector(tc.metricSelector)
247 if err != nil {
248 t.Errorf("invalid metric selector: %+v", tc.metricSelector)
249 }
250 val, timestamp, err := metricsClient.GetExternalMetric(tc.metricName, tc.namespace, tc.metricLabelSelector)
251 info := make(PodMetricsInfo, len(val))
252 for i, metricVal := range val {
253 info[fmt.Sprintf("%v-val-%v", tc.metricName, i)] = PodMetric{Value: metricVal}
254 }
255 tc.verifyResults(t, info, timestamp, err)
256 } else if tc.singleObject == nil {
257 info, timestamp, err := metricsClient.GetRawMetric(tc.metricName, tc.namespace, tc.selector, tc.metricLabelSelector)
258 tc.verifyResults(t, info, timestamp, err)
259 } else {
260 val, timestamp, err := metricsClient.GetObjectMetric(tc.metricName, tc.namespace, tc.singleObject, tc.metricLabelSelector)
261 info := PodMetricsInfo{tc.singleObject.Name: {Value: val}}
262 tc.verifyResults(t, info, timestamp, err)
263 }
264 }
265
266 func TestRESTClientPodCPU(t *testing.T) {
267 targetTimestamp := 1
268 window := 30 * time.Second
269 tc := restClientTestCase{
270 desiredMetricValues: PodMetricsInfo{
271 "test-pod-0": {Value: 5000, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
272 "test-pod-1": {Value: 5000, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
273 "test-pod-2": {Value: 5000, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
274 },
275 resourceName: v1.ResourceCPU,
276 targetTimestamp: targetTimestamp,
277 window: window,
278 reportedPodMetrics: []map[string]int64{{"test": 5000}, {"test": 5000}, {"test": 5000}},
279 }
280 tc.runTest(t)
281 }
282
283 func TestRESTClientContainerCPU(t *testing.T) {
284 targetTimestamp := 1
285 window := 30 * time.Second
286 tc := restClientTestCase{
287 desiredMetricValues: PodMetricsInfo{
288 "test-pod-0": {Value: 5000, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
289 "test-pod-1": {Value: 5000, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
290 "test-pod-2": {Value: 5000, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
291 },
292 container: "test-1",
293 resourceName: v1.ResourceCPU,
294 targetTimestamp: targetTimestamp,
295 window: window,
296 reportedPodMetrics: []map[string]int64{{"test-1": 5000, "test-2": 500}, {"test-1": 5000, "test-2": 500}, {"test-1": 5000, "test-2": 500}},
297 }
298 tc.runTest(t)
299 }
300
301 func TestRESTClientExternal(t *testing.T) {
302 tc := restClientTestCase{
303 desiredMetricValues: PodMetricsInfo{
304 "external-val-0": {Value: 10000}, "external-val-1": {Value: 20000}, "external-val-2": {Value: 10000},
305 },
306 metricSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"label": "value"}},
307 metricName: "external",
308 targetTimestamp: 1,
309 reportedMetricPoints: []metricPoint{{10000, 1}, {20000, 1}, {10000, 1}},
310 }
311 tc.runTest(t)
312 }
313
314 func TestRESTClientQPS(t *testing.T) {
315 targetTimestamp := 1
316 tc := restClientTestCase{
317 desiredMetricValues: PodMetricsInfo{
318 "test-pod-0": {Value: 10000, Timestamp: offsetTimestampBy(targetTimestamp), Window: metricServerDefaultMetricWindow},
319 "test-pod-1": {Value: 20000, Timestamp: offsetTimestampBy(targetTimestamp), Window: metricServerDefaultMetricWindow},
320 "test-pod-2": {Value: 10000, Timestamp: offsetTimestampBy(targetTimestamp), Window: metricServerDefaultMetricWindow},
321 },
322 metricName: "qps",
323 targetTimestamp: targetTimestamp,
324 reportedMetricPoints: []metricPoint{{10000, 1}, {20000, 1}, {10000, 1}},
325 }
326 tc.runTest(t)
327 }
328
329 func TestRESTClientSingleObject(t *testing.T) {
330 tc := restClientTestCase{
331 desiredMetricValues: PodMetricsInfo{"some-dep": {Value: 10}},
332 metricName: "queue-length",
333 targetTimestamp: 1,
334 reportedMetricPoints: []metricPoint{{10, 1}},
335 singleObject: &autoscalingapi.CrossVersionObjectReference{
336 APIVersion: "apps/v1",
337 Kind: "Deployment",
338 Name: "some-dep",
339 },
340 }
341 tc.runTest(t)
342 }
343
344 func TestRESTClientQpsSumEqualZero(t *testing.T) {
345 targetTimestamp := 0
346 tc := restClientTestCase{
347 desiredMetricValues: PodMetricsInfo{
348 "test-pod-0": {Value: 0, Timestamp: offsetTimestampBy(targetTimestamp), Window: metricServerDefaultMetricWindow},
349 "test-pod-1": {Value: 0, Timestamp: offsetTimestampBy(targetTimestamp), Window: metricServerDefaultMetricWindow},
350 "test-pod-2": {Value: 0, Timestamp: offsetTimestampBy(targetTimestamp), Window: metricServerDefaultMetricWindow},
351 },
352 metricName: "qps",
353 targetTimestamp: targetTimestamp,
354 reportedMetricPoints: []metricPoint{{0, 0}, {0, 0}, {0, 0}},
355 }
356 tc.runTest(t)
357 }
358
359 func TestRESTClientExternalSumEqualZero(t *testing.T) {
360 tc := restClientTestCase{
361 desiredMetricValues: PodMetricsInfo{
362 "external-val-0": {Value: 0}, "external-val-1": {Value: 0}, "external-val-2": {Value: 0},
363 },
364 metricSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"label": "value"}},
365 metricName: "external",
366 targetTimestamp: 0,
367 reportedMetricPoints: []metricPoint{{0, 0}, {0, 0}, {0, 0}},
368 }
369 tc.runTest(t)
370 }
371
372 func TestRESTClientQpsEmptyMetrics(t *testing.T) {
373 tc := restClientTestCase{
374 metricName: "qps",
375 desiredError: fmt.Errorf("no metrics returned from custom metrics API"),
376 reportedMetricPoints: []metricPoint{},
377 }
378
379 tc.runTest(t)
380 }
381
382 func TestRESTClientExternalEmptyMetrics(t *testing.T) {
383 tc := restClientTestCase{
384 metricName: "external",
385 metricSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"label": "value"}},
386 desiredError: fmt.Errorf("no metrics returned from external metrics API"),
387 reportedMetricPoints: []metricPoint{},
388 }
389
390 tc.runTest(t)
391 }
392
393 func TestRESTClientPodCPUEmptyMetrics(t *testing.T) {
394 tc := restClientTestCase{
395 resourceName: v1.ResourceCPU,
396 desiredError: fmt.Errorf("no metrics returned from resource metrics API"),
397 reportedMetricPoints: []metricPoint{},
398 reportedPodMetrics: []map[string]int64{},
399 }
400 tc.runTest(t)
401 }
402
403 func TestRESTClientPodCPUEmptyMetricsForOnePod(t *testing.T) {
404 targetTimestamp := 1
405 window := 30 * time.Second
406 tc := restClientTestCase{
407 resourceName: v1.ResourceCPU,
408 desiredMetricValues: PodMetricsInfo{
409 "test-pod-0": {Value: 100, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
410 "test-pod-1": {Value: 700, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
411 },
412 targetTimestamp: targetTimestamp,
413 window: window,
414 reportedPodMetrics: []map[string]int64{{"test-1": 100}, {"test-1": 300, "test-2": 400}, {}},
415 }
416 tc.runTest(t)
417 }
418
419 func TestRESTClientContainerCPUEmptyMetricsForOnePod(t *testing.T) {
420 targetTimestamp := 1
421 window := 30 * time.Second
422 tc := restClientTestCase{
423 resourceName: v1.ResourceCPU,
424 desiredMetricValues: PodMetricsInfo{
425 "test-pod-0": {Value: 100, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
426 "test-pod-1": {Value: 300, Timestamp: offsetTimestampBy(targetTimestamp), Window: window},
427 },
428 container: "test-1",
429 targetTimestamp: targetTimestamp,
430 window: window,
431 desiredError: fmt.Errorf("failed to get container metrics"),
432 reportedPodMetrics: []map[string]int64{{"test-1": 100}, {"test-1": 300, "test-2": 400}, {}},
433 }
434 tc.runTest(t)
435 }
436
437 func offsetTimestampBy(t int) time.Time {
438 return fixedTimestamp.Add(time.Duration(t) * time.Minute)
439 }
440
View as plain text