1
16
17 package podautoscaler
18
19 import (
20 "context"
21 "fmt"
22 "math"
23 "strings"
24 "sync"
25 "testing"
26 "time"
27
28 autoscalingv1 "k8s.io/api/autoscaling/v1"
29 autoscalingv2 "k8s.io/api/autoscaling/v2"
30 v1 "k8s.io/api/core/v1"
31 "k8s.io/apimachinery/pkg/api/meta/testrestmapper"
32 "k8s.io/apimachinery/pkg/api/resource"
33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34 "k8s.io/apimachinery/pkg/labels"
35 "k8s.io/apimachinery/pkg/runtime"
36 "k8s.io/apimachinery/pkg/runtime/schema"
37 "k8s.io/apimachinery/pkg/util/wait"
38 "k8s.io/apimachinery/pkg/watch"
39 "k8s.io/client-go/informers"
40 "k8s.io/client-go/kubernetes/fake"
41 scalefake "k8s.io/client-go/scale/fake"
42 core "k8s.io/client-go/testing"
43 "k8s.io/kubernetes/pkg/api/legacyscheme"
44 autoscalingapiv2 "k8s.io/kubernetes/pkg/apis/autoscaling/v2"
45 "k8s.io/kubernetes/pkg/controller"
46 "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics"
47 "k8s.io/kubernetes/pkg/controller/podautoscaler/monitor"
48 "k8s.io/kubernetes/pkg/controller/util/selectors"
49 "k8s.io/kubernetes/test/utils/ktesting"
50 cmapi "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2"
51 emapi "k8s.io/metrics/pkg/apis/external_metrics/v1beta1"
52 metricsapi "k8s.io/metrics/pkg/apis/metrics/v1beta1"
53 metricsfake "k8s.io/metrics/pkg/client/clientset/versioned/fake"
54 cmfake "k8s.io/metrics/pkg/client/custom_metrics/fake"
55 emfake "k8s.io/metrics/pkg/client/external_metrics/fake"
56 "k8s.io/utils/pointer"
57
58 "github.com/stretchr/testify/assert"
59
60 _ "k8s.io/kubernetes/pkg/apis/apps/install"
61 _ "k8s.io/kubernetes/pkg/apis/autoscaling/install"
62 )
63
64
65
66
67
68
69
70
71
72
73 var statusOk = []autoscalingv2.HorizontalPodAutoscalerCondition{
74 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededRescale"},
75 {Type: autoscalingv2.ScalingActive, Status: v1.ConditionTrue, Reason: "ValidMetricFound"},
76 {Type: autoscalingv2.ScalingLimited, Status: v1.ConditionFalse, Reason: "DesiredWithinRange"},
77 }
78
79
80 func statusOkWithOverrides(overrides ...autoscalingv2.HorizontalPodAutoscalerCondition) []autoscalingv2.HorizontalPodAutoscalerCondition {
81 resv2 := make([]autoscalingv2.HorizontalPodAutoscalerCondition, len(statusOk))
82 copy(resv2, statusOk)
83 for _, override := range overrides {
84 resv2 = setConditionInList(resv2, override.Type, override.Status, override.Reason, override.Message)
85 }
86
87
88 resv1 := make([]autoscalingv2.HorizontalPodAutoscalerCondition, len(resv2))
89 for i, cond := range resv2 {
90 resv1[i] = autoscalingv2.HorizontalPodAutoscalerCondition{
91 Type: autoscalingv2.HorizontalPodAutoscalerConditionType(cond.Type),
92 Status: cond.Status,
93 Reason: cond.Reason,
94 }
95 }
96
97 return resv1
98 }
99
100 func alwaysReady() bool { return true }
101
102 type fakeResource struct {
103 name string
104 apiVersion string
105 kind string
106 }
107
108 type testCase struct {
109 sync.Mutex
110 minReplicas int32
111 maxReplicas int32
112 specReplicas int32
113 statusReplicas int32
114 initialReplicas int32
115 scaleUpRules *autoscalingv2.HPAScalingRules
116 scaleDownRules *autoscalingv2.HPAScalingRules
117
118
119 CPUTarget int32
120 CPUCurrent int32
121 verifyCPUCurrent bool
122 reportedLevels []uint64
123 reportedCPURequests []resource.Quantity
124 reportedPodReadiness []v1.ConditionStatus
125 reportedPodStartTime []metav1.Time
126 reportedPodPhase []v1.PodPhase
127 reportedPodDeletionTimestamp []bool
128 scaleUpdated bool
129 statusUpdated bool
130 eventCreated bool
131 verifyEvents bool
132 useMetricsAPI bool
133 metricsTarget []autoscalingv2.MetricSpec
134 expectedDesiredReplicas int32
135 expectedConditions []autoscalingv2.HorizontalPodAutoscalerCondition
136
137 processed chan string
138
139
140 expectedReportedReconciliationActionLabel monitor.ActionLabel
141 expectedReportedReconciliationErrorLabel monitor.ErrorLabel
142 expectedReportedMetricComputationActionLabels map[autoscalingv2.MetricSourceType]monitor.ActionLabel
143 expectedReportedMetricComputationErrorLabels map[autoscalingv2.MetricSourceType]monitor.ErrorLabel
144
145
146 resource *fakeResource
147
148
149 lastScaleTime *metav1.Time
150
151
152 testClient *fake.Clientset
153 testMetricsClient *metricsfake.Clientset
154 testCMClient *cmfake.FakeCustomMetricsClient
155 testEMClient *emfake.FakeExternalMetricsClient
156 testScaleClient *scalefake.FakeScaleClient
157
158 recommendations []timestampedRecommendation
159 hpaSelectors *selectors.BiMultimap
160 }
161
162
163 func (tc *testCase) computeCPUCurrent() {
164 if len(tc.reportedLevels) != len(tc.reportedCPURequests) || len(tc.reportedLevels) == 0 {
165 return
166 }
167 reported := 0
168 for _, r := range tc.reportedLevels {
169 reported += int(r)
170 }
171 requested := 0
172 for _, req := range tc.reportedCPURequests {
173 requested += int(req.MilliValue())
174 }
175 tc.CPUCurrent = int32(100 * reported / requested)
176 }
177
178 func init() {
179
180 scaleUpLimitFactor = 8
181 }
182
183 func (tc *testCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfake.Clientset, *cmfake.FakeCustomMetricsClient, *emfake.FakeExternalMetricsClient, *scalefake.FakeScaleClient) {
184 namespace := "test-namespace"
185 hpaName := "test-hpa"
186 podNamePrefix := "test-pod"
187 labelSet := map[string]string{"name": podNamePrefix}
188 selector := labels.SelectorFromSet(labelSet).String()
189
190 tc.Lock()
191
192 tc.scaleUpdated = false
193 tc.statusUpdated = false
194 tc.eventCreated = false
195 tc.processed = make(chan string, 100)
196 if tc.CPUCurrent == 0 {
197 tc.computeCPUCurrent()
198 }
199
200 if tc.resource == nil {
201 tc.resource = &fakeResource{
202 name: "test-rc",
203 apiVersion: "v1",
204 kind: "ReplicationController",
205 }
206 }
207 tc.Unlock()
208
209 fakeClient := &fake.Clientset{}
210 fakeClient.AddReactor("list", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
211 tc.Lock()
212 defer tc.Unlock()
213 var behavior *autoscalingv2.HorizontalPodAutoscalerBehavior
214 if tc.scaleUpRules != nil || tc.scaleDownRules != nil {
215 behavior = &autoscalingv2.HorizontalPodAutoscalerBehavior{
216 ScaleUp: tc.scaleUpRules,
217 ScaleDown: tc.scaleDownRules,
218 }
219 }
220 hpa := autoscalingv2.HorizontalPodAutoscaler{
221 ObjectMeta: metav1.ObjectMeta{
222 Name: hpaName,
223 Namespace: namespace,
224 },
225 Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
226 ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{
227 Kind: tc.resource.kind,
228 Name: tc.resource.name,
229 APIVersion: tc.resource.apiVersion,
230 },
231 MinReplicas: &tc.minReplicas,
232 MaxReplicas: tc.maxReplicas,
233 Behavior: behavior,
234 },
235 Status: autoscalingv2.HorizontalPodAutoscalerStatus{
236 CurrentReplicas: tc.specReplicas,
237 DesiredReplicas: tc.specReplicas,
238 LastScaleTime: tc.lastScaleTime,
239 },
240 }
241
242 autoscalingapiv2.SetDefaults_HorizontalPodAutoscalerBehavior(&hpa)
243
244 obj := &autoscalingv2.HorizontalPodAutoscalerList{
245 Items: []autoscalingv2.HorizontalPodAutoscaler{hpa},
246 }
247
248 if tc.CPUTarget > 0 {
249 obj.Items[0].Spec.Metrics = []autoscalingv2.MetricSpec{
250 {
251 Type: autoscalingv2.ResourceMetricSourceType,
252 Resource: &autoscalingv2.ResourceMetricSource{
253 Name: v1.ResourceCPU,
254 Target: autoscalingv2.MetricTarget{
255 Type: autoscalingv2.UtilizationMetricType,
256 AverageUtilization: &tc.CPUTarget,
257 },
258 },
259 },
260 }
261 }
262 if len(tc.metricsTarget) > 0 {
263 obj.Items[0].Spec.Metrics = append(obj.Items[0].Spec.Metrics, tc.metricsTarget...)
264 }
265
266 if len(obj.Items[0].Spec.Metrics) == 0 {
267
268 obj.Items[0].Spec.Metrics = []autoscalingv2.MetricSpec{
269 {
270 Type: autoscalingv2.ResourceMetricSourceType,
271 Resource: &autoscalingv2.ResourceMetricSource{
272 Name: v1.ResourceCPU,
273 },
274 },
275 }
276 }
277
278 return true, obj, nil
279 })
280
281 fakeClient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
282 tc.Lock()
283 defer tc.Unlock()
284
285 obj := &v1.PodList{}
286
287 specifiedCPURequests := tc.reportedCPURequests != nil
288
289 numPodsToCreate := int(tc.statusReplicas)
290 if specifiedCPURequests {
291 numPodsToCreate = len(tc.reportedCPURequests)
292 }
293
294 for i := 0; i < numPodsToCreate; i++ {
295 podReadiness := v1.ConditionTrue
296 if tc.reportedPodReadiness != nil {
297 podReadiness = tc.reportedPodReadiness[i]
298 }
299 var podStartTime metav1.Time
300 if tc.reportedPodStartTime != nil {
301 podStartTime = tc.reportedPodStartTime[i]
302 }
303
304 podPhase := v1.PodRunning
305 if tc.reportedPodPhase != nil {
306 podPhase = tc.reportedPodPhase[i]
307 }
308
309 podDeletionTimestamp := false
310 if tc.reportedPodDeletionTimestamp != nil {
311 podDeletionTimestamp = tc.reportedPodDeletionTimestamp[i]
312 }
313
314 podName := fmt.Sprintf("%s-%d", podNamePrefix, i)
315
316 reportedCPURequest := resource.MustParse("1.0")
317 if specifiedCPURequests {
318 reportedCPURequest = tc.reportedCPURequests[i]
319 }
320
321 pod := v1.Pod{
322 Status: v1.PodStatus{
323 Phase: podPhase,
324 Conditions: []v1.PodCondition{
325 {
326 Type: v1.PodReady,
327 Status: podReadiness,
328 LastTransitionTime: podStartTime,
329 },
330 },
331 StartTime: &podStartTime,
332 },
333 ObjectMeta: metav1.ObjectMeta{
334 Name: podName,
335 Namespace: namespace,
336 Labels: map[string]string{
337 "name": podNamePrefix,
338 },
339 },
340
341 Spec: v1.PodSpec{
342 Containers: []v1.Container{
343 {
344 Name: "container1",
345 Resources: v1.ResourceRequirements{
346 Requests: v1.ResourceList{
347 v1.ResourceCPU: *resource.NewMilliQuantity(reportedCPURequest.MilliValue()/2, resource.DecimalSI),
348 },
349 },
350 },
351 {
352 Name: "container2",
353 Resources: v1.ResourceRequirements{
354 Requests: v1.ResourceList{
355 v1.ResourceCPU: *resource.NewMilliQuantity(reportedCPURequest.MilliValue()/2, resource.DecimalSI),
356 },
357 },
358 },
359 },
360 },
361 }
362 if podDeletionTimestamp {
363 pod.DeletionTimestamp = &metav1.Time{Time: time.Now()}
364 }
365 obj.Items = append(obj.Items, pod)
366 }
367 return true, obj, nil
368 })
369
370 fakeClient.AddReactor("update", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
371 handled, obj, err := func() (handled bool, ret *autoscalingv2.HorizontalPodAutoscaler, err error) {
372 tc.Lock()
373 defer tc.Unlock()
374
375 obj := action.(core.UpdateAction).GetObject().(*autoscalingv2.HorizontalPodAutoscaler)
376 assert.Equal(t, namespace, obj.Namespace, "the HPA namespace should be as expected")
377 assert.Equal(t, hpaName, obj.Name, "the HPA name should be as expected")
378 assert.Equal(t, tc.expectedDesiredReplicas, obj.Status.DesiredReplicas, "the desired replica count reported in the object status should be as expected")
379 if tc.verifyCPUCurrent {
380 if utilization := findCpuUtilization(obj.Status.CurrentMetrics); assert.NotNil(t, utilization, "the reported CPU utilization percentage should be non-nil") {
381 assert.Equal(t, tc.CPUCurrent, *utilization, "the report CPU utilization percentage should be as expected")
382 }
383 }
384 actualConditions := obj.Status.Conditions
385
386
387
388 if tc.expectedConditions == nil {
389 tc.expectedConditions = statusOkWithOverrides()
390 }
391
392 for i := range actualConditions {
393 actualConditions[i].Message = ""
394 actualConditions[i].LastTransitionTime = metav1.Time{}
395 }
396 assert.Equal(t, tc.expectedConditions, actualConditions, "the status conditions should have been as expected")
397 tc.statusUpdated = true
398
399 return true, obj, nil
400 }()
401 if obj != nil {
402 tc.processed <- obj.Name
403 }
404 return handled, obj, err
405 })
406
407 fakeScaleClient := &scalefake.FakeScaleClient{}
408 fakeScaleClient.AddReactor("get", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
409 tc.Lock()
410 defer tc.Unlock()
411
412 obj := &autoscalingv1.Scale{
413 ObjectMeta: metav1.ObjectMeta{
414 Name: tc.resource.name,
415 Namespace: namespace,
416 },
417 Spec: autoscalingv1.ScaleSpec{
418 Replicas: tc.specReplicas,
419 },
420 Status: autoscalingv1.ScaleStatus{
421 Replicas: tc.statusReplicas,
422 Selector: selector,
423 },
424 }
425 return true, obj, nil
426 })
427
428 fakeScaleClient.AddReactor("get", "deployments", func(action core.Action) (handled bool, ret runtime.Object, err error) {
429 tc.Lock()
430 defer tc.Unlock()
431
432 obj := &autoscalingv1.Scale{
433 ObjectMeta: metav1.ObjectMeta{
434 Name: tc.resource.name,
435 Namespace: namespace,
436 },
437 Spec: autoscalingv1.ScaleSpec{
438 Replicas: tc.specReplicas,
439 },
440 Status: autoscalingv1.ScaleStatus{
441 Replicas: tc.statusReplicas,
442 Selector: selector,
443 },
444 }
445 return true, obj, nil
446 })
447
448 fakeScaleClient.AddReactor("get", "replicasets", func(action core.Action) (handled bool, ret runtime.Object, err error) {
449 tc.Lock()
450 defer tc.Unlock()
451
452 obj := &autoscalingv1.Scale{
453 ObjectMeta: metav1.ObjectMeta{
454 Name: tc.resource.name,
455 Namespace: namespace,
456 },
457 Spec: autoscalingv1.ScaleSpec{
458 Replicas: tc.specReplicas,
459 },
460 Status: autoscalingv1.ScaleStatus{
461 Replicas: tc.statusReplicas,
462 Selector: selector,
463 },
464 }
465 return true, obj, nil
466 })
467
468 fakeScaleClient.AddReactor("update", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
469 tc.Lock()
470 defer tc.Unlock()
471
472 obj := action.(core.UpdateAction).GetObject().(*autoscalingv1.Scale)
473 replicas := action.(core.UpdateAction).GetObject().(*autoscalingv1.Scale).Spec.Replicas
474 assert.Equal(t, tc.expectedDesiredReplicas, replicas, "the replica count of the RC should be as expected")
475 tc.scaleUpdated = true
476 return true, obj, nil
477 })
478
479 fakeScaleClient.AddReactor("update", "deployments", func(action core.Action) (handled bool, ret runtime.Object, err error) {
480 tc.Lock()
481 defer tc.Unlock()
482
483 obj := action.(core.UpdateAction).GetObject().(*autoscalingv1.Scale)
484 replicas := action.(core.UpdateAction).GetObject().(*autoscalingv1.Scale).Spec.Replicas
485 assert.Equal(t, tc.expectedDesiredReplicas, replicas, "the replica count of the deployment should be as expected")
486 tc.scaleUpdated = true
487 return true, obj, nil
488 })
489
490 fakeScaleClient.AddReactor("update", "replicasets", func(action core.Action) (handled bool, ret runtime.Object, err error) {
491 tc.Lock()
492 defer tc.Unlock()
493
494 obj := action.(core.UpdateAction).GetObject().(*autoscalingv1.Scale)
495 replicas := action.(core.UpdateAction).GetObject().(*autoscalingv1.Scale).Spec.Replicas
496 assert.Equal(t, tc.expectedDesiredReplicas, replicas, "the replica count of the replicaset should be as expected")
497 tc.scaleUpdated = true
498 return true, obj, nil
499 })
500
501 fakeWatch := watch.NewFake()
502 fakeClient.AddWatchReactor("*", core.DefaultWatchReactor(fakeWatch, nil))
503
504 fakeMetricsClient := &metricsfake.Clientset{}
505 fakeMetricsClient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
506 tc.Lock()
507 defer tc.Unlock()
508
509 metrics := &metricsapi.PodMetricsList{}
510 for i, cpu := range tc.reportedLevels {
511
512
513 podMetric := metricsapi.PodMetrics{
514 ObjectMeta: metav1.ObjectMeta{
515 Name: fmt.Sprintf("%s-%d", podNamePrefix, i),
516 Namespace: namespace,
517 Labels: labelSet,
518 },
519 Timestamp: metav1.Time{Time: time.Now()},
520 Window: metav1.Duration{Duration: time.Minute},
521 Containers: []metricsapi.ContainerMetrics{
522 {
523 Name: "container1",
524 Usage: v1.ResourceList{
525 v1.ResourceCPU: *resource.NewMilliQuantity(
526 int64(cpu/2),
527 resource.DecimalSI),
528 v1.ResourceMemory: *resource.NewQuantity(
529 int64(1024*1024/2),
530 resource.BinarySI),
531 },
532 },
533 {
534 Name: "container2",
535 Usage: v1.ResourceList{
536 v1.ResourceCPU: *resource.NewMilliQuantity(
537 int64(cpu/2),
538 resource.DecimalSI),
539 v1.ResourceMemory: *resource.NewQuantity(
540 int64(1024*1024/2),
541 resource.BinarySI),
542 },
543 },
544 },
545 }
546 metrics.Items = append(metrics.Items, podMetric)
547 }
548
549 return true, metrics, nil
550 })
551
552 fakeCMClient := &cmfake.FakeCustomMetricsClient{}
553 fakeCMClient.AddReactor("get", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
554 tc.Lock()
555 defer tc.Unlock()
556
557 getForAction, wasGetFor := action.(cmfake.GetForAction)
558 if !wasGetFor {
559 return true, nil, fmt.Errorf("expected a get-for action, got %v instead", action)
560 }
561
562 if getForAction.GetName() == "*" {
563 metrics := &cmapi.MetricValueList{}
564
565
566 assert.Equal(t, "pods", getForAction.GetResource().Resource, "the type of object that we requested multiple metrics for should have been pods")
567 assert.Equal(t, "qps", getForAction.GetMetricName(), "the metric name requested should have been qps, as specified in the metric spec")
568
569 for i, level := range tc.reportedLevels {
570 podMetric := cmapi.MetricValue{
571 DescribedObject: v1.ObjectReference{
572 Kind: "Pod",
573 Name: fmt.Sprintf("%s-%d", podNamePrefix, i),
574 Namespace: namespace,
575 },
576 Timestamp: metav1.Time{Time: time.Now()},
577 Metric: cmapi.MetricIdentifier{
578 Name: "qps",
579 },
580 Value: *resource.NewMilliQuantity(int64(level), resource.DecimalSI),
581 }
582 metrics.Items = append(metrics.Items, podMetric)
583 }
584
585 return true, metrics, nil
586 }
587
588 name := getForAction.GetName()
589 mapper := testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme)
590 metrics := &cmapi.MetricValueList{}
591 var matchedTarget *autoscalingv2.MetricSpec
592 for i, target := range tc.metricsTarget {
593 if target.Type == autoscalingv2.ObjectMetricSourceType && name == target.Object.DescribedObject.Name {
594 gk := schema.FromAPIVersionAndKind(target.Object.DescribedObject.APIVersion, target.Object.DescribedObject.Kind).GroupKind()
595 mapping, err := mapper.RESTMapping(gk)
596 if err != nil {
597 t.Logf("unable to get mapping for %s: %v", gk.String(), err)
598 continue
599 }
600 groupResource := mapping.Resource.GroupResource()
601
602 if getForAction.GetResource().Resource == groupResource.String() {
603 matchedTarget = &tc.metricsTarget[i]
604 }
605 }
606 }
607 assert.NotNil(t, matchedTarget, "this request should have matched one of the metric specs")
608 assert.Equal(t, "qps", getForAction.GetMetricName(), "the metric name requested should have been qps, as specified in the metric spec")
609
610 metrics.Items = []cmapi.MetricValue{
611 {
612 DescribedObject: v1.ObjectReference{
613 Kind: matchedTarget.Object.DescribedObject.Kind,
614 APIVersion: matchedTarget.Object.DescribedObject.APIVersion,
615 Name: name,
616 },
617 Timestamp: metav1.Time{Time: time.Now()},
618 Metric: cmapi.MetricIdentifier{
619 Name: "qps",
620 },
621 Value: *resource.NewMilliQuantity(int64(tc.reportedLevels[0]), resource.DecimalSI),
622 },
623 }
624
625 return true, metrics, nil
626 })
627
628 fakeEMClient := &emfake.FakeExternalMetricsClient{}
629
630 fakeEMClient.AddReactor("list", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
631 tc.Lock()
632 defer tc.Unlock()
633
634 listAction, wasList := action.(core.ListAction)
635 if !wasList {
636 return true, nil, fmt.Errorf("expected a list action, got %v instead", action)
637 }
638
639 metrics := &emapi.ExternalMetricValueList{}
640
641 assert.Equal(t, "qps", listAction.GetResource().Resource, "the metric name requested should have been qps, as specified in the metric spec")
642
643 for _, level := range tc.reportedLevels {
644 metric := emapi.ExternalMetricValue{
645 Timestamp: metav1.Time{Time: time.Now()},
646 MetricName: "qps",
647 Value: *resource.NewMilliQuantity(int64(level), resource.DecimalSI),
648 }
649 metrics.Items = append(metrics.Items, metric)
650 }
651
652 return true, metrics, nil
653 })
654
655 return fakeClient, fakeMetricsClient, fakeCMClient, fakeEMClient, fakeScaleClient
656 }
657
658 func findCpuUtilization(metricStatus []autoscalingv2.MetricStatus) (utilization *int32) {
659 for _, s := range metricStatus {
660 if s.Type != autoscalingv2.ResourceMetricSourceType {
661 continue
662 }
663 if s.Resource == nil {
664 continue
665 }
666 if s.Resource.Name != v1.ResourceCPU {
667 continue
668 }
669 if s.Resource.Current.AverageUtilization == nil {
670 continue
671 }
672 return s.Resource.Current.AverageUtilization
673 }
674 return nil
675 }
676
677 func (tc *testCase) verifyResults(t *testing.T, m *mockMonitor) {
678 tc.Lock()
679 defer tc.Unlock()
680
681 assert.Equal(t, tc.specReplicas != tc.expectedDesiredReplicas, tc.scaleUpdated, "the scale should only be updated if we expected a change in replicas")
682 assert.True(t, tc.statusUpdated, "the status should have been updated")
683 if tc.verifyEvents {
684 assert.Equal(t, tc.specReplicas != tc.expectedDesiredReplicas, tc.eventCreated, "an event should have been created only if we expected a change in replicas")
685 }
686
687 tc.verifyRecordedMetric(t, m)
688 }
689
690 func (tc *testCase) verifyRecordedMetric(t *testing.T, m *mockMonitor) {
691
692 m.waitUntilRecorded(t)
693
694 assert.Equal(t, tc.expectedReportedReconciliationActionLabel, m.reconciliationActionLabels[0], "the reconciliation action should be recorded in monitor expectedly")
695 assert.Equal(t, tc.expectedReportedReconciliationErrorLabel, m.reconciliationErrorLabels[0], "the reconciliation error should be recorded in monitor expectedly")
696
697 if len(tc.expectedReportedMetricComputationActionLabels) != len(m.metricComputationActionLabels) {
698 t.Fatalf("the metric computation actions for %d types should be recorded, but actually only %d was recorded", len(tc.expectedReportedMetricComputationActionLabels), len(m.metricComputationActionLabels))
699 }
700 if len(tc.expectedReportedMetricComputationErrorLabels) != len(m.metricComputationErrorLabels) {
701 t.Fatalf("the metric computation errors for %d types should be recorded, but actually only %d was recorded", len(tc.expectedReportedMetricComputationErrorLabels), len(m.metricComputationErrorLabels))
702 }
703
704 for metricType, l := range tc.expectedReportedMetricComputationActionLabels {
705 _, ok := m.metricComputationActionLabels[metricType]
706 if !ok {
707 t.Fatalf("the metric computation action should be recorded with metricType %s, but actually nothing was recorded", metricType)
708 }
709 assert.Equal(t, l, m.metricComputationActionLabels[metricType][0], "the metric computation action should be recorded in monitor expectedly")
710 }
711 for metricType, l := range tc.expectedReportedMetricComputationErrorLabels {
712 _, ok := m.metricComputationErrorLabels[metricType]
713 if !ok {
714 t.Fatalf("the metric computation error should be recorded with metricType %s, but actually nothing was recorded", metricType)
715 }
716 assert.Equal(t, l, m.metricComputationErrorLabels[metricType][0], "the metric computation error should be recorded in monitor expectedly")
717 }
718 }
719
720 func (tc *testCase) setupController(t *testing.T) (*HorizontalController, informers.SharedInformerFactory) {
721 testClient, testMetricsClient, testCMClient, testEMClient, testScaleClient := tc.prepareTestClient(t)
722 if tc.testClient != nil {
723 testClient = tc.testClient
724 }
725 if tc.testMetricsClient != nil {
726 testMetricsClient = tc.testMetricsClient
727 }
728 if tc.testCMClient != nil {
729 testCMClient = tc.testCMClient
730 }
731 if tc.testEMClient != nil {
732 testEMClient = tc.testEMClient
733 }
734 if tc.testScaleClient != nil {
735 testScaleClient = tc.testScaleClient
736 }
737 metricsClient := metrics.NewRESTMetricsClient(
738 testMetricsClient.MetricsV1beta1(),
739 testCMClient,
740 testEMClient,
741 )
742
743 eventClient := &fake.Clientset{}
744 eventClient.AddReactor("create", "events", func(action core.Action) (handled bool, ret runtime.Object, err error) {
745 tc.Lock()
746 defer tc.Unlock()
747
748 obj := action.(core.CreateAction).GetObject().(*v1.Event)
749 if tc.verifyEvents {
750 switch obj.Reason {
751 case "SuccessfulRescale":
752 assert.Equal(t, fmt.Sprintf("New size: %d; reason: cpu resource utilization (percentage of request) above target", tc.expectedDesiredReplicas), obj.Message)
753 case "DesiredReplicasComputed":
754 assert.Equal(t, fmt.Sprintf(
755 "Computed the desired num of replicas: %d (avgCPUutil: %d, current replicas: %d)",
756 tc.expectedDesiredReplicas,
757 (int64(tc.reportedLevels[0])*100)/tc.reportedCPURequests[0].MilliValue(), tc.specReplicas), obj.Message)
758 default:
759 assert.False(t, true, fmt.Sprintf("Unexpected event: %s / %s", obj.Reason, obj.Message))
760 }
761 }
762 tc.eventCreated = true
763 return true, obj, nil
764 })
765
766 informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
767 defaultDownscalestabilizationWindow := 5 * time.Minute
768
769 tCtx := ktesting.Init(t)
770 hpaController := NewHorizontalController(
771 tCtx,
772 eventClient.CoreV1(),
773 testScaleClient,
774 testClient.AutoscalingV2(),
775 testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme),
776 metricsClient,
777 informerFactory.Autoscaling().V2().HorizontalPodAutoscalers(),
778 informerFactory.Core().V1().Pods(),
779 100*time.Millisecond,
780 defaultDownscalestabilizationWindow,
781 defaultTestingTolerance,
782 defaultTestingCPUInitializationPeriod,
783 defaultTestingDelayOfInitialReadinessStatus,
784 )
785 hpaController.hpaListerSynced = alwaysReady
786 if tc.recommendations != nil {
787 hpaController.recommendations["test-namespace/test-hpa"] = tc.recommendations
788 }
789 if tc.hpaSelectors != nil {
790 hpaController.hpaSelectors = tc.hpaSelectors
791 }
792
793 hpaController.monitor = newMockMonitor()
794 return hpaController, informerFactory
795 }
796
797 func hotCPUCreationTime() metav1.Time {
798 return metav1.Time{Time: time.Now()}
799 }
800
801 func coolCPUCreationTime() metav1.Time {
802 return metav1.Time{Time: time.Now().Add(-3 * time.Minute)}
803 }
804
805 func (tc *testCase) runTestWithController(t *testing.T, hpaController *HorizontalController, informerFactory informers.SharedInformerFactory) {
806 ctx, cancel := context.WithCancel(context.Background())
807 defer cancel()
808 informerFactory.Start(ctx.Done())
809 go hpaController.Run(ctx, 5)
810
811 tc.Lock()
812 shouldWait := tc.verifyEvents
813 tc.Unlock()
814
815 if shouldWait {
816
817 timeoutTime := time.Now().Add(2 * time.Second)
818 for now := time.Now(); timeoutTime.After(now); now = time.Now() {
819 sleepUntil := timeoutTime.Sub(now)
820 select {
821 case <-tc.processed:
822
823 case <-time.After(sleepUntil):
824
825 }
826 }
827 } else {
828
829 <-tc.processed
830 }
831 m, ok := hpaController.monitor.(*mockMonitor)
832 if !ok {
833 t.Fatalf("test HPA controller should have mockMonitor, but actually not")
834 }
835 tc.verifyResults(t, m)
836 }
837
838 func (tc *testCase) runTest(t *testing.T) {
839 hpaController, informerFactory := tc.setupController(t)
840 tc.runTestWithController(t, hpaController, informerFactory)
841 }
842
843
844
845 type mockMonitor struct {
846 sync.RWMutex
847 reconciliationActionLabels []monitor.ActionLabel
848 reconciliationErrorLabels []monitor.ErrorLabel
849
850 metricComputationActionLabels map[autoscalingv2.MetricSourceType][]monitor.ActionLabel
851 metricComputationErrorLabels map[autoscalingv2.MetricSourceType][]monitor.ErrorLabel
852 }
853
854 func newMockMonitor() *mockMonitor {
855 return &mockMonitor{
856 metricComputationActionLabels: make(map[autoscalingv2.MetricSourceType][]monitor.ActionLabel),
857 metricComputationErrorLabels: make(map[autoscalingv2.MetricSourceType][]monitor.ErrorLabel),
858 }
859 }
860
861 func (m *mockMonitor) ObserveReconciliationResult(action monitor.ActionLabel, err monitor.ErrorLabel, _ time.Duration) {
862 m.Lock()
863 defer m.Unlock()
864 m.reconciliationActionLabels = append(m.reconciliationActionLabels, action)
865 m.reconciliationErrorLabels = append(m.reconciliationErrorLabels, err)
866 }
867
868 func (m *mockMonitor) ObserveMetricComputationResult(action monitor.ActionLabel, err monitor.ErrorLabel, duration time.Duration, metricType autoscalingv2.MetricSourceType) {
869 m.Lock()
870 defer m.Unlock()
871
872 m.metricComputationActionLabels[metricType] = append(m.metricComputationActionLabels[metricType], action)
873 m.metricComputationErrorLabels[metricType] = append(m.metricComputationErrorLabels[metricType], err)
874 }
875
876
877 func (m *mockMonitor) waitUntilRecorded(t *testing.T) {
878 if err := wait.Poll(20*time.Millisecond, 100*time.Millisecond, func() (done bool, err error) {
879 m.RWMutex.RLock()
880 defer m.RWMutex.RUnlock()
881 if len(m.reconciliationActionLabels) == 0 || len(m.reconciliationErrorLabels) == 0 {
882 return false, nil
883 }
884 return true, nil
885 }); err != nil {
886 t.Fatalf("no reconciliation is recorded in the monitor, len(monitor.reconciliationActionLabels)=%v len(monitor.reconciliationErrorLabels)=%v ", len(m.reconciliationActionLabels), len(m.reconciliationErrorLabels))
887 }
888 }
889
890 func TestScaleUp(t *testing.T) {
891 tc := testCase{
892 minReplicas: 2,
893 maxReplicas: 6,
894 specReplicas: 3,
895 statusReplicas: 3,
896 expectedDesiredReplicas: 5,
897 CPUTarget: 30,
898 verifyCPUCurrent: true,
899 reportedLevels: []uint64{300, 500, 700},
900 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
901 useMetricsAPI: true,
902 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
903 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
904 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
905 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
906 },
907 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
908 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
909 },
910 }
911 tc.runTest(t)
912 }
913
914 func TestScaleUpContainer(t *testing.T) {
915 tc := testCase{
916 minReplicas: 2,
917 maxReplicas: 6,
918 specReplicas: 3,
919 statusReplicas: 3,
920 expectedDesiredReplicas: 5,
921 metricsTarget: []autoscalingv2.MetricSpec{{
922 Type: autoscalingv2.ContainerResourceMetricSourceType,
923 ContainerResource: &autoscalingv2.ContainerResourceMetricSource{
924 Name: v1.ResourceCPU,
925 Target: autoscalingv2.MetricTarget{
926 Type: autoscalingv2.UtilizationMetricType,
927 AverageUtilization: pointer.Int32(30),
928 },
929 Container: "container1",
930 },
931 }},
932 reportedLevels: []uint64{300, 500, 700},
933 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
934 useMetricsAPI: true,
935 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
936 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
937 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
938 autoscalingv2.ContainerResourceMetricSourceType: monitor.ActionLabelScaleUp,
939 },
940 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
941 autoscalingv2.ContainerResourceMetricSourceType: monitor.ErrorLabelNone,
942 },
943 }
944 tc.runTest(t)
945 }
946
947 func TestScaleUpUnreadyLessScale(t *testing.T) {
948 tc := testCase{
949 minReplicas: 2,
950 maxReplicas: 6,
951 specReplicas: 3,
952 statusReplicas: 3,
953 expectedDesiredReplicas: 4,
954 CPUTarget: 30,
955 CPUCurrent: 60,
956 verifyCPUCurrent: true,
957 reportedLevels: []uint64{300, 500, 700},
958 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
959 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionFalse, v1.ConditionTrue, v1.ConditionTrue},
960 useMetricsAPI: true,
961 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
962 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
963 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
964 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
965 },
966 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
967 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
968 },
969 }
970 tc.runTest(t)
971 }
972
973 func TestScaleUpHotCpuLessScale(t *testing.T) {
974 tc := testCase{
975 minReplicas: 2,
976 maxReplicas: 6,
977 specReplicas: 3,
978 statusReplicas: 3,
979 expectedDesiredReplicas: 4,
980 CPUTarget: 30,
981 CPUCurrent: 60,
982 verifyCPUCurrent: true,
983 reportedLevels: []uint64{300, 500, 700},
984 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
985 reportedPodStartTime: []metav1.Time{hotCPUCreationTime(), coolCPUCreationTime(), coolCPUCreationTime()},
986 useMetricsAPI: true,
987 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
988 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
989 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
990 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
991 },
992 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
993 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
994 },
995 }
996 tc.runTest(t)
997 }
998
999 func TestScaleUpUnreadyNoScale(t *testing.T) {
1000 tc := testCase{
1001 minReplicas: 2,
1002 maxReplicas: 6,
1003 specReplicas: 3,
1004 statusReplicas: 3,
1005 expectedDesiredReplicas: 3,
1006 CPUTarget: 30,
1007 CPUCurrent: 40,
1008 verifyCPUCurrent: true,
1009 reportedLevels: []uint64{400, 500, 700},
1010 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1011 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse},
1012 useMetricsAPI: true,
1013 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
1014 Type: autoscalingv2.AbleToScale,
1015 Status: v1.ConditionTrue,
1016 Reason: "ReadyForNewScale",
1017 }),
1018 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
1019 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1020 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1021 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelNone,
1022 },
1023 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1024 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1025 },
1026 }
1027 tc.runTest(t)
1028 }
1029
1030 func TestScaleUpHotCpuNoScale(t *testing.T) {
1031 tc := testCase{
1032 minReplicas: 2,
1033 maxReplicas: 6,
1034 specReplicas: 3,
1035 statusReplicas: 3,
1036 expectedDesiredReplicas: 3,
1037 CPUTarget: 30,
1038 CPUCurrent: 40,
1039 verifyCPUCurrent: true,
1040 reportedLevels: []uint64{400, 500, 700},
1041 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1042 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse},
1043 reportedPodStartTime: []metav1.Time{coolCPUCreationTime(), hotCPUCreationTime(), hotCPUCreationTime()},
1044 useMetricsAPI: true,
1045 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
1046 Type: autoscalingv2.AbleToScale,
1047 Status: v1.ConditionTrue,
1048 Reason: "ReadyForNewScale",
1049 }),
1050 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
1051 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1052 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1053 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelNone,
1054 },
1055 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1056 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1057 },
1058 }
1059 tc.runTest(t)
1060 }
1061
1062 func TestScaleUpIgnoresFailedPods(t *testing.T) {
1063 tc := testCase{
1064 minReplicas: 2,
1065 maxReplicas: 6,
1066 specReplicas: 2,
1067 statusReplicas: 2,
1068 expectedDesiredReplicas: 4,
1069 CPUTarget: 30,
1070 CPUCurrent: 60,
1071 verifyCPUCurrent: true,
1072 reportedLevels: []uint64{500, 700},
1073 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1074 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse},
1075 reportedPodPhase: []v1.PodPhase{v1.PodRunning, v1.PodRunning, v1.PodFailed, v1.PodFailed},
1076 useMetricsAPI: true,
1077 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1078 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1079 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1080 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
1081 },
1082 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1083 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1084 },
1085 }
1086 tc.runTest(t)
1087 }
1088
1089 func TestScaleUpIgnoresDeletionPods(t *testing.T) {
1090 tc := testCase{
1091 minReplicas: 2,
1092 maxReplicas: 6,
1093 specReplicas: 2,
1094 statusReplicas: 2,
1095 expectedDesiredReplicas: 4,
1096 CPUTarget: 30,
1097 CPUCurrent: 60,
1098 verifyCPUCurrent: true,
1099 reportedLevels: []uint64{500, 700},
1100 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1101 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse},
1102 reportedPodPhase: []v1.PodPhase{v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning},
1103 reportedPodDeletionTimestamp: []bool{false, false, true, true},
1104 useMetricsAPI: true,
1105 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1106 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1107 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1108 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
1109 },
1110 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1111 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1112 },
1113 }
1114 tc.runTest(t)
1115 }
1116
1117 func TestScaleUpDeployment(t *testing.T) {
1118 tc := testCase{
1119 minReplicas: 2,
1120 maxReplicas: 6,
1121 specReplicas: 3,
1122 statusReplicas: 3,
1123 expectedDesiredReplicas: 5,
1124 CPUTarget: 30,
1125 verifyCPUCurrent: true,
1126 reportedLevels: []uint64{300, 500, 700},
1127 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1128 useMetricsAPI: true,
1129 resource: &fakeResource{
1130 name: "test-dep",
1131 apiVersion: "apps/v1",
1132 kind: "Deployment",
1133 },
1134 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1135 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1136 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1137 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
1138 },
1139 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1140 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1141 },
1142 }
1143 tc.runTest(t)
1144 }
1145
1146 func TestScaleUpReplicaSet(t *testing.T) {
1147 tc := testCase{
1148 minReplicas: 2,
1149 maxReplicas: 6,
1150 specReplicas: 3,
1151 statusReplicas: 3,
1152 expectedDesiredReplicas: 5,
1153 CPUTarget: 30,
1154 verifyCPUCurrent: true,
1155 reportedLevels: []uint64{300, 500, 700},
1156 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1157 useMetricsAPI: true,
1158 resource: &fakeResource{
1159 name: "test-replicaset",
1160 apiVersion: "apps/v1",
1161 kind: "ReplicaSet",
1162 },
1163 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1164 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1165 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1166 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
1167 },
1168 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1169 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1170 },
1171 }
1172 tc.runTest(t)
1173 }
1174
1175 func TestScaleUpCM(t *testing.T) {
1176 averageValue := resource.MustParse("15.0")
1177 tc := testCase{
1178 minReplicas: 2,
1179 maxReplicas: 6,
1180 specReplicas: 3,
1181 statusReplicas: 3,
1182 expectedDesiredReplicas: 4,
1183 CPUTarget: 0,
1184 metricsTarget: []autoscalingv2.MetricSpec{
1185 {
1186 Type: autoscalingv2.PodsMetricSourceType,
1187 Pods: &autoscalingv2.PodsMetricSource{
1188 Metric: autoscalingv2.MetricIdentifier{
1189 Name: "qps",
1190 },
1191 Target: autoscalingv2.MetricTarget{
1192 Type: autoscalingv2.AverageValueMetricType,
1193 AverageValue: &averageValue,
1194 },
1195 },
1196 },
1197 },
1198 reportedLevels: []uint64{20000, 10000, 30000},
1199 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1200 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1201 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1202 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1203 autoscalingv2.PodsMetricSourceType: monitor.ActionLabelScaleUp,
1204 },
1205 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1206 autoscalingv2.PodsMetricSourceType: monitor.ErrorLabelNone,
1207 },
1208 }
1209 tc.runTest(t)
1210 }
1211
1212 func TestScaleUpCMUnreadyAndHotCpuNoLessScale(t *testing.T) {
1213 averageValue := resource.MustParse("15.0")
1214 tc := testCase{
1215 minReplicas: 2,
1216 maxReplicas: 6,
1217 specReplicas: 3,
1218 statusReplicas: 3,
1219 expectedDesiredReplicas: 6,
1220 CPUTarget: 0,
1221 metricsTarget: []autoscalingv2.MetricSpec{
1222 {
1223 Type: autoscalingv2.PodsMetricSourceType,
1224 Pods: &autoscalingv2.PodsMetricSource{
1225 Metric: autoscalingv2.MetricIdentifier{
1226 Name: "qps",
1227 },
1228 Target: autoscalingv2.MetricTarget{
1229 Type: autoscalingv2.AverageValueMetricType,
1230 AverageValue: &averageValue,
1231 },
1232 },
1233 },
1234 },
1235 reportedLevels: []uint64{50000, 10000, 30000},
1236 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse},
1237 reportedPodStartTime: []metav1.Time{coolCPUCreationTime(), coolCPUCreationTime(), hotCPUCreationTime()},
1238 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1239 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1240 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1241 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1242 autoscalingv2.PodsMetricSourceType: monitor.ActionLabelScaleUp,
1243 },
1244 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1245 autoscalingv2.PodsMetricSourceType: monitor.ErrorLabelNone,
1246 },
1247 }
1248 tc.runTest(t)
1249 }
1250
1251 func TestScaleUpCMUnreadyandCpuHot(t *testing.T) {
1252 averageValue := resource.MustParse("15.0")
1253 tc := testCase{
1254 minReplicas: 2,
1255 maxReplicas: 6,
1256 specReplicas: 3,
1257 statusReplicas: 3,
1258 expectedDesiredReplicas: 6,
1259 CPUTarget: 0,
1260 metricsTarget: []autoscalingv2.MetricSpec{
1261 {
1262 Type: autoscalingv2.PodsMetricSourceType,
1263 Pods: &autoscalingv2.PodsMetricSource{
1264 Metric: autoscalingv2.MetricIdentifier{
1265 Name: "qps",
1266 },
1267 Target: autoscalingv2.MetricTarget{
1268 Type: autoscalingv2.AverageValueMetricType,
1269 AverageValue: &averageValue,
1270 },
1271 },
1272 },
1273 },
1274 reportedLevels: []uint64{50000, 15000, 30000},
1275 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionFalse, v1.ConditionTrue, v1.ConditionFalse},
1276 reportedPodStartTime: []metav1.Time{hotCPUCreationTime(), coolCPUCreationTime(), hotCPUCreationTime()},
1277 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1278 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
1279 Type: autoscalingv2.AbleToScale,
1280 Status: v1.ConditionTrue,
1281 Reason: "SucceededRescale",
1282 }, autoscalingv2.HorizontalPodAutoscalerCondition{
1283 Type: autoscalingv2.ScalingLimited,
1284 Status: v1.ConditionTrue,
1285 Reason: "TooManyReplicas",
1286 }),
1287 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1288 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1289 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1290 autoscalingv2.PodsMetricSourceType: monitor.ActionLabelScaleUp,
1291 },
1292 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1293 autoscalingv2.PodsMetricSourceType: monitor.ErrorLabelNone,
1294 },
1295 }
1296 tc.runTest(t)
1297 }
1298
1299 func TestScaleUpHotCpuNoScaleWouldScaleDown(t *testing.T) {
1300 averageValue := resource.MustParse("15.0")
1301 tc := testCase{
1302 minReplicas: 2,
1303 maxReplicas: 6,
1304 specReplicas: 3,
1305 statusReplicas: 3,
1306 expectedDesiredReplicas: 6,
1307 CPUTarget: 0,
1308 metricsTarget: []autoscalingv2.MetricSpec{
1309 {
1310 Type: autoscalingv2.PodsMetricSourceType,
1311 Pods: &autoscalingv2.PodsMetricSource{
1312 Metric: autoscalingv2.MetricIdentifier{
1313 Name: "qps",
1314 },
1315 Target: autoscalingv2.MetricTarget{
1316 Type: autoscalingv2.AverageValueMetricType,
1317 AverageValue: &averageValue,
1318 },
1319 },
1320 },
1321 },
1322 reportedLevels: []uint64{50000, 15000, 30000},
1323 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1324 reportedPodStartTime: []metav1.Time{hotCPUCreationTime(), coolCPUCreationTime(), hotCPUCreationTime()},
1325 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
1326 Type: autoscalingv2.AbleToScale,
1327 Status: v1.ConditionTrue,
1328 Reason: "SucceededRescale",
1329 }, autoscalingv2.HorizontalPodAutoscalerCondition{
1330 Type: autoscalingv2.ScalingLimited,
1331 Status: v1.ConditionTrue,
1332 Reason: "TooManyReplicas",
1333 }),
1334 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1335 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1336 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1337 autoscalingv2.PodsMetricSourceType: monitor.ActionLabelScaleUp,
1338 },
1339 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1340 autoscalingv2.PodsMetricSourceType: monitor.ErrorLabelNone,
1341 },
1342 }
1343 tc.runTest(t)
1344 }
1345
1346 func TestScaleUpCMObject(t *testing.T) {
1347 targetValue := resource.MustParse("15.0")
1348 tc := testCase{
1349 minReplicas: 2,
1350 maxReplicas: 6,
1351 specReplicas: 3,
1352 statusReplicas: 3,
1353 expectedDesiredReplicas: 4,
1354 CPUTarget: 0,
1355 metricsTarget: []autoscalingv2.MetricSpec{
1356 {
1357 Type: autoscalingv2.ObjectMetricSourceType,
1358 Object: &autoscalingv2.ObjectMetricSource{
1359 DescribedObject: autoscalingv2.CrossVersionObjectReference{
1360 APIVersion: "apps/v1",
1361 Kind: "Deployment",
1362 Name: "some-deployment",
1363 },
1364 Metric: autoscalingv2.MetricIdentifier{
1365 Name: "qps",
1366 },
1367 Target: autoscalingv2.MetricTarget{
1368 Type: autoscalingv2.ValueMetricType,
1369 Value: &targetValue,
1370 },
1371 },
1372 },
1373 },
1374 reportedLevels: []uint64{20000},
1375 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1376 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1377 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1378 autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleUp,
1379 },
1380 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1381 autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone,
1382 },
1383 }
1384 tc.runTest(t)
1385 }
1386
1387 func TestScaleUpFromZeroCMObject(t *testing.T) {
1388 targetValue := resource.MustParse("15.0")
1389 tc := testCase{
1390 minReplicas: 0,
1391 maxReplicas: 6,
1392 specReplicas: 0,
1393 statusReplicas: 0,
1394 expectedDesiredReplicas: 2,
1395 CPUTarget: 0,
1396 metricsTarget: []autoscalingv2.MetricSpec{
1397 {
1398 Type: autoscalingv2.ObjectMetricSourceType,
1399 Object: &autoscalingv2.ObjectMetricSource{
1400 DescribedObject: autoscalingv2.CrossVersionObjectReference{
1401 APIVersion: "apps/v1",
1402 Kind: "Deployment",
1403 Name: "some-deployment",
1404 },
1405 Metric: autoscalingv2.MetricIdentifier{
1406 Name: "qps",
1407 },
1408 Target: autoscalingv2.MetricTarget{
1409 Type: autoscalingv2.ValueMetricType,
1410 Value: &targetValue,
1411 },
1412 },
1413 },
1414 },
1415 reportedLevels: []uint64{20000},
1416 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1417 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1418 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1419 autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleUp,
1420 },
1421 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1422 autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone,
1423 },
1424 }
1425 tc.runTest(t)
1426 }
1427
1428 func TestScaleUpFromZeroIgnoresToleranceCMObject(t *testing.T) {
1429 targetValue := resource.MustParse("1.0")
1430 tc := testCase{
1431 minReplicas: 0,
1432 maxReplicas: 6,
1433 specReplicas: 0,
1434 statusReplicas: 0,
1435 expectedDesiredReplicas: 1,
1436 CPUTarget: 0,
1437 metricsTarget: []autoscalingv2.MetricSpec{
1438 {
1439 Type: autoscalingv2.ObjectMetricSourceType,
1440 Object: &autoscalingv2.ObjectMetricSource{
1441 DescribedObject: autoscalingv2.CrossVersionObjectReference{
1442 APIVersion: "apps/v1",
1443 Kind: "Deployment",
1444 Name: "some-deployment",
1445 },
1446 Metric: autoscalingv2.MetricIdentifier{
1447 Name: "qps",
1448 },
1449 Target: autoscalingv2.MetricTarget{
1450 Type: autoscalingv2.ValueMetricType,
1451 Value: &targetValue,
1452 },
1453 },
1454 },
1455 },
1456 reportedLevels: []uint64{1000},
1457 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1458 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1459 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1460 autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleUp,
1461 },
1462 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1463 autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone,
1464 },
1465 }
1466 tc.runTest(t)
1467 }
1468
1469 func TestScaleUpPerPodCMObject(t *testing.T) {
1470 targetAverageValue := resource.MustParse("10.0")
1471 tc := testCase{
1472 minReplicas: 2,
1473 maxReplicas: 6,
1474 specReplicas: 3,
1475 statusReplicas: 3,
1476 expectedDesiredReplicas: 4,
1477 CPUTarget: 0,
1478 metricsTarget: []autoscalingv2.MetricSpec{
1479 {
1480 Type: autoscalingv2.ObjectMetricSourceType,
1481 Object: &autoscalingv2.ObjectMetricSource{
1482 DescribedObject: autoscalingv2.CrossVersionObjectReference{
1483 APIVersion: "apps/v1",
1484 Kind: "Deployment",
1485 Name: "some-deployment",
1486 },
1487 Metric: autoscalingv2.MetricIdentifier{
1488 Name: "qps",
1489 },
1490 Target: autoscalingv2.MetricTarget{
1491 Type: autoscalingv2.AverageValueMetricType,
1492 AverageValue: &targetAverageValue,
1493 },
1494 },
1495 },
1496 },
1497 reportedLevels: []uint64{40000},
1498 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1499 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1500 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1501 autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleUp,
1502 },
1503 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1504 autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone,
1505 },
1506 }
1507 tc.runTest(t)
1508 }
1509
1510 func TestScaleUpCMExternal(t *testing.T) {
1511 tc := testCase{
1512 minReplicas: 2,
1513 maxReplicas: 6,
1514 specReplicas: 3,
1515 statusReplicas: 3,
1516 expectedDesiredReplicas: 4,
1517 metricsTarget: []autoscalingv2.MetricSpec{
1518 {
1519 Type: autoscalingv2.ExternalMetricSourceType,
1520 External: &autoscalingv2.ExternalMetricSource{
1521 Metric: autoscalingv2.MetricIdentifier{
1522 Name: "qps",
1523 Selector: &metav1.LabelSelector{},
1524 },
1525 Target: autoscalingv2.MetricTarget{
1526 Type: autoscalingv2.ValueMetricType,
1527 Value: resource.NewMilliQuantity(6666, resource.DecimalSI),
1528 },
1529 },
1530 },
1531 },
1532 reportedLevels: []uint64{8600},
1533 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1534 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1535 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1536 autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelScaleUp,
1537 },
1538 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1539 autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelNone,
1540 },
1541 }
1542 tc.runTest(t)
1543 }
1544
1545 func TestScaleUpPerPodCMExternal(t *testing.T) {
1546 tc := testCase{
1547 minReplicas: 2,
1548 maxReplicas: 6,
1549 specReplicas: 3,
1550 statusReplicas: 3,
1551 expectedDesiredReplicas: 4,
1552 metricsTarget: []autoscalingv2.MetricSpec{
1553 {
1554 Type: autoscalingv2.ExternalMetricSourceType,
1555 External: &autoscalingv2.ExternalMetricSource{
1556 Metric: autoscalingv2.MetricIdentifier{
1557 Name: "qps",
1558 Selector: &metav1.LabelSelector{},
1559 },
1560 Target: autoscalingv2.MetricTarget{
1561 Type: autoscalingv2.AverageValueMetricType,
1562 AverageValue: resource.NewMilliQuantity(2222, resource.DecimalSI),
1563 },
1564 },
1565 },
1566 },
1567 reportedLevels: []uint64{8600},
1568 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1569 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1570 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1571 autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelScaleUp,
1572 },
1573 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1574 autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelNone,
1575 },
1576 }
1577 tc.runTest(t)
1578 }
1579
1580 func TestScaleDown(t *testing.T) {
1581 tc := testCase{
1582 minReplicas: 2,
1583 maxReplicas: 6,
1584 specReplicas: 5,
1585 statusReplicas: 5,
1586 expectedDesiredReplicas: 3,
1587 CPUTarget: 50,
1588 verifyCPUCurrent: true,
1589 reportedLevels: []uint64{100, 300, 500, 250, 250},
1590 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1591 useMetricsAPI: true,
1592 recommendations: []timestampedRecommendation{},
1593 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
1594 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1595 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1596 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
1597 },
1598 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1599 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1600 },
1601 }
1602 tc.runTest(t)
1603 }
1604
1605 func TestScaleDownContainerResource(t *testing.T) {
1606 tc := testCase{
1607 minReplicas: 2,
1608 maxReplicas: 6,
1609 specReplicas: 5,
1610 statusReplicas: 5,
1611 expectedDesiredReplicas: 3,
1612 reportedLevels: []uint64{100, 300, 500, 250, 250},
1613 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1614 metricsTarget: []autoscalingv2.MetricSpec{{
1615 Type: autoscalingv2.ContainerResourceMetricSourceType,
1616 ContainerResource: &autoscalingv2.ContainerResourceMetricSource{
1617 Container: "container2",
1618 Name: v1.ResourceCPU,
1619 Target: autoscalingv2.MetricTarget{
1620 Type: autoscalingv2.UtilizationMetricType,
1621 AverageUtilization: pointer.Int32(50),
1622 },
1623 },
1624 }},
1625 useMetricsAPI: true,
1626 recommendations: []timestampedRecommendation{},
1627 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
1628 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1629 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1630 autoscalingv2.ContainerResourceMetricSourceType: monitor.ActionLabelScaleDown,
1631 },
1632 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1633 autoscalingv2.ContainerResourceMetricSourceType: monitor.ErrorLabelNone,
1634 },
1635 }
1636 tc.runTest(t)
1637 }
1638
1639 func TestScaleDownWithScalingRules(t *testing.T) {
1640 tc := testCase{
1641 minReplicas: 2,
1642 maxReplicas: 6,
1643 scaleUpRules: generateScalingRules(0, 0, 100, 15, 30),
1644 specReplicas: 5,
1645 statusReplicas: 5,
1646 expectedDesiredReplicas: 3,
1647 CPUTarget: 50,
1648 verifyCPUCurrent: true,
1649 reportedLevels: []uint64{100, 300, 500, 250, 250},
1650 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1651 useMetricsAPI: true,
1652 recommendations: []timestampedRecommendation{},
1653 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
1654 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1655 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1656 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
1657 },
1658 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1659 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1660 },
1661 }
1662 tc.runTest(t)
1663 }
1664
1665 func TestScaleUpOneMetricInvalid(t *testing.T) {
1666 tc := testCase{
1667 minReplicas: 2,
1668 maxReplicas: 6,
1669 specReplicas: 3,
1670 statusReplicas: 3,
1671 expectedDesiredReplicas: 4,
1672 CPUTarget: 30,
1673 verifyCPUCurrent: true,
1674 metricsTarget: []autoscalingv2.MetricSpec{
1675 {
1676 Type: "CheddarCheese",
1677 },
1678 },
1679 reportedLevels: []uint64{300, 400, 500},
1680 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1681 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1682 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
1683 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1684 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
1685
1686 "CheddarCheese": monitor.ActionLabelNone,
1687 },
1688 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1689 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1690
1691 "CheddarCheese": monitor.ErrorLabelSpec,
1692 },
1693 }
1694 tc.runTest(t)
1695 }
1696
1697 func TestScaleUpFromZeroOneMetricInvalid(t *testing.T) {
1698 tc := testCase{
1699 minReplicas: 0,
1700 maxReplicas: 6,
1701 specReplicas: 0,
1702 statusReplicas: 0,
1703 expectedDesiredReplicas: 4,
1704 CPUTarget: 30,
1705 verifyCPUCurrent: true,
1706 metricsTarget: []autoscalingv2.MetricSpec{
1707 {
1708 Type: "CheddarCheese",
1709 },
1710 },
1711 reportedLevels: []uint64{300, 400, 500},
1712 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1713 recommendations: []timestampedRecommendation{},
1714 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
1715 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
1716 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1717 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
1718
1719 "CheddarCheese": monitor.ActionLabelNone,
1720 },
1721 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1722 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1723
1724 "CheddarCheese": monitor.ErrorLabelSpec,
1725 },
1726 }
1727 tc.runTest(t)
1728 }
1729
1730 func TestScaleUpBothMetricsEmpty(t *testing.T) {
1731 tc := testCase{
1732 minReplicas: 2,
1733 maxReplicas: 6,
1734 specReplicas: 3,
1735 statusReplicas: 3,
1736 expectedDesiredReplicas: 3,
1737 CPUTarget: 0,
1738 metricsTarget: []autoscalingv2.MetricSpec{
1739 {
1740 Type: "CheddarCheese",
1741 },
1742 },
1743 reportedLevels: []uint64{},
1744 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1745 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
1746 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"},
1747 {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "InvalidMetricSourceType"},
1748 },
1749 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
1750 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
1751 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1752
1753 "CheddarCheese": monitor.ActionLabelNone,
1754 },
1755 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1756
1757 "CheddarCheese": monitor.ErrorLabelSpec,
1758 },
1759 }
1760 tc.runTest(t)
1761 }
1762
1763 func TestScaleDownStabilizeInitialSize(t *testing.T) {
1764 tc := testCase{
1765 minReplicas: 2,
1766 maxReplicas: 6,
1767 specReplicas: 5,
1768 statusReplicas: 5,
1769 expectedDesiredReplicas: 5,
1770 CPUTarget: 50,
1771 verifyCPUCurrent: true,
1772 reportedLevels: []uint64{100, 300, 500, 250, 250},
1773 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1774 useMetricsAPI: true,
1775 recommendations: nil,
1776 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
1777 Type: autoscalingv2.AbleToScale,
1778 Status: v1.ConditionTrue,
1779 Reason: "ReadyForNewScale",
1780 }, autoscalingv2.HorizontalPodAutoscalerCondition{
1781 Type: autoscalingv2.AbleToScale,
1782 Status: v1.ConditionTrue,
1783 Reason: "ScaleDownStabilized",
1784 }),
1785 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
1786 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1787 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1788 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
1789 },
1790 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1791 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
1792 },
1793 }
1794 tc.runTest(t)
1795 }
1796
1797 func TestScaleDownCM(t *testing.T) {
1798 averageValue := resource.MustParse("20.0")
1799 tc := testCase{
1800 minReplicas: 2,
1801 maxReplicas: 6,
1802 specReplicas: 5,
1803 statusReplicas: 5,
1804 expectedDesiredReplicas: 3,
1805 CPUTarget: 0,
1806 metricsTarget: []autoscalingv2.MetricSpec{
1807 {
1808 Type: autoscalingv2.PodsMetricSourceType,
1809 Pods: &autoscalingv2.PodsMetricSource{
1810 Metric: autoscalingv2.MetricIdentifier{
1811 Name: "qps",
1812 },
1813 Target: autoscalingv2.MetricTarget{
1814 Type: autoscalingv2.AverageValueMetricType,
1815 AverageValue: &averageValue,
1816 },
1817 },
1818 },
1819 },
1820 reportedLevels: []uint64{12000, 12000, 12000, 12000, 12000},
1821 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1822 recommendations: []timestampedRecommendation{},
1823 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
1824 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1825 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1826 autoscalingv2.PodsMetricSourceType: monitor.ActionLabelScaleDown,
1827 },
1828 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1829 autoscalingv2.PodsMetricSourceType: monitor.ErrorLabelNone,
1830 },
1831 }
1832 tc.runTest(t)
1833 }
1834
1835 func TestScaleDownCMObject(t *testing.T) {
1836 targetValue := resource.MustParse("20.0")
1837 tc := testCase{
1838 minReplicas: 2,
1839 maxReplicas: 6,
1840 specReplicas: 5,
1841 statusReplicas: 5,
1842 expectedDesiredReplicas: 3,
1843 CPUTarget: 0,
1844 metricsTarget: []autoscalingv2.MetricSpec{
1845 {
1846 Type: autoscalingv2.ObjectMetricSourceType,
1847 Object: &autoscalingv2.ObjectMetricSource{
1848 DescribedObject: autoscalingv2.CrossVersionObjectReference{
1849 APIVersion: "apps/v1",
1850 Kind: "Deployment",
1851 Name: "some-deployment",
1852 },
1853 Metric: autoscalingv2.MetricIdentifier{
1854 Name: "qps",
1855 },
1856 Target: autoscalingv2.MetricTarget{
1857 Type: autoscalingv2.ValueMetricType,
1858 Value: &targetValue,
1859 },
1860 },
1861 },
1862 },
1863 reportedLevels: []uint64{12000},
1864 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1865 recommendations: []timestampedRecommendation{},
1866 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
1867 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1868 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1869 autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleDown,
1870 },
1871 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1872 autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone,
1873 },
1874 }
1875 tc.runTest(t)
1876 }
1877
1878 func TestScaleDownToZeroCMObject(t *testing.T) {
1879 targetValue := resource.MustParse("20.0")
1880 tc := testCase{
1881 minReplicas: 0,
1882 maxReplicas: 6,
1883 specReplicas: 5,
1884 statusReplicas: 5,
1885 expectedDesiredReplicas: 0,
1886 CPUTarget: 0,
1887 metricsTarget: []autoscalingv2.MetricSpec{
1888 {
1889 Type: autoscalingv2.ObjectMetricSourceType,
1890 Object: &autoscalingv2.ObjectMetricSource{
1891 DescribedObject: autoscalingv2.CrossVersionObjectReference{
1892 APIVersion: "apps/v1",
1893 Kind: "Deployment",
1894 Name: "some-deployment",
1895 },
1896 Metric: autoscalingv2.MetricIdentifier{
1897 Name: "qps",
1898 },
1899 Target: autoscalingv2.MetricTarget{
1900 Type: autoscalingv2.ValueMetricType,
1901 Value: &targetValue,
1902 },
1903 },
1904 },
1905 },
1906 reportedLevels: []uint64{0},
1907 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1908 recommendations: []timestampedRecommendation{},
1909 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
1910 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1911 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1912 autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleDown,
1913 },
1914 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1915 autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone,
1916 },
1917 }
1918 tc.runTest(t)
1919 }
1920
1921 func TestScaleDownPerPodCMObject(t *testing.T) {
1922 targetAverageValue := resource.MustParse("20.0")
1923 tc := testCase{
1924 minReplicas: 2,
1925 maxReplicas: 6,
1926 specReplicas: 5,
1927 statusReplicas: 5,
1928 expectedDesiredReplicas: 3,
1929 CPUTarget: 0,
1930 metricsTarget: []autoscalingv2.MetricSpec{
1931 {
1932 Type: autoscalingv2.ObjectMetricSourceType,
1933 Object: &autoscalingv2.ObjectMetricSource{
1934 DescribedObject: autoscalingv2.CrossVersionObjectReference{
1935 APIVersion: "apps/v1",
1936 Kind: "Deployment",
1937 Name: "some-deployment",
1938 },
1939 Metric: autoscalingv2.MetricIdentifier{
1940 Name: "qps",
1941 },
1942 Target: autoscalingv2.MetricTarget{
1943 Type: autoscalingv2.AverageValueMetricType,
1944 AverageValue: &targetAverageValue,
1945 },
1946 },
1947 },
1948 },
1949 reportedLevels: []uint64{60000},
1950 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
1951 recommendations: []timestampedRecommendation{},
1952 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
1953 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1954 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1955 autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleDown,
1956 },
1957 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1958 autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone,
1959 },
1960 }
1961 tc.runTest(t)
1962 }
1963
1964 func TestScaleDownCMExternal(t *testing.T) {
1965 tc := testCase{
1966 minReplicas: 2,
1967 maxReplicas: 6,
1968 specReplicas: 5,
1969 statusReplicas: 5,
1970 expectedDesiredReplicas: 3,
1971 metricsTarget: []autoscalingv2.MetricSpec{
1972 {
1973 Type: autoscalingv2.ExternalMetricSourceType,
1974 External: &autoscalingv2.ExternalMetricSource{
1975 Metric: autoscalingv2.MetricIdentifier{
1976 Name: "qps",
1977 Selector: &metav1.LabelSelector{},
1978 },
1979 Target: autoscalingv2.MetricTarget{
1980 Type: autoscalingv2.ValueMetricType,
1981 Value: resource.NewMilliQuantity(14400, resource.DecimalSI),
1982 },
1983 },
1984 },
1985 },
1986 reportedLevels: []uint64{8600},
1987 recommendations: []timestampedRecommendation{},
1988 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
1989 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
1990 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
1991 autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelScaleDown,
1992 },
1993 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
1994 autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelNone,
1995 },
1996 }
1997 tc.runTest(t)
1998 }
1999
2000 func TestScaleDownToZeroCMExternal(t *testing.T) {
2001 tc := testCase{
2002 minReplicas: 0,
2003 maxReplicas: 6,
2004 specReplicas: 5,
2005 statusReplicas: 5,
2006 expectedDesiredReplicas: 0,
2007 metricsTarget: []autoscalingv2.MetricSpec{
2008 {
2009 Type: autoscalingv2.ExternalMetricSourceType,
2010 External: &autoscalingv2.ExternalMetricSource{
2011 Metric: autoscalingv2.MetricIdentifier{
2012 Name: "qps",
2013 Selector: &metav1.LabelSelector{},
2014 },
2015 Target: autoscalingv2.MetricTarget{
2016 Type: autoscalingv2.ValueMetricType,
2017 Value: resource.NewMilliQuantity(14400, resource.DecimalSI),
2018 },
2019 },
2020 },
2021 },
2022 reportedLevels: []uint64{0},
2023 recommendations: []timestampedRecommendation{},
2024 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2025 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2026 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2027 autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelScaleDown,
2028 },
2029 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2030 autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelNone,
2031 },
2032 }
2033 tc.runTest(t)
2034 }
2035
2036 func TestScaleDownPerPodCMExternal(t *testing.T) {
2037 tc := testCase{
2038 minReplicas: 2,
2039 maxReplicas: 6,
2040 specReplicas: 5,
2041 statusReplicas: 5,
2042 expectedDesiredReplicas: 3,
2043 metricsTarget: []autoscalingv2.MetricSpec{
2044 {
2045 Type: autoscalingv2.ExternalMetricSourceType,
2046 External: &autoscalingv2.ExternalMetricSource{
2047 Metric: autoscalingv2.MetricIdentifier{
2048 Name: "qps",
2049 Selector: &metav1.LabelSelector{},
2050 },
2051 Target: autoscalingv2.MetricTarget{
2052 Type: autoscalingv2.AverageValueMetricType,
2053 AverageValue: resource.NewMilliQuantity(3000, resource.DecimalSI),
2054 },
2055 },
2056 },
2057 },
2058 reportedLevels: []uint64{8600},
2059 recommendations: []timestampedRecommendation{},
2060 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2061 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2062 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2063 autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelScaleDown,
2064 },
2065 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2066 autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelNone,
2067 },
2068 }
2069 tc.runTest(t)
2070 }
2071
2072 func TestScaleDownIncludeUnreadyPods(t *testing.T) {
2073 tc := testCase{
2074 minReplicas: 2,
2075 maxReplicas: 6,
2076 specReplicas: 5,
2077 statusReplicas: 5,
2078 expectedDesiredReplicas: 2,
2079 CPUTarget: 50,
2080 CPUCurrent: 30,
2081 verifyCPUCurrent: true,
2082 reportedLevels: []uint64{100, 300, 500, 250, 250},
2083 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
2084 useMetricsAPI: true,
2085 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse},
2086 recommendations: []timestampedRecommendation{},
2087 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2088 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2089 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2090 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
2091 },
2092 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2093 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2094 },
2095 }
2096 tc.runTest(t)
2097 }
2098
2099 func TestScaleDownIgnoreHotCpuPods(t *testing.T) {
2100 tc := testCase{
2101 minReplicas: 2,
2102 maxReplicas: 6,
2103 specReplicas: 5,
2104 statusReplicas: 5,
2105 expectedDesiredReplicas: 2,
2106 CPUTarget: 50,
2107 CPUCurrent: 30,
2108 verifyCPUCurrent: true,
2109 reportedLevels: []uint64{100, 300, 500, 250, 250},
2110 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
2111 useMetricsAPI: true,
2112 reportedPodStartTime: []metav1.Time{coolCPUCreationTime(), coolCPUCreationTime(), coolCPUCreationTime(), hotCPUCreationTime(), hotCPUCreationTime()},
2113 recommendations: []timestampedRecommendation{},
2114 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2115 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2116 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2117 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
2118 },
2119 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2120 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2121 },
2122 }
2123 tc.runTest(t)
2124 }
2125
2126 func TestScaleDownIgnoresFailedPods(t *testing.T) {
2127 tc := testCase{
2128 minReplicas: 2,
2129 maxReplicas: 6,
2130 specReplicas: 5,
2131 statusReplicas: 5,
2132 expectedDesiredReplicas: 3,
2133 CPUTarget: 50,
2134 CPUCurrent: 28,
2135 verifyCPUCurrent: true,
2136 reportedLevels: []uint64{100, 300, 500, 250, 250},
2137 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
2138 useMetricsAPI: true,
2139 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse},
2140 reportedPodPhase: []v1.PodPhase{v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodFailed, v1.PodFailed},
2141 recommendations: []timestampedRecommendation{},
2142 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2143 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2144 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2145 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
2146 },
2147 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2148 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2149 },
2150 }
2151 tc.runTest(t)
2152 }
2153
2154 func TestScaleDownIgnoresDeletionPods(t *testing.T) {
2155 tc := testCase{
2156 minReplicas: 2,
2157 maxReplicas: 6,
2158 specReplicas: 5,
2159 statusReplicas: 5,
2160 expectedDesiredReplicas: 3,
2161 CPUTarget: 50,
2162 CPUCurrent: 28,
2163 verifyCPUCurrent: true,
2164 reportedLevels: []uint64{100, 300, 500, 250, 250},
2165 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
2166 useMetricsAPI: true,
2167 reportedPodReadiness: []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionTrue, v1.ConditionFalse, v1.ConditionFalse},
2168 reportedPodPhase: []v1.PodPhase{v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning, v1.PodRunning},
2169 reportedPodDeletionTimestamp: []bool{false, false, false, false, false, true, true},
2170 recommendations: []timestampedRecommendation{},
2171 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2172 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2173 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2174 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
2175 },
2176 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2177 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2178 },
2179 }
2180 tc.runTest(t)
2181 }
2182
2183 func TestTolerance(t *testing.T) {
2184 tc := testCase{
2185 minReplicas: 1,
2186 maxReplicas: 5,
2187 specReplicas: 3,
2188 statusReplicas: 3,
2189 expectedDesiredReplicas: 3,
2190 CPUTarget: 100,
2191 reportedLevels: []uint64{1010, 1030, 1020},
2192 reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")},
2193 useMetricsAPI: true,
2194 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2195 Type: autoscalingv2.AbleToScale,
2196 Status: v1.ConditionTrue,
2197 Reason: "ReadyForNewScale",
2198 }),
2199 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2200 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2201 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2202 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelNone,
2203 },
2204 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2205 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2206 },
2207 }
2208 tc.runTest(t)
2209 }
2210
2211 func TestToleranceCM(t *testing.T) {
2212 averageValue := resource.MustParse("20.0")
2213 tc := testCase{
2214 minReplicas: 1,
2215 maxReplicas: 5,
2216 specReplicas: 3,
2217 statusReplicas: 3,
2218 expectedDesiredReplicas: 3,
2219 metricsTarget: []autoscalingv2.MetricSpec{
2220 {
2221 Type: autoscalingv2.PodsMetricSourceType,
2222 Pods: &autoscalingv2.PodsMetricSource{
2223 Metric: autoscalingv2.MetricIdentifier{
2224 Name: "qps",
2225 },
2226 Target: autoscalingv2.MetricTarget{
2227 Type: autoscalingv2.AverageValueMetricType,
2228 AverageValue: &averageValue,
2229 },
2230 },
2231 },
2232 },
2233 reportedLevels: []uint64{20000, 20001, 21000},
2234 reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")},
2235 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2236 Type: autoscalingv2.AbleToScale,
2237 Status: v1.ConditionTrue,
2238 Reason: "ReadyForNewScale",
2239 }),
2240 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2241 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2242 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2243 autoscalingv2.PodsMetricSourceType: monitor.ActionLabelNone,
2244 },
2245 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2246 autoscalingv2.PodsMetricSourceType: monitor.ErrorLabelNone,
2247 },
2248 }
2249 tc.runTest(t)
2250 }
2251
2252 func TestToleranceCMObject(t *testing.T) {
2253 targetValue := resource.MustParse("20.0")
2254 tc := testCase{
2255 minReplicas: 1,
2256 maxReplicas: 5,
2257 specReplicas: 3,
2258 statusReplicas: 3,
2259 expectedDesiredReplicas: 3,
2260 metricsTarget: []autoscalingv2.MetricSpec{
2261 {
2262 Type: autoscalingv2.ObjectMetricSourceType,
2263 Object: &autoscalingv2.ObjectMetricSource{
2264 DescribedObject: autoscalingv2.CrossVersionObjectReference{
2265 APIVersion: "apps/v1",
2266 Kind: "Deployment",
2267 Name: "some-deployment",
2268 },
2269 Metric: autoscalingv2.MetricIdentifier{
2270 Name: "qps",
2271 },
2272 Target: autoscalingv2.MetricTarget{
2273 Type: autoscalingv2.ValueMetricType,
2274 Value: &targetValue,
2275 },
2276 },
2277 },
2278 },
2279 reportedLevels: []uint64{20050},
2280 reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")},
2281 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2282 Type: autoscalingv2.AbleToScale,
2283 Status: v1.ConditionTrue,
2284 Reason: "ReadyForNewScale",
2285 }),
2286 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2287 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2288 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2289 autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelNone,
2290 },
2291 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2292 autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone,
2293 },
2294 }
2295 tc.runTest(t)
2296 }
2297
2298 func TestToleranceCMExternal(t *testing.T) {
2299 tc := testCase{
2300 minReplicas: 2,
2301 maxReplicas: 6,
2302 specReplicas: 4,
2303 statusReplicas: 4,
2304 expectedDesiredReplicas: 4,
2305 metricsTarget: []autoscalingv2.MetricSpec{
2306 {
2307 Type: autoscalingv2.ExternalMetricSourceType,
2308 External: &autoscalingv2.ExternalMetricSource{
2309 Metric: autoscalingv2.MetricIdentifier{
2310 Name: "qps",
2311 Selector: &metav1.LabelSelector{},
2312 },
2313 Target: autoscalingv2.MetricTarget{
2314 Type: autoscalingv2.ValueMetricType,
2315 Value: resource.NewMilliQuantity(8666, resource.DecimalSI),
2316 },
2317 },
2318 },
2319 },
2320 reportedLevels: []uint64{8600},
2321 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2322 Type: autoscalingv2.AbleToScale,
2323 Status: v1.ConditionTrue,
2324 Reason: "ReadyForNewScale",
2325 }),
2326 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2327 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2328 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2329 autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelNone,
2330 },
2331 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2332 autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelNone,
2333 },
2334 }
2335 tc.runTest(t)
2336 }
2337
2338 func TestTolerancePerPodCMObject(t *testing.T) {
2339 tc := testCase{
2340 minReplicas: 2,
2341 maxReplicas: 6,
2342 specReplicas: 4,
2343 statusReplicas: 4,
2344 expectedDesiredReplicas: 4,
2345 metricsTarget: []autoscalingv2.MetricSpec{
2346 {
2347 Type: autoscalingv2.ObjectMetricSourceType,
2348 Object: &autoscalingv2.ObjectMetricSource{
2349 DescribedObject: autoscalingv2.CrossVersionObjectReference{
2350 APIVersion: "apps/v1",
2351 Kind: "Deployment",
2352 Name: "some-deployment",
2353 },
2354 Metric: autoscalingv2.MetricIdentifier{
2355 Name: "qps",
2356 Selector: &metav1.LabelSelector{},
2357 },
2358 Target: autoscalingv2.MetricTarget{
2359 Type: autoscalingv2.AverageValueMetricType,
2360 AverageValue: resource.NewMilliQuantity(2200, resource.DecimalSI),
2361 },
2362 },
2363 },
2364 },
2365 reportedLevels: []uint64{8600},
2366 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2367 Type: autoscalingv2.AbleToScale,
2368 Status: v1.ConditionTrue,
2369 Reason: "ReadyForNewScale",
2370 }),
2371 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2372 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2373 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2374 autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelNone,
2375 },
2376 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2377 autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone,
2378 },
2379 }
2380 tc.runTest(t)
2381 }
2382
2383 func TestTolerancePerPodCMExternal(t *testing.T) {
2384 tc := testCase{
2385 minReplicas: 2,
2386 maxReplicas: 6,
2387 specReplicas: 4,
2388 statusReplicas: 4,
2389 expectedDesiredReplicas: 4,
2390 metricsTarget: []autoscalingv2.MetricSpec{
2391 {
2392 Type: autoscalingv2.ExternalMetricSourceType,
2393 External: &autoscalingv2.ExternalMetricSource{
2394 Metric: autoscalingv2.MetricIdentifier{
2395 Name: "qps",
2396 Selector: &metav1.LabelSelector{},
2397 },
2398 Target: autoscalingv2.MetricTarget{
2399 Type: autoscalingv2.AverageValueMetricType,
2400 AverageValue: resource.NewMilliQuantity(2200, resource.DecimalSI),
2401 },
2402 },
2403 },
2404 },
2405 reportedLevels: []uint64{8600},
2406 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2407 Type: autoscalingv2.AbleToScale,
2408 Status: v1.ConditionTrue,
2409 Reason: "ReadyForNewScale",
2410 }),
2411 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2412 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2413 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2414 autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelNone,
2415 },
2416 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2417 autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelNone,
2418 },
2419 }
2420 tc.runTest(t)
2421 }
2422
2423 func TestMinReplicas(t *testing.T) {
2424 tc := testCase{
2425 minReplicas: 2,
2426 maxReplicas: 5,
2427 specReplicas: 3,
2428 statusReplicas: 3,
2429 expectedDesiredReplicas: 2,
2430 CPUTarget: 90,
2431 reportedLevels: []uint64{10, 95, 10},
2432 reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")},
2433 useMetricsAPI: true,
2434 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2435 Type: autoscalingv2.ScalingLimited,
2436 Status: v1.ConditionTrue,
2437 Reason: "TooFewReplicas",
2438 }),
2439 recommendations: []timestampedRecommendation{},
2440 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2441 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2442 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2443 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
2444 },
2445 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2446 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2447 },
2448 }
2449 tc.runTest(t)
2450 }
2451
2452 func TestZeroMinReplicasDesiredZero(t *testing.T) {
2453 tc := testCase{
2454 minReplicas: 0,
2455 maxReplicas: 5,
2456 specReplicas: 3,
2457 statusReplicas: 3,
2458 expectedDesiredReplicas: 0,
2459 CPUTarget: 90,
2460 reportedLevels: []uint64{0, 0, 0},
2461 reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")},
2462 useMetricsAPI: true,
2463 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2464 Type: autoscalingv2.ScalingLimited,
2465 Status: v1.ConditionFalse,
2466 Reason: "DesiredWithinRange",
2467 }),
2468 recommendations: []timestampedRecommendation{},
2469 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2470 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2471 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2472 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
2473 },
2474 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2475 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2476 },
2477 }
2478 tc.runTest(t)
2479 }
2480
2481 func TestMinReplicasDesiredZero(t *testing.T) {
2482 tc := testCase{
2483 minReplicas: 2,
2484 maxReplicas: 5,
2485 specReplicas: 3,
2486 statusReplicas: 3,
2487 expectedDesiredReplicas: 2,
2488 CPUTarget: 90,
2489 reportedLevels: []uint64{0, 0, 0},
2490 reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")},
2491 useMetricsAPI: true,
2492 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2493 Type: autoscalingv2.ScalingLimited,
2494 Status: v1.ConditionTrue,
2495 Reason: "TooFewReplicas",
2496 }),
2497 recommendations: []timestampedRecommendation{},
2498 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2499 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2500 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2501 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
2502 },
2503 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2504 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2505 },
2506 }
2507 tc.runTest(t)
2508 }
2509
2510 func TestZeroReplicas(t *testing.T) {
2511 tc := testCase{
2512 minReplicas: 3,
2513 maxReplicas: 5,
2514 specReplicas: 0,
2515 statusReplicas: 0,
2516 expectedDesiredReplicas: 0,
2517 CPUTarget: 90,
2518 reportedLevels: []uint64{},
2519 reportedCPURequests: []resource.Quantity{},
2520 useMetricsAPI: true,
2521 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
2522 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"},
2523 {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "ScalingDisabled"},
2524 },
2525 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2526 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2527 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
2528 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
2529 }
2530 tc.runTest(t)
2531 }
2532
2533 func TestTooFewReplicas(t *testing.T) {
2534 tc := testCase{
2535 minReplicas: 3,
2536 maxReplicas: 5,
2537 specReplicas: 2,
2538 statusReplicas: 2,
2539 expectedDesiredReplicas: 3,
2540 CPUTarget: 90,
2541 reportedLevels: []uint64{},
2542 reportedCPURequests: []resource.Quantity{},
2543 useMetricsAPI: true,
2544 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
2545 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededRescale"},
2546 },
2547 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
2548 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2549 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
2550 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
2551 }
2552 tc.runTest(t)
2553 }
2554
2555 func TestTooManyReplicas(t *testing.T) {
2556 tc := testCase{
2557 minReplicas: 3,
2558 maxReplicas: 5,
2559 specReplicas: 10,
2560 statusReplicas: 10,
2561 expectedDesiredReplicas: 5,
2562 CPUTarget: 90,
2563 reportedLevels: []uint64{},
2564 reportedCPURequests: []resource.Quantity{},
2565 useMetricsAPI: true,
2566 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
2567 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededRescale"},
2568 },
2569 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2570 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2571 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
2572 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
2573 }
2574 tc.runTest(t)
2575 }
2576
2577 func TestMaxReplicas(t *testing.T) {
2578 tc := testCase{
2579 minReplicas: 2,
2580 maxReplicas: 5,
2581 specReplicas: 3,
2582 statusReplicas: 3,
2583 expectedDesiredReplicas: 5,
2584 CPUTarget: 90,
2585 reportedLevels: []uint64{8000, 9500, 1000},
2586 reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")},
2587 useMetricsAPI: true,
2588 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2589 Type: autoscalingv2.ScalingLimited,
2590 Status: v1.ConditionTrue,
2591 Reason: "TooManyReplicas",
2592 }),
2593 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
2594 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2595 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2596 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
2597 },
2598 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2599 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2600 },
2601 }
2602 tc.runTest(t)
2603 }
2604
2605 func TestSuperfluousMetrics(t *testing.T) {
2606 tc := testCase{
2607 minReplicas: 2,
2608 maxReplicas: 6,
2609 specReplicas: 4,
2610 statusReplicas: 4,
2611 expectedDesiredReplicas: 6,
2612 CPUTarget: 100,
2613 reportedLevels: []uint64{4000, 9500, 3000, 7000, 3200, 2000},
2614 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
2615 useMetricsAPI: true,
2616 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2617 Type: autoscalingv2.ScalingLimited,
2618 Status: v1.ConditionTrue,
2619 Reason: "TooManyReplicas",
2620 }),
2621 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
2622 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2623 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2624 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
2625 },
2626 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2627 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2628 },
2629 }
2630 tc.runTest(t)
2631 }
2632
2633 func TestMissingMetrics(t *testing.T) {
2634 tc := testCase{
2635 minReplicas: 2,
2636 maxReplicas: 6,
2637 specReplicas: 4,
2638 statusReplicas: 4,
2639 expectedDesiredReplicas: 3,
2640 CPUTarget: 100,
2641 reportedLevels: []uint64{400, 95},
2642 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
2643 useMetricsAPI: true,
2644 recommendations: []timestampedRecommendation{},
2645 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2646 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2647 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2648 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
2649 },
2650 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2651 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2652 },
2653 }
2654 tc.runTest(t)
2655 }
2656
2657 func TestEmptyMetrics(t *testing.T) {
2658 tc := testCase{
2659 minReplicas: 2,
2660 maxReplicas: 6,
2661 specReplicas: 4,
2662 statusReplicas: 4,
2663 expectedDesiredReplicas: 4,
2664 CPUTarget: 100,
2665 reportedLevels: []uint64{},
2666 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
2667 useMetricsAPI: true,
2668 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
2669 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"},
2670 {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "FailedGetResourceMetric"},
2671 },
2672 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2673 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
2674 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2675 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelNone,
2676 },
2677 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2678 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelInternal,
2679 },
2680 }
2681 tc.runTest(t)
2682 }
2683
2684 func TestEmptyCPURequest(t *testing.T) {
2685 tc := testCase{
2686 minReplicas: 1,
2687 maxReplicas: 5,
2688 specReplicas: 1,
2689 statusReplicas: 1,
2690 expectedDesiredReplicas: 1,
2691 CPUTarget: 100,
2692 reportedLevels: []uint64{200},
2693 reportedCPURequests: []resource.Quantity{},
2694 useMetricsAPI: true,
2695 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
2696 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"},
2697 {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "FailedGetResourceMetric"},
2698 },
2699 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2700 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
2701 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2702 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelNone,
2703 },
2704 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2705 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelInternal,
2706 },
2707 }
2708 tc.runTest(t)
2709 }
2710
2711 func TestEventCreated(t *testing.T) {
2712 tc := testCase{
2713 minReplicas: 1,
2714 maxReplicas: 5,
2715 specReplicas: 1,
2716 statusReplicas: 1,
2717 expectedDesiredReplicas: 2,
2718 CPUTarget: 50,
2719 reportedLevels: []uint64{200},
2720 reportedCPURequests: []resource.Quantity{resource.MustParse("0.2")},
2721 verifyEvents: true,
2722 useMetricsAPI: true,
2723 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
2724 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2725 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2726 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
2727 },
2728 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2729 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2730 },
2731 }
2732 tc.runTest(t)
2733 }
2734
2735 func TestEventNotCreated(t *testing.T) {
2736 tc := testCase{
2737 minReplicas: 1,
2738 maxReplicas: 5,
2739 specReplicas: 2,
2740 statusReplicas: 2,
2741 expectedDesiredReplicas: 2,
2742 CPUTarget: 50,
2743 reportedLevels: []uint64{200, 200},
2744 reportedCPURequests: []resource.Quantity{resource.MustParse("0.4"), resource.MustParse("0.4")},
2745 verifyEvents: true,
2746 useMetricsAPI: true,
2747 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2748 Type: autoscalingv2.AbleToScale,
2749 Status: v1.ConditionTrue,
2750 Reason: "ReadyForNewScale",
2751 }),
2752 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2753 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2754 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2755 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelNone,
2756 },
2757 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2758 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2759 },
2760 }
2761 tc.runTest(t)
2762 }
2763
2764 func TestMissingReports(t *testing.T) {
2765 tc := testCase{
2766 minReplicas: 1,
2767 maxReplicas: 5,
2768 specReplicas: 4,
2769 statusReplicas: 4,
2770 expectedDesiredReplicas: 2,
2771 CPUTarget: 50,
2772 reportedLevels: []uint64{200},
2773 reportedCPURequests: []resource.Quantity{resource.MustParse("0.2")},
2774 useMetricsAPI: true,
2775 recommendations: []timestampedRecommendation{},
2776 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
2777 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2778 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2779 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
2780 },
2781 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2782 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2783 },
2784 }
2785 tc.runTest(t)
2786 }
2787
2788 func TestUpscaleCap(t *testing.T) {
2789 tc := testCase{
2790 minReplicas: 1,
2791 maxReplicas: 100,
2792 specReplicas: 3,
2793 statusReplicas: 3,
2794 scaleUpRules: generateScalingRules(0, 0, 700, 60, 0),
2795 initialReplicas: 3,
2796 expectedDesiredReplicas: 24,
2797 CPUTarget: 10,
2798 reportedLevels: []uint64{100, 200, 300},
2799 reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
2800 useMetricsAPI: true,
2801 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2802 Type: autoscalingv2.ScalingLimited,
2803 Status: v1.ConditionTrue,
2804 Reason: "ScaleUpLimit",
2805 }),
2806 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
2807 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2808 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2809 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
2810 },
2811 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2812 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2813 },
2814 }
2815 tc.runTest(t)
2816 }
2817
2818 func TestUpscaleCapGreaterThanMaxReplicas(t *testing.T) {
2819 tc := testCase{
2820 minReplicas: 1,
2821 maxReplicas: 20,
2822 specReplicas: 3,
2823 statusReplicas: 3,
2824 scaleUpRules: generateScalingRules(0, 0, 700, 60, 0),
2825 initialReplicas: 3,
2826
2827 expectedDesiredReplicas: 20,
2828 CPUTarget: 10,
2829 reportedLevels: []uint64{100, 200, 300},
2830 reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
2831 useMetricsAPI: true,
2832 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2833 Type: autoscalingv2.ScalingLimited,
2834 Status: v1.ConditionTrue,
2835 Reason: "TooManyReplicas",
2836 }),
2837 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
2838 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2839 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2840 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
2841 },
2842 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2843 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2844 },
2845 }
2846 tc.runTest(t)
2847 }
2848
2849 func TestMoreReplicasThanSpecNoScale(t *testing.T) {
2850 tc := testCase{
2851 minReplicas: 1,
2852 maxReplicas: 8,
2853 specReplicas: 4,
2854 statusReplicas: 5,
2855 expectedDesiredReplicas: 4,
2856 CPUTarget: 50,
2857 reportedLevels: []uint64{500, 500, 500, 500, 500},
2858 reportedCPURequests: []resource.Quantity{
2859 resource.MustParse("1"),
2860 resource.MustParse("1"),
2861 resource.MustParse("1"),
2862 resource.MustParse("1"),
2863 resource.MustParse("1"),
2864 },
2865 useMetricsAPI: true,
2866 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
2867 Type: autoscalingv2.AbleToScale,
2868 Status: v1.ConditionTrue,
2869 Reason: "ReadyForNewScale",
2870 }),
2871 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2872 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
2873 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
2874 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelNone,
2875 },
2876 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
2877 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
2878 },
2879 }
2880 tc.runTest(t)
2881 }
2882
2883 func TestConditionInvalidSelectorMissing(t *testing.T) {
2884 tc := testCase{
2885 minReplicas: 1,
2886 maxReplicas: 100,
2887 specReplicas: 3,
2888 statusReplicas: 3,
2889 expectedDesiredReplicas: 3,
2890 CPUTarget: 10,
2891 reportedLevels: []uint64{100, 200, 300},
2892 reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
2893 useMetricsAPI: true,
2894 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
2895 {
2896 Type: autoscalingv2.AbleToScale,
2897 Status: v1.ConditionTrue,
2898 Reason: "SucceededGetScale",
2899 },
2900 {
2901 Type: autoscalingv2.ScalingActive,
2902 Status: v1.ConditionFalse,
2903 Reason: "InvalidSelector",
2904 },
2905 },
2906 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2907 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
2908 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
2909 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
2910 }
2911
2912 _, _, _, _, testScaleClient := tc.prepareTestClient(t)
2913 tc.testScaleClient = testScaleClient
2914
2915 testScaleClient.PrependReactor("get", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
2916 obj := &autoscalingv1.Scale{
2917 ObjectMeta: metav1.ObjectMeta{
2918 Name: tc.resource.name,
2919 },
2920 Spec: autoscalingv1.ScaleSpec{
2921 Replicas: tc.specReplicas,
2922 },
2923 Status: autoscalingv1.ScaleStatus{
2924 Replicas: tc.specReplicas,
2925 },
2926 }
2927 return true, obj, nil
2928 })
2929
2930 tc.runTest(t)
2931 }
2932
2933 func TestConditionInvalidSelectorUnparsable(t *testing.T) {
2934 tc := testCase{
2935 minReplicas: 1,
2936 maxReplicas: 100,
2937 specReplicas: 3,
2938 statusReplicas: 3,
2939 expectedDesiredReplicas: 3,
2940 CPUTarget: 10,
2941 reportedLevels: []uint64{100, 200, 300},
2942 reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
2943 useMetricsAPI: true,
2944 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
2945 {
2946 Type: autoscalingv2.AbleToScale,
2947 Status: v1.ConditionTrue,
2948 Reason: "SucceededGetScale",
2949 },
2950 {
2951 Type: autoscalingv2.ScalingActive,
2952 Status: v1.ConditionFalse,
2953 Reason: "InvalidSelector",
2954 },
2955 },
2956 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
2957 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
2958 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
2959 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
2960 }
2961
2962 _, _, _, _, testScaleClient := tc.prepareTestClient(t)
2963 tc.testScaleClient = testScaleClient
2964
2965 testScaleClient.PrependReactor("get", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
2966 obj := &autoscalingv1.Scale{
2967 ObjectMeta: metav1.ObjectMeta{
2968 Name: tc.resource.name,
2969 },
2970 Spec: autoscalingv1.ScaleSpec{
2971 Replicas: tc.specReplicas,
2972 },
2973 Status: autoscalingv1.ScaleStatus{
2974 Replicas: tc.specReplicas,
2975 Selector: "cheddar cheese",
2976 },
2977 }
2978 return true, obj, nil
2979 })
2980
2981 tc.runTest(t)
2982 }
2983
2984 func TestConditionNoAmbiguousSelectorWhenNoSelectorOverlapBetweenHPAs(t *testing.T) {
2985 hpaSelectors := selectors.NewBiMultimap()
2986 hpaSelectors.PutSelector(selectors.Key{Name: "test-hpa-2", Namespace: testNamespace}, labels.SelectorFromSet(labels.Set{"cheddar": "cheese"}))
2987
2988 tc := testCase{
2989 minReplicas: 2,
2990 maxReplicas: 6,
2991 specReplicas: 3,
2992 statusReplicas: 3,
2993 expectedDesiredReplicas: 5,
2994 CPUTarget: 30,
2995 reportedLevels: []uint64{300, 500, 700},
2996 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
2997 useMetricsAPI: true,
2998 hpaSelectors: hpaSelectors,
2999 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
3000 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
3001 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3002 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
3003 },
3004 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3005 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
3006 },
3007 }
3008 tc.runTest(t)
3009 }
3010
3011 func TestConditionAmbiguousSelectorWhenFullSelectorOverlapBetweenHPAs(t *testing.T) {
3012 hpaSelectors := selectors.NewBiMultimap()
3013 hpaSelectors.PutSelector(selectors.Key{Name: "test-hpa-2", Namespace: testNamespace}, labels.SelectorFromSet(labels.Set{"name": podNamePrefix}))
3014
3015 tc := testCase{
3016 minReplicas: 2,
3017 maxReplicas: 6,
3018 specReplicas: 3,
3019 statusReplicas: 3,
3020 expectedDesiredReplicas: 3,
3021 CPUTarget: 30,
3022 reportedLevels: []uint64{300, 500, 700},
3023 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
3024 useMetricsAPI: true,
3025 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
3026 {
3027 Type: autoscalingv2.AbleToScale,
3028 Status: v1.ConditionTrue,
3029 Reason: "SucceededGetScale",
3030 },
3031 {
3032 Type: autoscalingv2.ScalingActive,
3033 Status: v1.ConditionFalse,
3034 Reason: "AmbiguousSelector",
3035 },
3036 },
3037 hpaSelectors: hpaSelectors,
3038 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
3039 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
3040 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
3041 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
3042 }
3043 tc.runTest(t)
3044 }
3045
3046 func TestConditionAmbiguousSelectorWhenPartialSelectorOverlapBetweenHPAs(t *testing.T) {
3047 hpaSelectors := selectors.NewBiMultimap()
3048 hpaSelectors.PutSelector(selectors.Key{Name: "test-hpa-2", Namespace: testNamespace}, labels.SelectorFromSet(labels.Set{"cheddar": "cheese"}))
3049
3050 tc := testCase{
3051 minReplicas: 2,
3052 maxReplicas: 6,
3053 specReplicas: 3,
3054 statusReplicas: 3,
3055 expectedDesiredReplicas: 3,
3056 CPUTarget: 30,
3057 reportedLevels: []uint64{300, 500, 700},
3058 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
3059 useMetricsAPI: true,
3060 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
3061 {
3062 Type: autoscalingv2.AbleToScale,
3063 Status: v1.ConditionTrue,
3064 Reason: "SucceededGetScale",
3065 },
3066 {
3067 Type: autoscalingv2.ScalingActive,
3068 Status: v1.ConditionFalse,
3069 Reason: "AmbiguousSelector",
3070 },
3071 },
3072 hpaSelectors: hpaSelectors,
3073 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
3074 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
3075 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
3076 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
3077 }
3078
3079 testClient, _, _, _, _ := tc.prepareTestClient(t)
3080 tc.testClient = testClient
3081
3082 testClient.PrependReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
3083 tc.Lock()
3084 defer tc.Unlock()
3085
3086 obj := &v1.PodList{}
3087 for i := range tc.reportedCPURequests {
3088 pod := v1.Pod{
3089 ObjectMeta: metav1.ObjectMeta{
3090 Name: fmt.Sprintf("%s-%d", podNamePrefix, i),
3091 Namespace: testNamespace,
3092 Labels: map[string]string{
3093 "name": podNamePrefix,
3094 "cheddar": "cheese",
3095 },
3096 },
3097 }
3098 obj.Items = append(obj.Items, pod)
3099 }
3100 return true, obj, nil
3101 })
3102
3103 tc.runTest(t)
3104 }
3105
3106 func TestConditionFailedGetMetrics(t *testing.T) {
3107 targetValue := resource.MustParse("15.0")
3108 averageValue := resource.MustParse("15.0")
3109 metricsTargets := map[string][]autoscalingv2.MetricSpec{
3110 "FailedGetResourceMetric": nil,
3111 "FailedGetPodsMetric": {
3112 {
3113 Type: autoscalingv2.PodsMetricSourceType,
3114 Pods: &autoscalingv2.PodsMetricSource{
3115 Metric: autoscalingv2.MetricIdentifier{
3116 Name: "qps",
3117 },
3118 Target: autoscalingv2.MetricTarget{
3119 Type: autoscalingv2.AverageValueMetricType,
3120 AverageValue: &averageValue,
3121 },
3122 },
3123 },
3124 },
3125 "FailedGetObjectMetric": {
3126 {
3127 Type: autoscalingv2.ObjectMetricSourceType,
3128 Object: &autoscalingv2.ObjectMetricSource{
3129 DescribedObject: autoscalingv2.CrossVersionObjectReference{
3130 APIVersion: "apps/v1",
3131 Kind: "Deployment",
3132 Name: "some-deployment",
3133 },
3134 Metric: autoscalingv2.MetricIdentifier{
3135 Name: "qps",
3136 },
3137 Target: autoscalingv2.MetricTarget{
3138 Type: autoscalingv2.ValueMetricType,
3139 Value: &targetValue,
3140 },
3141 },
3142 },
3143 },
3144 "FailedGetExternalMetric": {
3145 {
3146 Type: autoscalingv2.ExternalMetricSourceType,
3147 External: &autoscalingv2.ExternalMetricSource{
3148 Metric: autoscalingv2.MetricIdentifier{
3149 Name: "qps",
3150 Selector: &metav1.LabelSelector{},
3151 },
3152 Target: autoscalingv2.MetricTarget{
3153 Type: autoscalingv2.ValueMetricType,
3154 Value: resource.NewMilliQuantity(300, resource.DecimalSI),
3155 },
3156 },
3157 },
3158 },
3159 }
3160
3161 for reason, specs := range metricsTargets {
3162 metricType := autoscalingv2.ResourceMetricSourceType
3163 if specs != nil {
3164 metricType = specs[0].Type
3165 }
3166 tc := testCase{
3167 minReplicas: 1,
3168 maxReplicas: 100,
3169 specReplicas: 3,
3170 statusReplicas: 3,
3171 expectedDesiredReplicas: 3,
3172 CPUTarget: 10,
3173 reportedLevels: []uint64{100, 200, 300},
3174 reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
3175 useMetricsAPI: true,
3176 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
3177 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
3178 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3179 metricType: monitor.ActionLabelNone,
3180 },
3181 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3182 metricType: monitor.ErrorLabelInternal,
3183 },
3184 }
3185 _, testMetricsClient, testCMClient, testEMClient, _ := tc.prepareTestClient(t)
3186 tc.testMetricsClient = testMetricsClient
3187 tc.testCMClient = testCMClient
3188 tc.testEMClient = testEMClient
3189
3190 testMetricsClient.PrependReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
3191 return true, &metricsapi.PodMetricsList{}, fmt.Errorf("something went wrong")
3192 })
3193 testCMClient.PrependReactor("get", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
3194 return true, &cmapi.MetricValueList{}, fmt.Errorf("something went wrong")
3195 })
3196 testEMClient.PrependReactor("list", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
3197 return true, &emapi.ExternalMetricValueList{}, fmt.Errorf("something went wrong")
3198 })
3199
3200 tc.expectedConditions = []autoscalingv2.HorizontalPodAutoscalerCondition{
3201 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"},
3202 {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: reason},
3203 }
3204 if specs != nil {
3205 tc.CPUTarget = 0
3206 } else {
3207 tc.CPUTarget = 10
3208 }
3209 tc.metricsTarget = specs
3210 tc.runTest(t)
3211 }
3212 }
3213
3214 func TestConditionInvalidSourceType(t *testing.T) {
3215 tc := testCase{
3216 minReplicas: 2,
3217 maxReplicas: 6,
3218 specReplicas: 3,
3219 statusReplicas: 3,
3220 expectedDesiredReplicas: 3,
3221 CPUTarget: 0,
3222 metricsTarget: []autoscalingv2.MetricSpec{
3223 {
3224 Type: "CheddarCheese",
3225 },
3226 },
3227 reportedLevels: []uint64{20000},
3228 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
3229 {
3230 Type: autoscalingv2.AbleToScale,
3231 Status: v1.ConditionTrue,
3232 Reason: "SucceededGetScale",
3233 },
3234 {
3235 Type: autoscalingv2.ScalingActive,
3236 Status: v1.ConditionFalse,
3237 Reason: "InvalidMetricSourceType",
3238 },
3239 },
3240 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
3241 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
3242 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3243
3244 "CheddarCheese": monitor.ActionLabelNone,
3245 },
3246 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3247
3248 "CheddarCheese": monitor.ErrorLabelSpec,
3249 },
3250 }
3251 tc.runTest(t)
3252 }
3253
3254 func TestConditionFailedGetScale(t *testing.T) {
3255 tc := testCase{
3256 minReplicas: 1,
3257 maxReplicas: 100,
3258 specReplicas: 3,
3259 statusReplicas: 3,
3260 expectedDesiredReplicas: 3,
3261 CPUTarget: 10,
3262 reportedLevels: []uint64{100, 200, 300},
3263 reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
3264 useMetricsAPI: true,
3265 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
3266 {
3267 Type: autoscalingv2.AbleToScale,
3268 Status: v1.ConditionFalse,
3269 Reason: "FailedGetScale",
3270 },
3271 },
3272 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
3273 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
3274 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
3275 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
3276 }
3277
3278 _, _, _, _, testScaleClient := tc.prepareTestClient(t)
3279 tc.testScaleClient = testScaleClient
3280
3281 testScaleClient.PrependReactor("get", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
3282 return true, &autoscalingv1.Scale{}, fmt.Errorf("something went wrong")
3283 })
3284
3285 tc.runTest(t)
3286 }
3287
3288 func TestConditionFailedUpdateScale(t *testing.T) {
3289 tc := testCase{
3290 minReplicas: 1,
3291 maxReplicas: 5,
3292 specReplicas: 3,
3293 statusReplicas: 3,
3294 expectedDesiredReplicas: 3,
3295 CPUTarget: 100,
3296 reportedLevels: []uint64{150, 150, 150},
3297 reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
3298 useMetricsAPI: true,
3299 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
3300 Type: autoscalingv2.AbleToScale,
3301 Status: v1.ConditionFalse,
3302 Reason: "FailedUpdateScale",
3303 }),
3304 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
3305 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
3306 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3307 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
3308 },
3309 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3310 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
3311 },
3312 }
3313
3314 _, _, _, _, testScaleClient := tc.prepareTestClient(t)
3315 tc.testScaleClient = testScaleClient
3316
3317 testScaleClient.PrependReactor("update", "replicationcontrollers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
3318 return true, &autoscalingv1.Scale{}, fmt.Errorf("something went wrong")
3319 })
3320
3321 tc.runTest(t)
3322 }
3323
3324 func TestNoBackoffUpscaleCM(t *testing.T) {
3325 averageValue := resource.MustParse("15.0")
3326 time := metav1.Time{Time: time.Now()}
3327 tc := testCase{
3328 minReplicas: 1,
3329 maxReplicas: 5,
3330 specReplicas: 3,
3331 statusReplicas: 3,
3332 expectedDesiredReplicas: 4,
3333 CPUTarget: 0,
3334 metricsTarget: []autoscalingv2.MetricSpec{
3335 {
3336 Type: autoscalingv2.PodsMetricSourceType,
3337 Pods: &autoscalingv2.PodsMetricSource{
3338 Metric: autoscalingv2.MetricIdentifier{
3339 Name: "qps",
3340 },
3341 Target: autoscalingv2.MetricTarget{
3342 Type: autoscalingv2.AverageValueMetricType,
3343 AverageValue: &averageValue,
3344 },
3345 },
3346 },
3347 },
3348 reportedLevels: []uint64{20000, 10000, 30000},
3349 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
3350
3351 lastScaleTime: &time,
3352 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
3353 Type: autoscalingv2.AbleToScale,
3354 Status: v1.ConditionTrue,
3355 Reason: "ReadyForNewScale",
3356 }, autoscalingv2.HorizontalPodAutoscalerCondition{
3357 Type: autoscalingv2.AbleToScale,
3358 Status: v1.ConditionTrue,
3359 Reason: "SucceededRescale",
3360 }, autoscalingv2.HorizontalPodAutoscalerCondition{
3361 Type: autoscalingv2.ScalingLimited,
3362 Status: v1.ConditionFalse,
3363 Reason: "DesiredWithinRange",
3364 }),
3365 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
3366 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
3367 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3368 autoscalingv2.PodsMetricSourceType: monitor.ActionLabelScaleUp,
3369 },
3370 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3371 autoscalingv2.PodsMetricSourceType: monitor.ErrorLabelNone,
3372 },
3373 }
3374 tc.runTest(t)
3375 }
3376
3377 func TestNoBackoffUpscaleCMNoBackoffCpu(t *testing.T) {
3378 averageValue := resource.MustParse("15.0")
3379 time := metav1.Time{Time: time.Now()}
3380 tc := testCase{
3381 minReplicas: 1,
3382 maxReplicas: 5,
3383 specReplicas: 3,
3384 statusReplicas: 3,
3385 expectedDesiredReplicas: 5,
3386 CPUTarget: 10,
3387 metricsTarget: []autoscalingv2.MetricSpec{
3388 {
3389 Type: autoscalingv2.PodsMetricSourceType,
3390 Pods: &autoscalingv2.PodsMetricSource{
3391 Metric: autoscalingv2.MetricIdentifier{
3392 Name: "qps",
3393 },
3394 Target: autoscalingv2.MetricTarget{
3395 Type: autoscalingv2.AverageValueMetricType,
3396 AverageValue: &averageValue,
3397 },
3398 },
3399 },
3400 },
3401 reportedLevels: []uint64{20000, 10000, 30000},
3402 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
3403 useMetricsAPI: true,
3404 lastScaleTime: &time,
3405 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
3406 Type: autoscalingv2.AbleToScale,
3407 Status: v1.ConditionTrue,
3408 Reason: "ReadyForNewScale",
3409 }, autoscalingv2.HorizontalPodAutoscalerCondition{
3410 Type: autoscalingv2.AbleToScale,
3411 Status: v1.ConditionTrue,
3412 Reason: "SucceededRescale",
3413 }, autoscalingv2.HorizontalPodAutoscalerCondition{
3414 Type: autoscalingv2.ScalingLimited,
3415 Status: v1.ConditionTrue,
3416 Reason: "TooManyReplicas",
3417 }),
3418 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
3419 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
3420 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3421 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
3422 autoscalingv2.PodsMetricSourceType: monitor.ActionLabelScaleUp,
3423 },
3424 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3425 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
3426 autoscalingv2.PodsMetricSourceType: monitor.ErrorLabelNone,
3427 },
3428 }
3429 tc.runTest(t)
3430 }
3431
3432 func TestStabilizeDownscale(t *testing.T) {
3433 tc := testCase{
3434 minReplicas: 1,
3435 maxReplicas: 5,
3436 specReplicas: 4,
3437 statusReplicas: 4,
3438 expectedDesiredReplicas: 4,
3439 CPUTarget: 100,
3440 reportedLevels: []uint64{50, 50, 50},
3441 reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
3442 useMetricsAPI: true,
3443 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
3444 Type: autoscalingv2.AbleToScale,
3445 Status: v1.ConditionTrue,
3446 Reason: "ReadyForNewScale",
3447 }, autoscalingv2.HorizontalPodAutoscalerCondition{
3448 Type: autoscalingv2.AbleToScale,
3449 Status: v1.ConditionTrue,
3450 Reason: "ScaleDownStabilized",
3451 }),
3452 recommendations: []timestampedRecommendation{
3453 {10, time.Now().Add(-10 * time.Minute)},
3454 {4, time.Now().Add(-1 * time.Minute)},
3455 },
3456 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
3457 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
3458 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3459 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
3460 },
3461 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3462 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
3463 },
3464 }
3465 tc.runTest(t)
3466 }
3467
3468
3469
3470
3471 func TestComputedToleranceAlgImplementation(t *testing.T) {
3472
3473 startPods := int32(10)
3474
3475 totalUsedCPUOfAllPods := uint64(startPods * 150)
3476
3477
3478 totalRequestedCPUOfAllPods := int32(2 * totalUsedCPUOfAllPods)
3479 requestedToUsed := float64(totalRequestedCPUOfAllPods / int32(totalUsedCPUOfAllPods))
3480
3481 perPodRequested := totalRequestedCPUOfAllPods / startPods
3482
3483
3484 target := math.Abs(1/(requestedToUsed*(1-defaultTestingTolerance))) + .01
3485 finalCPUPercentTarget := int32(target * 100)
3486 resourcesUsedRatio := float64(totalUsedCPUOfAllPods) / float64(float64(totalRequestedCPUOfAllPods)*target)
3487
3488
3489 finalPods := int32(math.Ceil(resourcesUsedRatio * float64(startPods)))
3490
3491
3492 tc1 := testCase{
3493 minReplicas: 0,
3494 maxReplicas: 1000,
3495 specReplicas: startPods,
3496 statusReplicas: startPods,
3497 expectedDesiredReplicas: finalPods,
3498 CPUTarget: finalCPUPercentTarget,
3499 reportedLevels: []uint64{
3500 totalUsedCPUOfAllPods / 10,
3501 totalUsedCPUOfAllPods / 10,
3502 totalUsedCPUOfAllPods / 10,
3503 totalUsedCPUOfAllPods / 10,
3504 totalUsedCPUOfAllPods / 10,
3505 totalUsedCPUOfAllPods / 10,
3506 totalUsedCPUOfAllPods / 10,
3507 totalUsedCPUOfAllPods / 10,
3508 totalUsedCPUOfAllPods / 10,
3509 totalUsedCPUOfAllPods / 10,
3510 },
3511 reportedCPURequests: []resource.Quantity{
3512 resource.MustParse(fmt.Sprint(perPodRequested+100) + "m"),
3513 resource.MustParse(fmt.Sprint(perPodRequested-100) + "m"),
3514 resource.MustParse(fmt.Sprint(perPodRequested+10) + "m"),
3515 resource.MustParse(fmt.Sprint(perPodRequested-10) + "m"),
3516 resource.MustParse(fmt.Sprint(perPodRequested+2) + "m"),
3517 resource.MustParse(fmt.Sprint(perPodRequested-2) + "m"),
3518 resource.MustParse(fmt.Sprint(perPodRequested+1) + "m"),
3519 resource.MustParse(fmt.Sprint(perPodRequested-1) + "m"),
3520 resource.MustParse(fmt.Sprint(perPodRequested) + "m"),
3521 resource.MustParse(fmt.Sprint(perPodRequested) + "m"),
3522 },
3523 useMetricsAPI: true,
3524 recommendations: []timestampedRecommendation{},
3525 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
3526 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
3527 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3528 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
3529 },
3530 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3531 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
3532 },
3533 }
3534 tc1.runTest(t)
3535
3536 target = math.Abs(1/(requestedToUsed*(1-defaultTestingTolerance))) + .004
3537 finalCPUPercentTarget = int32(target * 100)
3538 tc2 := testCase{
3539 minReplicas: 0,
3540 maxReplicas: 1000,
3541 specReplicas: startPods,
3542 statusReplicas: startPods,
3543 expectedDesiredReplicas: startPods,
3544 CPUTarget: finalCPUPercentTarget,
3545 reportedLevels: []uint64{
3546 totalUsedCPUOfAllPods / 10,
3547 totalUsedCPUOfAllPods / 10,
3548 totalUsedCPUOfAllPods / 10,
3549 totalUsedCPUOfAllPods / 10,
3550 totalUsedCPUOfAllPods / 10,
3551 totalUsedCPUOfAllPods / 10,
3552 totalUsedCPUOfAllPods / 10,
3553 totalUsedCPUOfAllPods / 10,
3554 totalUsedCPUOfAllPods / 10,
3555 totalUsedCPUOfAllPods / 10,
3556 },
3557 reportedCPURequests: []resource.Quantity{
3558 resource.MustParse(fmt.Sprint(perPodRequested+100) + "m"),
3559 resource.MustParse(fmt.Sprint(perPodRequested-100) + "m"),
3560 resource.MustParse(fmt.Sprint(perPodRequested+10) + "m"),
3561 resource.MustParse(fmt.Sprint(perPodRequested-10) + "m"),
3562 resource.MustParse(fmt.Sprint(perPodRequested+2) + "m"),
3563 resource.MustParse(fmt.Sprint(perPodRequested-2) + "m"),
3564 resource.MustParse(fmt.Sprint(perPodRequested+1) + "m"),
3565 resource.MustParse(fmt.Sprint(perPodRequested-1) + "m"),
3566 resource.MustParse(fmt.Sprint(perPodRequested) + "m"),
3567 resource.MustParse(fmt.Sprint(perPodRequested) + "m"),
3568 },
3569 useMetricsAPI: true,
3570 recommendations: []timestampedRecommendation{},
3571 expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
3572 Type: autoscalingv2.AbleToScale,
3573 Status: v1.ConditionTrue,
3574 Reason: "ReadyForNewScale",
3575 }),
3576 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
3577 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
3578 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3579 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelNone,
3580 },
3581 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3582 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
3583 },
3584 }
3585 tc2.runTest(t)
3586 }
3587
3588 func TestScaleUpRCImmediately(t *testing.T) {
3589 time := metav1.Time{Time: time.Now()}
3590 tc := testCase{
3591 minReplicas: 2,
3592 maxReplicas: 6,
3593 specReplicas: 1,
3594 statusReplicas: 1,
3595 expectedDesiredReplicas: 2,
3596 verifyCPUCurrent: false,
3597 reportedLevels: []uint64{0, 0, 0, 0},
3598 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
3599 useMetricsAPI: true,
3600 lastScaleTime: &time,
3601 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
3602 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededRescale"},
3603 },
3604 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
3605 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
3606 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
3607 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
3608 }
3609 tc.runTest(t)
3610 }
3611
3612 func TestScaleDownRCImmediately(t *testing.T) {
3613 time := metav1.Time{Time: time.Now()}
3614 tc := testCase{
3615 minReplicas: 2,
3616 maxReplicas: 5,
3617 specReplicas: 6,
3618 statusReplicas: 6,
3619 expectedDesiredReplicas: 5,
3620 CPUTarget: 50,
3621 reportedLevels: []uint64{8000, 9500, 1000},
3622 reportedCPURequests: []resource.Quantity{resource.MustParse("0.9"), resource.MustParse("1.0"), resource.MustParse("1.1")},
3623 useMetricsAPI: true,
3624 lastScaleTime: &time,
3625 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
3626 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededRescale"},
3627 },
3628 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown,
3629 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
3630 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{},
3631 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{},
3632 }
3633 tc.runTest(t)
3634 }
3635
3636 func TestAvoidUnnecessaryUpdates(t *testing.T) {
3637 now := metav1.Time{Time: time.Now().Add(-time.Hour)}
3638 tc := testCase{
3639 minReplicas: 2,
3640 maxReplicas: 6,
3641 specReplicas: 2,
3642 statusReplicas: 2,
3643 expectedDesiredReplicas: 2,
3644 CPUTarget: 30,
3645 CPUCurrent: 40,
3646 verifyCPUCurrent: true,
3647 reportedLevels: []uint64{400, 500, 700},
3648 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
3649 reportedPodStartTime: []metav1.Time{coolCPUCreationTime(), hotCPUCreationTime(), hotCPUCreationTime()},
3650 useMetricsAPI: true,
3651 lastScaleTime: &now,
3652 recommendations: []timestampedRecommendation{},
3653 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
3654 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone,
3655 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
3656 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
3657 },
3658 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
3659 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
3660 },
3661 }
3662 testClient, _, _, _, _ := tc.prepareTestClient(t)
3663 tc.testClient = testClient
3664 testClient.PrependReactor("list", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
3665 tc.Lock()
3666 defer tc.Unlock()
3667
3668 go func() {
3669
3670
3671 time.Sleep(1 * time.Second)
3672 tc.Lock()
3673 tc.statusUpdated = true
3674 tc.Unlock()
3675 tc.processed <- "test-hpa"
3676 }()
3677
3678 var eighty int32 = 80
3679
3680 quantity := resource.MustParse("400m")
3681 obj := &autoscalingv2.HorizontalPodAutoscalerList{
3682 Items: []autoscalingv2.HorizontalPodAutoscaler{
3683 {
3684 ObjectMeta: metav1.ObjectMeta{
3685 Name: "test-hpa",
3686 Namespace: "test-namespace",
3687 },
3688 Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
3689 ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{
3690 Kind: "ReplicationController",
3691 Name: "test-rc",
3692 APIVersion: "v1",
3693 },
3694 Metrics: []autoscalingv2.MetricSpec{{
3695 Type: autoscalingv2.ResourceMetricSourceType,
3696 Resource: &autoscalingv2.ResourceMetricSource{
3697 Name: v1.ResourceCPU,
3698 Target: autoscalingv2.MetricTarget{
3699 Type: autoscalingv2.UtilizationMetricType,
3700
3701
3702
3703
3704
3705 AverageUtilization: &eighty,
3706 },
3707 },
3708 }},
3709 MinReplicas: &tc.minReplicas,
3710 MaxReplicas: tc.maxReplicas,
3711 },
3712 Status: autoscalingv2.HorizontalPodAutoscalerStatus{
3713 CurrentReplicas: tc.specReplicas,
3714 DesiredReplicas: tc.specReplicas,
3715 LastScaleTime: tc.lastScaleTime,
3716 CurrentMetrics: []autoscalingv2.MetricStatus{
3717 {
3718 Type: autoscalingv2.ResourceMetricSourceType,
3719 Resource: &autoscalingv2.ResourceMetricStatus{
3720 Name: v1.ResourceCPU,
3721 Current: autoscalingv2.MetricValueStatus{
3722 AverageValue: &quantity,
3723 AverageUtilization: &tc.CPUCurrent,
3724 },
3725 },
3726 },
3727 },
3728 Conditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
3729 {
3730 Type: autoscalingv2.AbleToScale,
3731 Status: v1.ConditionTrue,
3732 LastTransitionTime: *tc.lastScaleTime,
3733 Reason: "ReadyForNewScale",
3734 Message: "recommended size matches current size",
3735 },
3736 {
3737 Type: autoscalingv2.ScalingActive,
3738 Status: v1.ConditionTrue,
3739 LastTransitionTime: *tc.lastScaleTime,
3740 Reason: "ValidMetricFound",
3741 Message: "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)",
3742 },
3743 {
3744 Type: autoscalingv2.ScalingLimited,
3745 Status: v1.ConditionTrue,
3746 LastTransitionTime: *tc.lastScaleTime,
3747 Reason: "TooFewReplicas",
3748 Message: "the desired replica count is less than the minimum replica count",
3749 },
3750 },
3751 },
3752 },
3753 },
3754 }
3755
3756 return true, obj, nil
3757 })
3758 testClient.PrependReactor("update", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
3759 assert.Fail(t, "should not have attempted to update the HPA when nothing changed")
3760
3761 tc.processed <- ""
3762 return true, nil, fmt.Errorf("unexpected call")
3763 })
3764
3765 controller, informerFactory := tc.setupController(t)
3766 tc.runTestWithController(t, controller, informerFactory)
3767 }
3768
3769 func TestConvertDesiredReplicasWithRules(t *testing.T) {
3770 conversionTestCases := []struct {
3771 currentReplicas int32
3772 expectedDesiredReplicas int32
3773 hpaMinReplicas int32
3774 hpaMaxReplicas int32
3775 expectedConvertedDesiredReplicas int32
3776 expectedCondition string
3777 annotation string
3778 }{
3779 {
3780 currentReplicas: 5,
3781 expectedDesiredReplicas: 7,
3782 hpaMinReplicas: 3,
3783 hpaMaxReplicas: 8,
3784 expectedConvertedDesiredReplicas: 7,
3785 expectedCondition: "DesiredWithinRange",
3786 annotation: "prenormalized desired replicas within range",
3787 },
3788 {
3789 currentReplicas: 3,
3790 expectedDesiredReplicas: 1,
3791 hpaMinReplicas: 2,
3792 hpaMaxReplicas: 8,
3793 expectedConvertedDesiredReplicas: 2,
3794 expectedCondition: "TooFewReplicas",
3795 annotation: "prenormalized desired replicas < minReplicas",
3796 },
3797 {
3798 currentReplicas: 1,
3799 expectedDesiredReplicas: 0,
3800 hpaMinReplicas: 0,
3801 hpaMaxReplicas: 10,
3802 expectedConvertedDesiredReplicas: 0,
3803 expectedCondition: "DesiredWithinRange",
3804 annotation: "prenormalized desired zeroed replicas within range",
3805 },
3806 {
3807 currentReplicas: 20,
3808 expectedDesiredReplicas: 1000,
3809 hpaMinReplicas: 1,
3810 hpaMaxReplicas: 10,
3811 expectedConvertedDesiredReplicas: 10,
3812 expectedCondition: "TooManyReplicas",
3813 annotation: "maxReplicas is the limit because maxReplicas < scaleUpLimit",
3814 },
3815 {
3816 currentReplicas: 3,
3817 expectedDesiredReplicas: 1000,
3818 hpaMinReplicas: 1,
3819 hpaMaxReplicas: 2000,
3820 expectedConvertedDesiredReplicas: calculateScaleUpLimit(3),
3821 expectedCondition: "ScaleUpLimit",
3822 annotation: "scaleUpLimit is the limit because scaleUpLimit < maxReplicas",
3823 },
3824 }
3825
3826 for _, ctc := range conversionTestCases {
3827 t.Run(ctc.annotation, func(t *testing.T) {
3828 actualConvertedDesiredReplicas, actualCondition, _ := convertDesiredReplicasWithRules(
3829 ctc.currentReplicas, ctc.expectedDesiredReplicas, ctc.hpaMinReplicas, ctc.hpaMaxReplicas,
3830 )
3831
3832 assert.Equal(t, ctc.expectedConvertedDesiredReplicas, actualConvertedDesiredReplicas, ctc.annotation)
3833 assert.Equal(t, ctc.expectedCondition, actualCondition, ctc.annotation)
3834 })
3835 }
3836 }
3837
3838 func TestCalculateScaleUpLimitWithScalingRules(t *testing.T) {
3839 policy := autoscalingv2.MinChangePolicySelect
3840
3841 calculated := calculateScaleUpLimitWithScalingRules(1, []timestampedScaleEvent{}, []timestampedScaleEvent{}, &autoscalingv2.HPAScalingRules{
3842 StabilizationWindowSeconds: pointer.Int32(300),
3843 SelectPolicy: &policy,
3844 Policies: []autoscalingv2.HPAScalingPolicy{
3845 {
3846 Type: autoscalingv2.PodsScalingPolicy,
3847 Value: 2,
3848 PeriodSeconds: 60,
3849 },
3850 {
3851 Type: autoscalingv2.PercentScalingPolicy,
3852 Value: 50,
3853 PeriodSeconds: 60,
3854 },
3855 },
3856 })
3857 assert.Equal(t, calculated, int32(2))
3858 }
3859
3860 func TestCalculateScaleDownLimitWithBehaviors(t *testing.T) {
3861 policy := autoscalingv2.MinChangePolicySelect
3862
3863 calculated := calculateScaleDownLimitWithBehaviors(5, []timestampedScaleEvent{}, []timestampedScaleEvent{}, &autoscalingv2.HPAScalingRules{
3864 StabilizationWindowSeconds: pointer.Int32(300),
3865 SelectPolicy: &policy,
3866 Policies: []autoscalingv2.HPAScalingPolicy{
3867 {
3868 Type: autoscalingv2.PodsScalingPolicy,
3869 Value: 2,
3870 PeriodSeconds: 60,
3871 },
3872 {
3873 Type: autoscalingv2.PercentScalingPolicy,
3874 Value: 50,
3875 PeriodSeconds: 60,
3876 },
3877 },
3878 })
3879 assert.Equal(t, calculated, int32(3))
3880 }
3881
3882 func generateScalingRules(pods, podsPeriod, percent, percentPeriod, stabilizationWindow int32) *autoscalingv2.HPAScalingRules {
3883 policy := autoscalingv2.MaxChangePolicySelect
3884 directionBehavior := autoscalingv2.HPAScalingRules{
3885 StabilizationWindowSeconds: pointer.Int32(stabilizationWindow),
3886 SelectPolicy: &policy,
3887 }
3888 if pods != 0 {
3889 directionBehavior.Policies = append(directionBehavior.Policies,
3890 autoscalingv2.HPAScalingPolicy{Type: autoscalingv2.PodsScalingPolicy, Value: pods, PeriodSeconds: podsPeriod})
3891 }
3892 if percent != 0 {
3893 directionBehavior.Policies = append(directionBehavior.Policies,
3894 autoscalingv2.HPAScalingPolicy{Type: autoscalingv2.PercentScalingPolicy, Value: percent, PeriodSeconds: percentPeriod})
3895 }
3896 return &directionBehavior
3897 }
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917 func generateEventsUniformDistribution(rawEvents []int, periodSeconds int) []timestampedScaleEvent {
3918 events := make([]timestampedScaleEvent, len(rawEvents))
3919 segmentDuration := float64(periodSeconds) / float64(len(rawEvents))
3920 for idx, event := range rawEvents {
3921 segmentBoundary := time.Duration(float64(periodSeconds) - segmentDuration*float64(idx+1) + segmentDuration/float64(2))
3922 events[idx] = timestampedScaleEvent{
3923 replicaChange: int32(event),
3924 timestamp: time.Now().Add(-time.Second * segmentBoundary),
3925 }
3926 }
3927 return events
3928 }
3929
3930 func TestNormalizeDesiredReplicas(t *testing.T) {
3931 tests := []struct {
3932 name string
3933 key string
3934 recommendations []timestampedRecommendation
3935 prenormalizedDesiredReplicas int32
3936 expectedStabilizedReplicas int32
3937 expectedLogLength int
3938 }{
3939 {
3940 "empty log",
3941 "",
3942 []timestampedRecommendation{},
3943 5,
3944 5,
3945 1,
3946 },
3947 {
3948 "stabilize",
3949 "",
3950 []timestampedRecommendation{
3951 {4, time.Now().Add(-2 * time.Minute)},
3952 {5, time.Now().Add(-1 * time.Minute)},
3953 },
3954 3,
3955 5,
3956 3,
3957 },
3958 {
3959 "no stabilize",
3960 "",
3961 []timestampedRecommendation{
3962 {1, time.Now().Add(-2 * time.Minute)},
3963 {2, time.Now().Add(-1 * time.Minute)},
3964 },
3965 3,
3966 3,
3967 3,
3968 },
3969 {
3970 "no stabilize - old recommendations",
3971 "",
3972 []timestampedRecommendation{
3973 {10, time.Now().Add(-10 * time.Minute)},
3974 {9, time.Now().Add(-9 * time.Minute)},
3975 },
3976 3,
3977 3,
3978 2,
3979 },
3980 {
3981 "stabilize - old recommendations",
3982 "",
3983 []timestampedRecommendation{
3984 {10, time.Now().Add(-10 * time.Minute)},
3985 {4, time.Now().Add(-1 * time.Minute)},
3986 {5, time.Now().Add(-2 * time.Minute)},
3987 {9, time.Now().Add(-9 * time.Minute)},
3988 },
3989 3,
3990 5,
3991 4,
3992 },
3993 }
3994 for _, tc := range tests {
3995 hc := HorizontalController{
3996 downscaleStabilisationWindow: 5 * time.Minute,
3997 recommendations: map[string][]timestampedRecommendation{
3998 tc.key: tc.recommendations,
3999 },
4000 }
4001 r := hc.stabilizeRecommendation(tc.key, tc.prenormalizedDesiredReplicas)
4002 if r != tc.expectedStabilizedReplicas {
4003 t.Errorf("[%s] got %d stabilized replicas, expected %d", tc.name, r, tc.expectedStabilizedReplicas)
4004 }
4005 if len(hc.recommendations[tc.key]) != tc.expectedLogLength {
4006 t.Errorf("[%s] after stabilization recommendations log has %d entries, expected %d", tc.name, len(hc.recommendations[tc.key]), tc.expectedLogLength)
4007 }
4008 }
4009 }
4010
4011 func TestScalingWithRules(t *testing.T) {
4012 type TestCase struct {
4013 name string
4014 key string
4015
4016 scaleUpEvents []timestampedScaleEvent
4017 scaleDownEvents []timestampedScaleEvent
4018
4019 specMinReplicas int32
4020 specMaxReplicas int32
4021 scaleUpRules *autoscalingv2.HPAScalingRules
4022 scaleDownRules *autoscalingv2.HPAScalingRules
4023
4024 currentReplicas int32
4025 prenormalizedDesiredReplicas int32
4026
4027 expectedReplicas int32
4028 expectedCondition string
4029
4030 testThis bool
4031 }
4032
4033 tests := []TestCase{
4034 {
4035 currentReplicas: 5,
4036 prenormalizedDesiredReplicas: 7,
4037 specMinReplicas: 3,
4038 specMaxReplicas: 8,
4039 expectedReplicas: 7,
4040 expectedCondition: "DesiredWithinRange",
4041 name: "prenormalized desired replicas within range",
4042 },
4043 {
4044 currentReplicas: 3,
4045 prenormalizedDesiredReplicas: 1,
4046 specMinReplicas: 2,
4047 specMaxReplicas: 8,
4048 expectedReplicas: 2,
4049 expectedCondition: "TooFewReplicas",
4050 name: "prenormalized desired replicas < minReplicas",
4051 },
4052 {
4053 currentReplicas: 1,
4054 prenormalizedDesiredReplicas: 0,
4055 specMinReplicas: 0,
4056 specMaxReplicas: 10,
4057 expectedReplicas: 0,
4058 expectedCondition: "DesiredWithinRange",
4059 name: "prenormalized desired replicas within range when minReplicas is 0",
4060 },
4061 {
4062 currentReplicas: 20,
4063 prenormalizedDesiredReplicas: 1000,
4064 specMinReplicas: 1,
4065 specMaxReplicas: 10,
4066 expectedReplicas: 10,
4067 expectedCondition: "TooManyReplicas",
4068 name: "maxReplicas is the limit because maxReplicas < scaleUpLimit",
4069 },
4070 {
4071 currentReplicas: 100,
4072 prenormalizedDesiredReplicas: 1000,
4073 specMinReplicas: 100,
4074 specMaxReplicas: 150,
4075 expectedReplicas: 150,
4076 expectedCondition: "TooManyReplicas",
4077 name: "desired replica count is more than the maximum replica count",
4078 },
4079 {
4080 currentReplicas: 3,
4081 prenormalizedDesiredReplicas: 1000,
4082 specMinReplicas: 1,
4083 specMaxReplicas: 2000,
4084 expectedReplicas: 4,
4085 expectedCondition: "ScaleUpLimit",
4086 scaleUpRules: generateScalingRules(0, 0, 1, 60, 0),
4087 name: "scaleUpLimit is the limit because scaleUpLimit < maxReplicas with user policies",
4088 },
4089 {
4090 currentReplicas: 1000,
4091 prenormalizedDesiredReplicas: 3,
4092 specMinReplicas: 3,
4093 specMaxReplicas: 2000,
4094 scaleDownRules: generateScalingRules(20, 60, 0, 0, 0),
4095 expectedReplicas: 980,
4096 expectedCondition: "ScaleDownLimit",
4097 name: "scaleDownLimit is the limit because scaleDownLimit > minReplicas with user defined policies",
4098 testThis: true,
4099 },
4100
4101 {
4102 name: "scaleUp with default behavior",
4103 specMinReplicas: 1,
4104 specMaxReplicas: 1000,
4105 currentReplicas: 10,
4106 prenormalizedDesiredReplicas: 50,
4107 expectedReplicas: 20,
4108 expectedCondition: "ScaleUpLimit",
4109 },
4110 {
4111 name: "scaleUp with pods policy larger than percent policy",
4112 specMinReplicas: 1,
4113 specMaxReplicas: 1000,
4114 scaleUpRules: generateScalingRules(100, 60, 100, 60, 0),
4115 currentReplicas: 10,
4116 prenormalizedDesiredReplicas: 500,
4117 expectedReplicas: 110,
4118 expectedCondition: "ScaleUpLimit",
4119 },
4120 {
4121 name: "scaleUp with percent policy larger than pods policy",
4122 specMinReplicas: 1,
4123 specMaxReplicas: 1000,
4124 scaleUpRules: generateScalingRules(2, 60, 100, 60, 0),
4125 currentReplicas: 10,
4126 prenormalizedDesiredReplicas: 500,
4127 expectedReplicas: 20,
4128 expectedCondition: "ScaleUpLimit",
4129 },
4130 {
4131 name: "scaleUp with spec MaxReplicas limitation with large pod policy",
4132 specMinReplicas: 1,
4133 specMaxReplicas: 1000,
4134 scaleUpRules: generateScalingRules(100, 60, 0, 0, 0),
4135 currentReplicas: 10,
4136 prenormalizedDesiredReplicas: 50,
4137 expectedReplicas: 50,
4138 expectedCondition: "DesiredWithinRange",
4139 },
4140 {
4141 name: "scaleUp with spec MaxReplicas limitation with large percent policy",
4142 specMinReplicas: 1,
4143 specMaxReplicas: 1000,
4144 scaleUpRules: generateScalingRules(10000, 60, 0, 0, 0),
4145 currentReplicas: 10,
4146 prenormalizedDesiredReplicas: 50,
4147 expectedReplicas: 50,
4148 expectedCondition: "DesiredWithinRange",
4149 },
4150 {
4151 name: "scaleUp with pod policy limitation",
4152 specMinReplicas: 1,
4153 specMaxReplicas: 1000,
4154 scaleUpRules: generateScalingRules(30, 60, 0, 0, 0),
4155 currentReplicas: 10,
4156 prenormalizedDesiredReplicas: 50,
4157 expectedReplicas: 40,
4158 expectedCondition: "ScaleUpLimit",
4159 },
4160 {
4161 name: "scaleUp with percent policy limitation",
4162 specMinReplicas: 1,
4163 specMaxReplicas: 1000,
4164 scaleUpRules: generateScalingRules(0, 0, 200, 60, 0),
4165 currentReplicas: 10,
4166 prenormalizedDesiredReplicas: 50,
4167 expectedReplicas: 30,
4168 expectedCondition: "ScaleUpLimit",
4169 },
4170 {
4171 name: "scaleDown with percent policy larger than pod policy",
4172 specMinReplicas: 1,
4173 specMaxReplicas: 1000,
4174 scaleDownRules: generateScalingRules(20, 60, 1, 60, 300),
4175 currentReplicas: 100,
4176 prenormalizedDesiredReplicas: 2,
4177 expectedReplicas: 80,
4178 expectedCondition: "ScaleDownLimit",
4179 },
4180 {
4181 name: "scaleDown with pod policy larger than percent policy",
4182 specMinReplicas: 1,
4183 specMaxReplicas: 1000,
4184 scaleDownRules: generateScalingRules(2, 60, 1, 60, 300),
4185 currentReplicas: 100,
4186 prenormalizedDesiredReplicas: 2,
4187 expectedReplicas: 98,
4188 expectedCondition: "ScaleDownLimit",
4189 },
4190 {
4191 name: "scaleDown with spec MinReplicas=nil limitation with large pod policy",
4192 specMinReplicas: 1,
4193 specMaxReplicas: 1000,
4194 scaleDownRules: generateScalingRules(100, 60, 0, 0, 300),
4195 currentReplicas: 10,
4196 prenormalizedDesiredReplicas: 0,
4197 expectedReplicas: 1,
4198 expectedCondition: "TooFewReplicas",
4199 },
4200 {
4201 name: "scaleDown with spec MinReplicas limitation with large pod policy",
4202 specMinReplicas: 1,
4203 specMaxReplicas: 1000,
4204 scaleDownRules: generateScalingRules(100, 60, 0, 0, 300),
4205 currentReplicas: 10,
4206 prenormalizedDesiredReplicas: 0,
4207 expectedReplicas: 1,
4208 expectedCondition: "TooFewReplicas",
4209 },
4210 {
4211 name: "scaleDown with spec MinReplicas limitation with large percent policy",
4212 specMinReplicas: 5,
4213 specMaxReplicas: 1000,
4214 scaleDownRules: generateScalingRules(0, 0, 100, 60, 300),
4215 currentReplicas: 10,
4216 prenormalizedDesiredReplicas: 2,
4217 expectedReplicas: 5,
4218 expectedCondition: "TooFewReplicas",
4219 },
4220 {
4221 name: "scaleDown with pod policy limitation",
4222 specMinReplicas: 1,
4223 specMaxReplicas: 1000,
4224 scaleDownRules: generateScalingRules(5, 60, 0, 0, 300),
4225 currentReplicas: 10,
4226 prenormalizedDesiredReplicas: 2,
4227 expectedReplicas: 5,
4228 expectedCondition: "ScaleDownLimit",
4229 },
4230 {
4231 name: "scaleDown with percent policy limitation",
4232 specMinReplicas: 1,
4233 specMaxReplicas: 1000,
4234 scaleDownRules: generateScalingRules(0, 0, 50, 60, 300),
4235 currentReplicas: 10,
4236 prenormalizedDesiredReplicas: 5,
4237 expectedReplicas: 5,
4238 expectedCondition: "DesiredWithinRange",
4239 },
4240 {
4241 name: "scaleUp with spec MaxReplicas limitation with large pod policy and events",
4242 scaleUpEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4243 specMinReplicas: 1,
4244 specMaxReplicas: 200,
4245 scaleUpRules: generateScalingRules(300, 60, 0, 0, 0),
4246 currentReplicas: 100,
4247 prenormalizedDesiredReplicas: 500,
4248 expectedReplicas: 200,
4249 expectedCondition: "TooManyReplicas",
4250 },
4251 {
4252 name: "scaleUp with spec MaxReplicas limitation with large percent policy and events",
4253 scaleUpEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4254 specMinReplicas: 1,
4255 specMaxReplicas: 200,
4256 scaleUpRules: generateScalingRules(0, 0, 10000, 60, 0),
4257 currentReplicas: 100,
4258 prenormalizedDesiredReplicas: 500,
4259 expectedReplicas: 200,
4260 expectedCondition: "TooManyReplicas",
4261 },
4262 {
4263
4264
4265 name: "scaleUp with currentReplicas limitation with rate.PeriodSeconds with a lot of recent scale up events",
4266 scaleUpEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4267 specMinReplicas: 1,
4268 specMaxReplicas: 1000,
4269 scaleUpRules: generateScalingRules(5, 120, 0, 0, 0),
4270 currentReplicas: 100,
4271 prenormalizedDesiredReplicas: 500,
4272 expectedReplicas: 100,
4273 expectedCondition: "ScaleUpLimit",
4274 },
4275 {
4276 name: "scaleUp with pod policy and previous scale up events",
4277 scaleUpEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4278 specMinReplicas: 1,
4279 specMaxReplicas: 1000,
4280 scaleUpRules: generateScalingRules(150, 120, 0, 0, 0),
4281 currentReplicas: 100,
4282 prenormalizedDesiredReplicas: 500,
4283 expectedReplicas: 235,
4284 expectedCondition: "ScaleUpLimit",
4285 },
4286 {
4287 name: "scaleUp with percent policy and previous scale up events",
4288 scaleUpEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4289 specMinReplicas: 1,
4290 specMaxReplicas: 1000,
4291 scaleUpRules: generateScalingRules(0, 0, 200, 120, 0),
4292 currentReplicas: 100,
4293 prenormalizedDesiredReplicas: 500,
4294 expectedReplicas: 255,
4295 expectedCondition: "ScaleUpLimit",
4296 },
4297 {
4298 name: "scaleUp with percent policy and previous scale up and down events",
4299 scaleUpEvents: generateEventsUniformDistribution([]int{4}, 120),
4300 scaleDownEvents: generateEventsUniformDistribution([]int{2}, 120),
4301 specMinReplicas: 1,
4302 specMaxReplicas: 1000,
4303 scaleUpRules: generateScalingRules(0, 0, 300, 300, 0),
4304 currentReplicas: 6,
4305 prenormalizedDesiredReplicas: 24,
4306 expectedReplicas: 16,
4307 expectedCondition: "ScaleUpLimit",
4308 },
4309
4310 {
4311 name: "scaleDown with default policy and previous events",
4312 scaleDownEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4313 specMinReplicas: 1,
4314 specMaxReplicas: 1000,
4315 currentReplicas: 10,
4316 prenormalizedDesiredReplicas: 5,
4317 expectedReplicas: 5,
4318 expectedCondition: "DesiredWithinRange",
4319 },
4320 {
4321 name: "scaleDown with spec MinReplicas=nil limitation with large pod policy and previous events",
4322 scaleDownEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4323 specMinReplicas: 1,
4324 specMaxReplicas: 1000,
4325 scaleDownRules: generateScalingRules(115, 120, 0, 0, 300),
4326 currentReplicas: 100,
4327 prenormalizedDesiredReplicas: 0,
4328 expectedReplicas: 1,
4329 expectedCondition: "TooFewReplicas",
4330 },
4331 {
4332 name: "scaleDown with spec MinReplicas limitation with large pod policy and previous events",
4333 scaleDownEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4334 specMinReplicas: 5,
4335 specMaxReplicas: 1000,
4336 scaleDownRules: generateScalingRules(130, 120, 0, 0, 300),
4337 currentReplicas: 100,
4338 prenormalizedDesiredReplicas: 0,
4339 expectedReplicas: 5,
4340 expectedCondition: "TooFewReplicas",
4341 },
4342 {
4343 name: "scaleDown with spec MinReplicas limitation with large percent policy and previous events",
4344 scaleDownEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4345 specMinReplicas: 5,
4346 specMaxReplicas: 1000,
4347 scaleDownRules: generateScalingRules(0, 0, 100, 120, 300),
4348 currentReplicas: 100,
4349 prenormalizedDesiredReplicas: 2,
4350 expectedReplicas: 5,
4351 expectedCondition: "TooFewReplicas",
4352 },
4353 {
4354 name: "scaleDown with pod policy limitation and previous events",
4355 scaleDownEvents: generateEventsUniformDistribution([]int{1, 5, 9}, 120),
4356 specMinReplicas: 1,
4357 specMaxReplicas: 1000,
4358 scaleDownRules: generateScalingRules(5, 120, 0, 0, 300),
4359 currentReplicas: 100,
4360 prenormalizedDesiredReplicas: 2,
4361 expectedReplicas: 100,
4362 expectedCondition: "ScaleDownLimit",
4363 },
4364 {
4365 name: "scaleDown with percent policy limitation and previous events",
4366 scaleDownEvents: generateEventsUniformDistribution([]int{2, 4, 6}, 120),
4367 specMinReplicas: 1,
4368 specMaxReplicas: 1000,
4369 scaleDownRules: generateScalingRules(0, 0, 50, 120, 300),
4370 currentReplicas: 100,
4371 prenormalizedDesiredReplicas: 0,
4372 expectedReplicas: 56,
4373 expectedCondition: "ScaleDownLimit",
4374 },
4375 {
4376 name: "scaleDown with percent policy and previous scale up and down events",
4377 scaleUpEvents: generateEventsUniformDistribution([]int{2}, 120),
4378 scaleDownEvents: generateEventsUniformDistribution([]int{4}, 120),
4379 specMinReplicas: 1,
4380 specMaxReplicas: 1000,
4381 scaleDownRules: generateScalingRules(0, 0, 50, 180, 0),
4382 currentReplicas: 10,
4383 prenormalizedDesiredReplicas: 1,
4384 expectedReplicas: 6,
4385 expectedCondition: "ScaleDownLimit",
4386 },
4387 {
4388
4389
4390 name: "scaleDown with previous events preventing further scale down",
4391 scaleDownEvents: generateEventsUniformDistribution([]int{10, 10, 10}, 120),
4392 specMinReplicas: 1,
4393 specMaxReplicas: 1000,
4394 scaleDownRules: generateScalingRules(0, 0, 10, 120, 300),
4395 currentReplicas: 100,
4396 prenormalizedDesiredReplicas: 0,
4397 expectedReplicas: 100,
4398 expectedCondition: "ScaleDownLimit",
4399 },
4400 {
4401
4402 name: "scaleDown with with previous events still allowing more scale down",
4403 scaleDownEvents: generateEventsUniformDistribution([]int{10, 10, 10}, 120),
4404 specMinReplicas: 1,
4405 specMaxReplicas: 1000,
4406 scaleDownRules: generateScalingRules(0, 0, 1000, 120, 300),
4407 currentReplicas: 10,
4408 prenormalizedDesiredReplicas: 5,
4409 expectedReplicas: 5,
4410 expectedCondition: "DesiredWithinRange",
4411 },
4412 {
4413 name: "check 'outdated' flag for events for one behavior for up",
4414 scaleUpEvents: generateEventsUniformDistribution([]int{8, 12, 9, 11}, 120),
4415 specMinReplicas: 1,
4416 specMaxReplicas: 1000,
4417 scaleUpRules: generateScalingRules(1000, 60, 0, 0, 0),
4418 currentReplicas: 100,
4419 prenormalizedDesiredReplicas: 200,
4420 expectedReplicas: 200,
4421 expectedCondition: "DesiredWithinRange",
4422 },
4423 {
4424 name: "check that events were not marked 'outdated' for two different policies in the behavior for up",
4425 scaleUpEvents: generateEventsUniformDistribution([]int{8, 12, 9, 11}, 120),
4426 specMinReplicas: 1,
4427 specMaxReplicas: 1000,
4428 scaleUpRules: generateScalingRules(1000, 120, 100, 60, 0),
4429 currentReplicas: 100,
4430 prenormalizedDesiredReplicas: 200,
4431 expectedReplicas: 200,
4432 expectedCondition: "DesiredWithinRange",
4433 },
4434 {
4435 name: "check that events were marked 'outdated' for two different policies in the behavior for up",
4436 scaleUpEvents: generateEventsUniformDistribution([]int{8, 12, 9, 11}, 120),
4437 specMinReplicas: 1,
4438 specMaxReplicas: 1000,
4439 scaleUpRules: generateScalingRules(1000, 30, 100, 60, 0),
4440 currentReplicas: 100,
4441 prenormalizedDesiredReplicas: 200,
4442 expectedReplicas: 200,
4443 expectedCondition: "DesiredWithinRange",
4444 },
4445 {
4446 name: "check 'outdated' flag for events for one behavior for down",
4447 scaleDownEvents: generateEventsUniformDistribution([]int{8, 12, 9, 11}, 120),
4448 specMinReplicas: 1,
4449 specMaxReplicas: 1000,
4450 scaleDownRules: generateScalingRules(1000, 60, 0, 0, 300),
4451 currentReplicas: 100,
4452 prenormalizedDesiredReplicas: 5,
4453 expectedReplicas: 5,
4454 expectedCondition: "DesiredWithinRange",
4455 },
4456 {
4457 name: "check that events were not marked 'outdated' for two different policies in the behavior for down",
4458 scaleDownEvents: generateEventsUniformDistribution([]int{8, 12, 9, 11}, 120),
4459 specMinReplicas: 1,
4460 specMaxReplicas: 1000,
4461 scaleDownRules: generateScalingRules(1000, 120, 100, 60, 300),
4462 currentReplicas: 100,
4463 prenormalizedDesiredReplicas: 5,
4464 expectedReplicas: 5,
4465 expectedCondition: "DesiredWithinRange",
4466 },
4467 {
4468 name: "check that events were marked 'outdated' for two different policies in the behavior for down",
4469 scaleDownEvents: generateEventsUniformDistribution([]int{8, 12, 9, 11}, 120),
4470 specMinReplicas: 1,
4471 specMaxReplicas: 1000,
4472 scaleDownRules: generateScalingRules(1000, 30, 100, 60, 300),
4473 currentReplicas: 100,
4474 prenormalizedDesiredReplicas: 5,
4475 expectedReplicas: 5,
4476 expectedCondition: "DesiredWithinRange",
4477 },
4478 }
4479
4480 for _, tc := range tests {
4481 t.Run(tc.name, func(t *testing.T) {
4482
4483 if tc.testThis {
4484 return
4485 }
4486 hc := HorizontalController{
4487 scaleUpEvents: map[string][]timestampedScaleEvent{
4488 tc.key: tc.scaleUpEvents,
4489 },
4490 scaleDownEvents: map[string][]timestampedScaleEvent{
4491 tc.key: tc.scaleDownEvents,
4492 },
4493 }
4494 arg := NormalizationArg{
4495 Key: tc.key,
4496 ScaleUpBehavior: autoscalingapiv2.GenerateHPAScaleUpRules(tc.scaleUpRules),
4497 ScaleDownBehavior: autoscalingapiv2.GenerateHPAScaleDownRules(tc.scaleDownRules),
4498 MinReplicas: tc.specMinReplicas,
4499 MaxReplicas: tc.specMaxReplicas,
4500 DesiredReplicas: tc.prenormalizedDesiredReplicas,
4501 CurrentReplicas: tc.currentReplicas,
4502 }
4503
4504 replicas, condition, _ := hc.convertDesiredReplicasWithBehaviorRate(arg)
4505 assert.Equal(t, tc.expectedReplicas, replicas, "expected replicas do not match with converted replicas")
4506 assert.Equal(t, tc.expectedCondition, condition, "HPA condition does not match with expected condition")
4507 })
4508 }
4509
4510 }
4511
4512
4513 func TestStoreScaleEvents(t *testing.T) {
4514 type TestCase struct {
4515 name string
4516 key string
4517 replicaChange int32
4518 prevScaleEvents []timestampedScaleEvent
4519 newScaleEvents []timestampedScaleEvent
4520 scalingRules *autoscalingv2.HPAScalingRules
4521 expectedReplicasChange int32
4522 }
4523 tests := []TestCase{
4524 {
4525 name: "empty entries with default behavior",
4526 replicaChange: 5,
4527 prevScaleEvents: []timestampedScaleEvent{},
4528 newScaleEvents: []timestampedScaleEvent{},
4529 expectedReplicasChange: 0,
4530 },
4531 {
4532 name: "empty entries with two-policy-behavior",
4533 replicaChange: 5,
4534 prevScaleEvents: []timestampedScaleEvent{},
4535 newScaleEvents: []timestampedScaleEvent{{5, time.Now(), false}},
4536 scalingRules: generateScalingRules(10, 60, 100, 60, 0),
4537 expectedReplicasChange: 0,
4538 },
4539 {
4540 name: "one outdated entry to be kept untouched without behavior",
4541 replicaChange: 5,
4542 prevScaleEvents: []timestampedScaleEvent{
4543 {7, time.Now().Add(-time.Second * time.Duration(61)), false},
4544 },
4545 newScaleEvents: []timestampedScaleEvent{
4546 {7, time.Now(), false},
4547 },
4548 expectedReplicasChange: 0,
4549 },
4550 {
4551 name: "one outdated entry to be replaced with behavior",
4552 replicaChange: 5,
4553 prevScaleEvents: []timestampedScaleEvent{
4554 {7, time.Now().Add(-time.Second * time.Duration(61)), false},
4555 },
4556 newScaleEvents: []timestampedScaleEvent{
4557 {5, time.Now(), false},
4558 },
4559 scalingRules: generateScalingRules(10, 60, 100, 60, 0),
4560 expectedReplicasChange: 0,
4561 },
4562 {
4563 name: "one actual entry to be not touched with behavior",
4564 replicaChange: 5,
4565 prevScaleEvents: []timestampedScaleEvent{
4566 {7, time.Now().Add(-time.Second * time.Duration(58)), false},
4567 },
4568 newScaleEvents: []timestampedScaleEvent{
4569 {7, time.Now(), false},
4570 {5, time.Now(), false},
4571 },
4572 scalingRules: generateScalingRules(10, 60, 100, 60, 0),
4573 expectedReplicasChange: 7,
4574 },
4575 {
4576 name: "two entries, one of them to be replaced",
4577 replicaChange: 5,
4578 prevScaleEvents: []timestampedScaleEvent{
4579 {7, time.Now().Add(-time.Second * time.Duration(61)), false},
4580 {6, time.Now().Add(-time.Second * time.Duration(59)), false},
4581 },
4582 newScaleEvents: []timestampedScaleEvent{
4583 {5, time.Now(), false},
4584 {6, time.Now(), false},
4585 },
4586 scalingRules: generateScalingRules(10, 60, 0, 0, 0),
4587 expectedReplicasChange: 6,
4588 },
4589 {
4590 name: "replace one entry, use policies with different periods",
4591 replicaChange: 5,
4592 prevScaleEvents: []timestampedScaleEvent{
4593 {8, time.Now().Add(-time.Second * time.Duration(29)), false},
4594 {6, time.Now().Add(-time.Second * time.Duration(59)), false},
4595 {7, time.Now().Add(-time.Second * time.Duration(61)), false},
4596 {9, time.Now().Add(-time.Second * time.Duration(61)), false},
4597 },
4598 newScaleEvents: []timestampedScaleEvent{
4599 {8, time.Now(), false},
4600 {6, time.Now(), false},
4601 {7, time.Now(), true},
4602 {5, time.Now(), false},
4603 },
4604 scalingRules: generateScalingRules(10, 60, 100, 30, 0),
4605 expectedReplicasChange: 14,
4606 },
4607 {
4608 name: "two entries, both actual",
4609 replicaChange: 5,
4610 prevScaleEvents: []timestampedScaleEvent{
4611 {7, time.Now().Add(-time.Second * time.Duration(58)), false},
4612 {6, time.Now().Add(-time.Second * time.Duration(59)), false},
4613 },
4614 newScaleEvents: []timestampedScaleEvent{
4615 {7, time.Now(), false},
4616 {6, time.Now(), false},
4617 {5, time.Now(), false},
4618 },
4619 scalingRules: generateScalingRules(10, 120, 100, 30, 0),
4620 expectedReplicasChange: 13,
4621 },
4622 }
4623
4624 for _, tc := range tests {
4625 t.Run(tc.name, func(t *testing.T) {
4626
4627 var behaviorUp *autoscalingv2.HorizontalPodAutoscalerBehavior
4628 if tc.scalingRules != nil {
4629 behaviorUp = &autoscalingv2.HorizontalPodAutoscalerBehavior{
4630 ScaleUp: tc.scalingRules,
4631 }
4632 }
4633 hcUp := HorizontalController{
4634 scaleUpEvents: map[string][]timestampedScaleEvent{
4635 tc.key: append([]timestampedScaleEvent{}, tc.prevScaleEvents...),
4636 },
4637 }
4638 gotReplicasChangeUp := getReplicasChangePerPeriod(60, hcUp.scaleUpEvents[tc.key])
4639 assert.Equal(t, tc.expectedReplicasChange, gotReplicasChangeUp)
4640 hcUp.storeScaleEvent(behaviorUp, tc.key, 10, 10+tc.replicaChange)
4641 if !assert.Len(t, hcUp.scaleUpEvents[tc.key], len(tc.newScaleEvents), "up: scale events differ in length") {
4642 return
4643 }
4644 for i, gotEvent := range hcUp.scaleUpEvents[tc.key] {
4645 expEvent := tc.newScaleEvents[i]
4646 assert.Equal(t, expEvent.replicaChange, gotEvent.replicaChange, "up: idx:%v replicaChange", i)
4647 assert.Equal(t, expEvent.outdated, gotEvent.outdated, "up: idx:%v outdated", i)
4648 }
4649
4650 var behaviorDown *autoscalingv2.HorizontalPodAutoscalerBehavior
4651 if tc.scalingRules != nil {
4652 behaviorDown = &autoscalingv2.HorizontalPodAutoscalerBehavior{
4653 ScaleDown: tc.scalingRules,
4654 }
4655 }
4656 hcDown := HorizontalController{
4657 scaleDownEvents: map[string][]timestampedScaleEvent{
4658 tc.key: append([]timestampedScaleEvent{}, tc.prevScaleEvents...),
4659 },
4660 }
4661 gotReplicasChangeDown := getReplicasChangePerPeriod(60, hcDown.scaleDownEvents[tc.key])
4662 assert.Equal(t, tc.expectedReplicasChange, gotReplicasChangeDown)
4663 hcDown.storeScaleEvent(behaviorDown, tc.key, 10, 10-tc.replicaChange)
4664 if !assert.Len(t, hcDown.scaleDownEvents[tc.key], len(tc.newScaleEvents), "down: scale events differ in length") {
4665 return
4666 }
4667 for i, gotEvent := range hcDown.scaleDownEvents[tc.key] {
4668 expEvent := tc.newScaleEvents[i]
4669 assert.Equal(t, expEvent.replicaChange, gotEvent.replicaChange, "down: idx:%v replicaChange", i)
4670 assert.Equal(t, expEvent.outdated, gotEvent.outdated, "down: idx:%v outdated", i)
4671 }
4672 })
4673 }
4674 }
4675
4676 func TestNormalizeDesiredReplicasWithBehavior(t *testing.T) {
4677 now := time.Now()
4678 type TestCase struct {
4679 name string
4680 key string
4681 recommendations []timestampedRecommendation
4682 currentReplicas int32
4683 prenormalizedDesiredReplicas int32
4684 expectedStabilizedReplicas int32
4685 expectedRecommendations []timestampedRecommendation
4686 scaleUpStabilizationWindowSeconds int32
4687 scaleDownStabilizationWindowSeconds int32
4688 }
4689 tests := []TestCase{
4690 {
4691 name: "empty recommendations for scaling down",
4692 key: "",
4693 recommendations: []timestampedRecommendation{},
4694 currentReplicas: 100,
4695 prenormalizedDesiredReplicas: 5,
4696 expectedStabilizedReplicas: 5,
4697 expectedRecommendations: []timestampedRecommendation{
4698 {5, now},
4699 },
4700 },
4701 {
4702 name: "simple scale down stabilization",
4703 key: "",
4704 recommendations: []timestampedRecommendation{
4705 {4, now.Add(-2 * time.Minute)},
4706 {5, now.Add(-1 * time.Minute)}},
4707 currentReplicas: 100,
4708 prenormalizedDesiredReplicas: 3,
4709 expectedStabilizedReplicas: 5,
4710 expectedRecommendations: []timestampedRecommendation{
4711 {4, now},
4712 {5, now},
4713 {3, now},
4714 },
4715 scaleDownStabilizationWindowSeconds: 60 * 3,
4716 },
4717 {
4718 name: "simple scale up stabilization",
4719 key: "",
4720 recommendations: []timestampedRecommendation{
4721 {4, now.Add(-2 * time.Minute)},
4722 {5, now.Add(-1 * time.Minute)}},
4723 currentReplicas: 1,
4724 prenormalizedDesiredReplicas: 7,
4725 expectedStabilizedReplicas: 4,
4726 expectedRecommendations: []timestampedRecommendation{
4727 {4, now},
4728 {5, now},
4729 {7, now},
4730 },
4731 scaleUpStabilizationWindowSeconds: 60 * 5,
4732 },
4733 {
4734 name: "no scale down stabilization",
4735 key: "",
4736 recommendations: []timestampedRecommendation{
4737 {1, now.Add(-2 * time.Minute)},
4738 {2, now.Add(-1 * time.Minute)}},
4739 currentReplicas: 100,
4740 prenormalizedDesiredReplicas: 3,
4741 expectedStabilizedReplicas: 3,
4742 expectedRecommendations: []timestampedRecommendation{
4743 {1, now},
4744 {2, now},
4745 {3, now},
4746 },
4747 scaleUpStabilizationWindowSeconds: 60 * 5,
4748 },
4749 {
4750 name: "no scale up stabilization",
4751 key: "",
4752 recommendations: []timestampedRecommendation{
4753 {4, now.Add(-2 * time.Minute)},
4754 {5, now.Add(-1 * time.Minute)}},
4755 currentReplicas: 1,
4756 prenormalizedDesiredReplicas: 3,
4757 expectedStabilizedReplicas: 3,
4758 expectedRecommendations: []timestampedRecommendation{
4759 {4, now},
4760 {5, now},
4761 {3, now},
4762 },
4763 scaleDownStabilizationWindowSeconds: 60 * 5,
4764 },
4765 {
4766 name: "no scale down stabilization, reuse recommendation element",
4767 key: "",
4768 recommendations: []timestampedRecommendation{
4769 {10, now.Add(-10 * time.Minute)},
4770 {9, now.Add(-9 * time.Minute)}},
4771 currentReplicas: 100,
4772 prenormalizedDesiredReplicas: 3,
4773 expectedStabilizedReplicas: 3,
4774 expectedRecommendations: []timestampedRecommendation{
4775 {10, now},
4776 {3, now},
4777 },
4778 },
4779 {
4780 name: "no scale up stabilization, reuse recommendation element",
4781 key: "",
4782 recommendations: []timestampedRecommendation{
4783 {10, now.Add(-10 * time.Minute)},
4784 {9, now.Add(-9 * time.Minute)}},
4785 currentReplicas: 1,
4786 prenormalizedDesiredReplicas: 100,
4787 expectedStabilizedReplicas: 100,
4788 expectedRecommendations: []timestampedRecommendation{
4789 {10, now},
4790 {100, now},
4791 },
4792 },
4793 {
4794 name: "scale down stabilization, reuse one of obsolete recommendation element",
4795 key: "",
4796 recommendations: []timestampedRecommendation{
4797 {10, now.Add(-10 * time.Minute)},
4798 {4, now.Add(-1 * time.Minute)},
4799 {5, now.Add(-2 * time.Minute)},
4800 {9, now.Add(-9 * time.Minute)}},
4801 currentReplicas: 100,
4802 prenormalizedDesiredReplicas: 3,
4803 expectedStabilizedReplicas: 5,
4804 expectedRecommendations: []timestampedRecommendation{
4805 {10, now},
4806 {4, now},
4807 {5, now},
4808 {3, now},
4809 },
4810 scaleDownStabilizationWindowSeconds: 3 * 60,
4811 },
4812 {
4813
4814
4815
4816 name: "scale up stabilization, reuse one of obsolete recommendation element",
4817 key: "",
4818 recommendations: []timestampedRecommendation{
4819 {10, now.Add(-100 * time.Minute)},
4820 {6, now.Add(-1 * time.Minute)},
4821 {5, now.Add(-2 * time.Minute)},
4822 {9, now.Add(-3 * time.Minute)}},
4823 currentReplicas: 1,
4824 prenormalizedDesiredReplicas: 100,
4825 expectedStabilizedReplicas: 5,
4826 expectedRecommendations: []timestampedRecommendation{
4827 {100, now},
4828 {6, now},
4829 {5, now},
4830 {9, now},
4831 },
4832 scaleUpStabilizationWindowSeconds: 300,
4833 }, {
4834 name: "scale up and down stabilization, do not scale up when prenormalized rec goes down",
4835 key: "",
4836 recommendations: []timestampedRecommendation{
4837 {2, now.Add(-100 * time.Minute)},
4838 {3, now.Add(-3 * time.Minute)},
4839 },
4840 currentReplicas: 2,
4841 prenormalizedDesiredReplicas: 1,
4842 expectedStabilizedReplicas: 2,
4843 scaleUpStabilizationWindowSeconds: 300,
4844 scaleDownStabilizationWindowSeconds: 300,
4845 }, {
4846 name: "scale up and down stabilization, do not scale down when prenormalized rec goes up",
4847 key: "",
4848 recommendations: []timestampedRecommendation{
4849 {2, now.Add(-100 * time.Minute)},
4850 {1, now.Add(-3 * time.Minute)},
4851 },
4852 currentReplicas: 2,
4853 prenormalizedDesiredReplicas: 3,
4854 expectedStabilizedReplicas: 2,
4855 scaleUpStabilizationWindowSeconds: 300,
4856 scaleDownStabilizationWindowSeconds: 300,
4857 },
4858 }
4859 for _, tc := range tests {
4860 t.Run(tc.name, func(t *testing.T) {
4861 hc := HorizontalController{
4862 recommendations: map[string][]timestampedRecommendation{
4863 tc.key: tc.recommendations,
4864 },
4865 }
4866 arg := NormalizationArg{
4867 Key: tc.key,
4868 DesiredReplicas: tc.prenormalizedDesiredReplicas,
4869 CurrentReplicas: tc.currentReplicas,
4870 ScaleUpBehavior: &autoscalingv2.HPAScalingRules{
4871 StabilizationWindowSeconds: &tc.scaleUpStabilizationWindowSeconds,
4872 },
4873 ScaleDownBehavior: &autoscalingv2.HPAScalingRules{
4874 StabilizationWindowSeconds: &tc.scaleDownStabilizationWindowSeconds,
4875 },
4876 }
4877 r, _, _ := hc.stabilizeRecommendationWithBehaviors(arg)
4878 assert.Equal(t, tc.expectedStabilizedReplicas, r, "expected replicas do not match")
4879 if tc.expectedRecommendations != nil {
4880 if !assert.Len(t, hc.recommendations[tc.key], len(tc.expectedRecommendations), "stored recommendations differ in length") {
4881 return
4882 }
4883 for i, r := range hc.recommendations[tc.key] {
4884 expectedRecommendation := tc.expectedRecommendations[i]
4885 assert.Equal(t, expectedRecommendation.recommendation, r.recommendation, "stored recommendation differs at position %d", i)
4886 }
4887 }
4888 })
4889 }
4890 }
4891
4892 func TestScaleUpOneMetricEmpty(t *testing.T) {
4893 tc := testCase{
4894 minReplicas: 2,
4895 maxReplicas: 6,
4896 specReplicas: 3,
4897 statusReplicas: 3,
4898 expectedDesiredReplicas: 4,
4899 CPUTarget: 30,
4900 verifyCPUCurrent: true,
4901 metricsTarget: []autoscalingv2.MetricSpec{
4902 {
4903 Type: autoscalingv2.ExternalMetricSourceType,
4904 External: &autoscalingv2.ExternalMetricSource{
4905 Metric: autoscalingv2.MetricIdentifier{
4906 Name: "qps",
4907 Selector: &metav1.LabelSelector{},
4908 },
4909 Target: autoscalingv2.MetricTarget{
4910 Type: autoscalingv2.ValueMetricType,
4911 Value: resource.NewMilliQuantity(100, resource.DecimalSI),
4912 },
4913 },
4914 },
4915 },
4916 reportedLevels: []uint64{300, 400, 500},
4917 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
4918 expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp,
4919 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
4920 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
4921 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp,
4922 autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelNone,
4923 },
4924 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
4925 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
4926 autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelInternal,
4927 },
4928 }
4929 _, _, _, testEMClient, _ := tc.prepareTestClient(t)
4930 testEMClient.PrependReactor("list", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
4931 return true, &emapi.ExternalMetricValueList{}, fmt.Errorf("something went wrong")
4932 })
4933 tc.testEMClient = testEMClient
4934 tc.runTest(t)
4935 }
4936
4937 func TestNoScaleDownOneMetricInvalid(t *testing.T) {
4938 tc := testCase{
4939 minReplicas: 2,
4940 maxReplicas: 6,
4941 specReplicas: 5,
4942 statusReplicas: 5,
4943 expectedDesiredReplicas: 5,
4944 CPUTarget: 50,
4945 metricsTarget: []autoscalingv2.MetricSpec{
4946 {
4947 Type: "CheddarCheese",
4948 },
4949 },
4950 reportedLevels: []uint64{100, 300, 500, 250, 250},
4951 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
4952 useMetricsAPI: true,
4953 recommendations: []timestampedRecommendation{},
4954 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
4955 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"},
4956 {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "InvalidMetricSourceType"},
4957 },
4958 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
4959 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
4960 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
4961 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
4962 "CheddarCheese": monitor.ActionLabelNone,
4963 },
4964 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
4965 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
4966 "CheddarCheese": monitor.ErrorLabelSpec,
4967 },
4968 }
4969
4970 tc.runTest(t)
4971 }
4972
4973 func TestNoScaleDownOneMetricEmpty(t *testing.T) {
4974 tc := testCase{
4975 minReplicas: 2,
4976 maxReplicas: 6,
4977 specReplicas: 5,
4978 statusReplicas: 5,
4979 expectedDesiredReplicas: 5,
4980 CPUTarget: 50,
4981 metricsTarget: []autoscalingv2.MetricSpec{
4982 {
4983 Type: autoscalingv2.ExternalMetricSourceType,
4984 External: &autoscalingv2.ExternalMetricSource{
4985 Metric: autoscalingv2.MetricIdentifier{
4986 Name: "qps",
4987 Selector: &metav1.LabelSelector{},
4988 },
4989 Target: autoscalingv2.MetricTarget{
4990 Type: autoscalingv2.ValueMetricType,
4991 Value: resource.NewMilliQuantity(1000, resource.DecimalSI),
4992 },
4993 },
4994 },
4995 },
4996 reportedLevels: []uint64{100, 300, 500, 250, 250},
4997 reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")},
4998 useMetricsAPI: true,
4999 recommendations: []timestampedRecommendation{},
5000 expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
5001 {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"},
5002 {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "FailedGetExternalMetric"},
5003 },
5004 expectedReportedReconciliationActionLabel: monitor.ActionLabelNone,
5005 expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal,
5006 expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{
5007 autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleDown,
5008 autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelNone,
5009 },
5010 expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{
5011 autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone,
5012 autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelInternal,
5013 },
5014 }
5015 _, _, _, testEMClient, _ := tc.prepareTestClient(t)
5016 testEMClient.PrependReactor("list", "*", func(action core.Action) (handled bool, ret runtime.Object, err error) {
5017 return true, &emapi.ExternalMetricValueList{}, fmt.Errorf("something went wrong")
5018 })
5019 tc.testEMClient = testEMClient
5020 tc.runTest(t)
5021 }
5022
5023 func TestMultipleHPAs(t *testing.T) {
5024 const hpaCount = 1000
5025 const testNamespace = "dummy-namespace"
5026
5027 processed := make(chan string, hpaCount)
5028
5029 testClient := &fake.Clientset{}
5030 testScaleClient := &scalefake.FakeScaleClient{}
5031 testMetricsClient := &metricsfake.Clientset{}
5032
5033 hpaList := [hpaCount]autoscalingv2.HorizontalPodAutoscaler{}
5034 scaleUpEventsMap := map[string][]timestampedScaleEvent{}
5035 scaleDownEventsMap := map[string][]timestampedScaleEvent{}
5036 scaleList := map[string]*autoscalingv1.Scale{}
5037 podList := map[string]*v1.Pod{}
5038
5039 var minReplicas int32 = 1
5040 var cpuTarget int32 = 10
5041
5042
5043 for i := 0; i < hpaCount; i++ {
5044 hpaName := fmt.Sprintf("dummy-hpa-%v", i)
5045 deploymentName := fmt.Sprintf("dummy-target-%v", i)
5046 labelSet := map[string]string{"name": deploymentName}
5047 selector := labels.SelectorFromSet(labelSet).String()
5048
5049
5050 h := autoscalingv2.HorizontalPodAutoscaler{
5051 ObjectMeta: metav1.ObjectMeta{
5052 Name: hpaName,
5053 Namespace: testNamespace,
5054 },
5055 Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
5056 ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{
5057 APIVersion: "apps/v1",
5058 Kind: "Deployment",
5059 Name: deploymentName,
5060 },
5061 MinReplicas: &minReplicas,
5062 MaxReplicas: 10,
5063 Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{
5064 ScaleUp: generateScalingRules(100, 60, 0, 0, 0),
5065 ScaleDown: generateScalingRules(2, 60, 1, 60, 300),
5066 },
5067 Metrics: []autoscalingv2.MetricSpec{
5068 {
5069 Type: autoscalingv2.ResourceMetricSourceType,
5070 Resource: &autoscalingv2.ResourceMetricSource{
5071 Name: v1.ResourceCPU,
5072 Target: autoscalingv2.MetricTarget{
5073 Type: autoscalingv2.UtilizationMetricType,
5074 AverageUtilization: &cpuTarget,
5075 },
5076 },
5077 },
5078 },
5079 },
5080 Status: autoscalingv2.HorizontalPodAutoscalerStatus{
5081 CurrentReplicas: 1,
5082 DesiredReplicas: 5,
5083 LastScaleTime: &metav1.Time{Time: time.Now()},
5084 },
5085 }
5086 hpaList[i] = h
5087
5088
5089 scaleList[deploymentName] = &autoscalingv1.Scale{
5090 ObjectMeta: metav1.ObjectMeta{
5091 Name: deploymentName,
5092 Namespace: testNamespace,
5093 },
5094 Spec: autoscalingv1.ScaleSpec{
5095 Replicas: 1,
5096 },
5097 Status: autoscalingv1.ScaleStatus{
5098 Replicas: 1,
5099 Selector: selector,
5100 },
5101 }
5102
5103
5104 cpuRequest := resource.MustParse("1.0")
5105 pod := v1.Pod{
5106 Status: v1.PodStatus{
5107 Phase: v1.PodRunning,
5108 Conditions: []v1.PodCondition{
5109 {
5110 Type: v1.PodReady,
5111 Status: v1.ConditionTrue,
5112 },
5113 },
5114 StartTime: &metav1.Time{Time: time.Now().Add(-10 * time.Minute)},
5115 },
5116 ObjectMeta: metav1.ObjectMeta{
5117 Name: fmt.Sprintf("%s-0", deploymentName),
5118 Namespace: testNamespace,
5119 Labels: labelSet,
5120 },
5121
5122 Spec: v1.PodSpec{
5123 Containers: []v1.Container{
5124 {
5125 Name: "container1",
5126 Resources: v1.ResourceRequirements{
5127 Requests: v1.ResourceList{
5128 v1.ResourceCPU: *resource.NewMilliQuantity(cpuRequest.MilliValue()/2, resource.DecimalSI),
5129 },
5130 },
5131 },
5132 {
5133 Name: "container2",
5134 Resources: v1.ResourceRequirements{
5135 Requests: v1.ResourceList{
5136 v1.ResourceCPU: *resource.NewMilliQuantity(cpuRequest.MilliValue()/2, resource.DecimalSI),
5137 },
5138 },
5139 },
5140 },
5141 },
5142 }
5143 podList[deploymentName] = &pod
5144
5145 scaleUpEventsMap[fmt.Sprintf("%s/%s", testNamespace, hpaName)] = generateEventsUniformDistribution([]int{8, 12, 9, 11}, 120)
5146 scaleDownEventsMap[fmt.Sprintf("%s/%s", testNamespace, hpaName)] = generateEventsUniformDistribution([]int{10, 10, 10}, 120)
5147 }
5148
5149 testMetricsClient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
5150 podNamePrefix := ""
5151 labelSet := map[string]string{}
5152
5153
5154 selector := action.(core.ListAction).GetListRestrictions().Labels
5155 parsedSelector := strings.Split(selector.String(), "=")
5156 if len(parsedSelector) > 1 {
5157 labelSet[parsedSelector[0]] = parsedSelector[1]
5158 podNamePrefix = parsedSelector[1]
5159 }
5160
5161 podMetric := metricsapi.PodMetrics{
5162 ObjectMeta: metav1.ObjectMeta{
5163 Name: fmt.Sprintf("%s-0", podNamePrefix),
5164 Namespace: testNamespace,
5165 Labels: labelSet,
5166 },
5167 Timestamp: metav1.Time{Time: time.Now()},
5168 Window: metav1.Duration{Duration: time.Minute},
5169 Containers: []metricsapi.ContainerMetrics{
5170 {
5171 Name: "container1",
5172 Usage: v1.ResourceList{
5173 v1.ResourceCPU: *resource.NewMilliQuantity(
5174 int64(200),
5175 resource.DecimalSI),
5176 v1.ResourceMemory: *resource.NewQuantity(
5177 int64(1024*1024/2),
5178 resource.BinarySI),
5179 },
5180 },
5181 {
5182 Name: "container2",
5183 Usage: v1.ResourceList{
5184 v1.ResourceCPU: *resource.NewMilliQuantity(
5185 int64(300),
5186 resource.DecimalSI),
5187 v1.ResourceMemory: *resource.NewQuantity(
5188 int64(1024*1024/2),
5189 resource.BinarySI),
5190 },
5191 },
5192 },
5193 }
5194 metrics := &metricsapi.PodMetricsList{}
5195 metrics.Items = append(metrics.Items, podMetric)
5196
5197 return true, metrics, nil
5198 })
5199
5200 metricsClient := metrics.NewRESTMetricsClient(
5201 testMetricsClient.MetricsV1beta1(),
5202 &cmfake.FakeCustomMetricsClient{},
5203 &emfake.FakeExternalMetricsClient{},
5204 )
5205
5206 testScaleClient.AddReactor("get", "deployments", func(action core.Action) (handled bool, ret runtime.Object, err error) {
5207 deploymentName := action.(core.GetAction).GetName()
5208 obj := scaleList[deploymentName]
5209 return true, obj, nil
5210 })
5211
5212 testClient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
5213 obj := &v1.PodList{}
5214
5215
5216 selector := action.(core.ListAction).GetListRestrictions().Labels
5217 parsedSelector := strings.Split(selector.String(), "=")
5218
5219
5220 if len(parsedSelector) > 1 {
5221 obj.Items = append(obj.Items, *podList[parsedSelector[1]])
5222 } else {
5223
5224 for _, p := range podList {
5225 obj.Items = append(obj.Items, *p)
5226 }
5227 }
5228
5229 return true, obj, nil
5230 })
5231
5232 testClient.AddReactor("list", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
5233 obj := &autoscalingv2.HorizontalPodAutoscalerList{
5234 Items: hpaList[:],
5235 }
5236 return true, obj, nil
5237 })
5238
5239 testClient.AddReactor("update", "horizontalpodautoscalers", func(action core.Action) (handled bool, ret runtime.Object, err error) {
5240 handled, obj, err := func() (handled bool, ret *autoscalingv2.HorizontalPodAutoscaler, err error) {
5241 obj := action.(core.UpdateAction).GetObject().(*autoscalingv2.HorizontalPodAutoscaler)
5242 assert.Equal(t, testNamespace, obj.Namespace, "the HPA namespace should be as expected")
5243
5244 return true, obj, nil
5245 }()
5246 processed <- obj.Name
5247
5248 return handled, obj, err
5249 })
5250
5251 informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
5252
5253 tCtx := ktesting.Init(t)
5254 hpaController := NewHorizontalController(
5255 tCtx,
5256 testClient.CoreV1(),
5257 testScaleClient,
5258 testClient.AutoscalingV2(),
5259 testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme),
5260 metricsClient,
5261 informerFactory.Autoscaling().V2().HorizontalPodAutoscalers(),
5262 informerFactory.Core().V1().Pods(),
5263 100*time.Millisecond,
5264 5*time.Minute,
5265 defaultTestingTolerance,
5266 defaultTestingCPUInitializationPeriod,
5267 defaultTestingDelayOfInitialReadinessStatus,
5268 )
5269 hpaController.scaleUpEvents = scaleUpEventsMap
5270 hpaController.scaleDownEvents = scaleDownEventsMap
5271
5272 informerFactory.Start(tCtx.Done())
5273 go hpaController.Run(tCtx, 5)
5274
5275 timeoutTime := time.After(15 * time.Second)
5276 timeout := false
5277 processedHPA := make(map[string]bool)
5278 for timeout == false && len(processedHPA) < hpaCount {
5279 select {
5280 case hpaName := <-processed:
5281 processedHPA[hpaName] = true
5282 case <-timeoutTime:
5283 timeout = true
5284 }
5285 }
5286
5287 assert.Equal(t, hpaCount, len(processedHPA), "Expected to process all HPAs")
5288 }
5289
View as plain text