horizontal_pod_autoscaling.go

Documentation: k8s.io/kubernetes/test/e2e/autoscaling

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package autoscaling
    18  
    19  import (
    20  	"context"
    21  	"time"
    22  
    23  	"github.com/onsi/ginkgo/v2"
    24  	"k8s.io/pod-security-admission/api"
    25  
    26  	autoscalingv2 "k8s.io/api/autoscaling/v2"
    27  	v1 "k8s.io/api/core/v1"
    28  	"k8s.io/apimachinery/pkg/runtime/schema"
    29  	"k8s.io/kubernetes/test/e2e/feature"
    30  	"k8s.io/kubernetes/test/e2e/framework"
    31  	e2eautoscaling "k8s.io/kubernetes/test/e2e/framework/autoscaling"
    32  )
    33  
    34  const (
    35  	titleUp                 = "Should scale from 1 pod to 3 pods and then from 3 pods to 5 pods"
    36  	titleDown               = "Should scale from 5 pods to 3 pods and then from 3 pods to 1 pod"
    37  	titleAverageUtilization = " using Average Utilization for aggregation"
    38  	titleAverageValue       = " using Average Value for aggregation"
    39  	valueMetricType         = autoscalingv2.AverageValueMetricType
    40  	utilizationMetricType   = autoscalingv2.UtilizationMetricType
    41  	cpuResource             = v1.ResourceCPU
    42  	memResource             = v1.ResourceMemory
    43  )
    44  
    45  // These tests don't seem to be running properly in parallel: issue: #20338.
    46  var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (scale resource: CPU)", func() {
    47  	f := framework.NewDefaultFramework("horizontal-pod-autoscaling")
    48  	f.NamespacePodSecurityLevel = api.LevelBaseline
    49  
    50  	f.Describe(framework.WithSerial(), framework.WithSlow(), "Deployment (Pod Resource)", func() {
    51  		ginkgo.It(titleUp+titleAverageUtilization, func(ctx context.Context) {
    52  			scaleUp(ctx, "test-deployment", e2eautoscaling.KindDeployment, cpuResource, utilizationMetricType, false, f)
    53  		})
    54  		ginkgo.It(titleDown+titleAverageUtilization, func(ctx context.Context) {
    55  			scaleDown(ctx, "test-deployment", e2eautoscaling.KindDeployment, cpuResource, utilizationMetricType, false, f)
    56  		})
    57  		ginkgo.It(titleUp+titleAverageValue, func(ctx context.Context) {
    58  			scaleUp(ctx, "test-deployment", e2eautoscaling.KindDeployment, cpuResource, valueMetricType, false, f)
    59  		})
    60  	})
    61  
    62  	f.Describe(framework.WithSerial(), framework.WithSlow(), "Deployment (Container Resource)", func() {
    63  		ginkgo.It(titleUp+titleAverageUtilization, func(ctx context.Context) {
    64  			scaleUpContainerResource(ctx, "test-deployment", e2eautoscaling.KindDeployment, cpuResource, utilizationMetricType, f)
    65  		})
    66  		ginkgo.It(titleUp+titleAverageValue, func(ctx context.Context) {
    67  			scaleUpContainerResource(ctx, "test-deployment", e2eautoscaling.KindDeployment, cpuResource, valueMetricType, f)
    68  		})
    69  	})
    70  
    71  	f.Describe(framework.WithSerial(), framework.WithSlow(), "ReplicaSet", func() {
    72  		ginkgo.It(titleUp, func(ctx context.Context) {
    73  			scaleUp(ctx, "rs", e2eautoscaling.KindReplicaSet, cpuResource, utilizationMetricType, false, f)
    74  		})
    75  		ginkgo.It(titleDown, func(ctx context.Context) {
    76  			scaleDown(ctx, "rs", e2eautoscaling.KindReplicaSet, cpuResource, utilizationMetricType, false, f)
    77  		})
    78  	})
    79  
    80  	// These tests take ~20 minutes each.
    81  	f.Describe(framework.WithSerial(), framework.WithSlow(), "ReplicationController", func() {
    82  		ginkgo.It(titleUp+" and verify decision stability", func(ctx context.Context) {
    83  			scaleUp(ctx, "rc", e2eautoscaling.KindRC, cpuResource, utilizationMetricType, true, f)
    84  		})
    85  		ginkgo.It(titleDown+" and verify decision stability", func(ctx context.Context) {
    86  			scaleDown(ctx, "rc", e2eautoscaling.KindRC, cpuResource, utilizationMetricType, true, f)
    87  		})
    88  	})
    89  
    90  	f.Describe("ReplicationController light", func() {
    91  		ginkgo.It("Should scale from 1 pod to 2 pods", func(ctx context.Context) {
    92  			st := &HPAScaleTest{
    93  				initPods:         1,
    94  				initCPUTotal:     150,
    95  				perPodCPURequest: 200,
    96  				targetValue:      50,
    97  				minPods:          1,
    98  				maxPods:          2,
    99  				firstScale:       2,
   100  				resourceType:     cpuResource,
   101  				metricTargetType: utilizationMetricType,
   102  			}
   103  			st.run(ctx, "rc-light", e2eautoscaling.KindRC, f)
   104  		})
   105  		f.It(f.WithSlow(), "Should scale from 2 pods to 1 pod", func(ctx context.Context) {
   106  			st := &HPAScaleTest{
   107  				initPods:         2,
   108  				initCPUTotal:     50,
   109  				perPodCPURequest: 200,
   110  				targetValue:      50,
   111  				minPods:          1,
   112  				maxPods:          2,
   113  				firstScale:       1,
   114  				resourceType:     cpuResource,
   115  				metricTargetType: utilizationMetricType,
   116  			}
   117  			st.run(ctx, "rc-light", e2eautoscaling.KindRC, f)
   118  		})
   119  	})
   120  
   121  	f.Describe(framework.WithSerial(), framework.WithSlow(), "ReplicaSet with idle sidecar (ContainerResource use case)", func() {
   122  		// ContainerResource CPU autoscaling on idle sidecar
   123  		ginkgo.It(titleUp+" on a busy application with an idle sidecar container", func(ctx context.Context) {
   124  			scaleOnIdleSideCar(ctx, "rs", e2eautoscaling.KindReplicaSet, cpuResource, utilizationMetricType, false, f)
   125  		})
   126  
   127  		// ContainerResource CPU autoscaling on busy sidecar
   128  		ginkgo.It("Should not scale up on a busy sidecar with an idle application", func(ctx context.Context) {
   129  			doNotScaleOnBusySidecar(ctx, "rs", e2eautoscaling.KindReplicaSet, cpuResource, utilizationMetricType, true, f)
   130  		})
   131  	})
   132  
   133  	f.Describe("CustomResourceDefinition", func() {
   134  		ginkgo.It("Should scale with a CRD targetRef", func(ctx context.Context) {
   135  			scaleTest := &HPAScaleTest{
   136  				initPods:         1,
   137  				initCPUTotal:     150,
   138  				perPodCPURequest: 200,
   139  				targetValue:      50,
   140  				minPods:          1,
   141  				maxPods:          2,
   142  				firstScale:       2,
   143  				resourceType:     cpuResource,
   144  				metricTargetType: utilizationMetricType,
   145  			}
   146  			scaleTest.run(ctx, "foo-crd", e2eautoscaling.KindCRD, f)
   147  		})
   148  	})
   149  })
   150  
   151  var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (scale resource: Memory)", func() {
   152  	f := framework.NewDefaultFramework("horizontal-pod-autoscaling")
   153  	f.NamespacePodSecurityLevel = api.LevelBaseline
   154  
   155  	f.Describe(framework.WithSerial(), framework.WithSlow(), "Deployment (Pod Resource)", func() {
   156  		ginkgo.It(titleUp+titleAverageUtilization, func(ctx context.Context) {
   157  			scaleUp(ctx, "test-deployment", e2eautoscaling.KindDeployment, memResource, utilizationMetricType, false, f)
   158  		})
   159  		ginkgo.It(titleUp+titleAverageValue, func(ctx context.Context) {
   160  			scaleUp(ctx, "test-deployment", e2eautoscaling.KindDeployment, memResource, valueMetricType, false, f)
   161  		})
   162  	})
   163  
   164  	f.Describe(framework.WithSerial(), framework.WithSlow(), "Deployment (Container Resource)", func() {
   165  		ginkgo.It(titleUp+titleAverageUtilization, func(ctx context.Context) {
   166  			scaleUpContainerResource(ctx, "test-deployment", e2eautoscaling.KindDeployment, memResource, utilizationMetricType, f)
   167  		})
   168  		ginkgo.It(titleUp+titleAverageValue, func(ctx context.Context) {
   169  			scaleUpContainerResource(ctx, "test-deployment", e2eautoscaling.KindDeployment, memResource, valueMetricType, f)
   170  		})
   171  	})
   172  })
   173  
   174  // HPAScaleTest struct is used by the scale(...) function.
   175  type HPAScaleTest struct {
   176  	initPods         int
   177  	initCPUTotal     int
   178  	initMemTotal     int
   179  	perPodCPURequest int64
   180  	perPodMemRequest int64
   181  	targetValue      int32
   182  	minPods          int32
   183  	maxPods          int32
   184  	firstScale       int
   185  	firstScaleStasis time.Duration
   186  	cpuBurst         int
   187  	memBurst         int
   188  	secondScale      int32
   189  	resourceType     v1.ResourceName
   190  	metricTargetType autoscalingv2.MetricTargetType
   191  }
   192  
   193  // run is a method which runs an HPA lifecycle, from a starting state, to an expected
   194  // The initial state is defined by the initPods parameter.
   195  // The first state change is due to the CPU being consumed initially, which HPA responds to by changing pod counts.
   196  // The second state change (optional) is due to the CPU burst parameter, which HPA again responds to.
   197  // TODO The use of 3 states is arbitrary, we could eventually make this test handle "n" states once this test stabilizes.
   198  func (st *HPAScaleTest) run(ctx context.Context, name string, kind schema.GroupVersionKind, f *framework.Framework) {
   199  	const timeToWait = 15 * time.Minute
   200  	initCPUTotal, initMemTotal := 0, 0
   201  	if st.resourceType == cpuResource {
   202  		initCPUTotal = st.initCPUTotal
   203  	} else if st.resourceType == memResource {
   204  		initMemTotal = st.initMemTotal
   205  	}
   206  	rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, name, f.Namespace.Name, kind, st.initPods, initCPUTotal, initMemTotal, 0, st.perPodCPURequest, st.perPodMemRequest, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle)
   207  	ginkgo.DeferCleanup(rc.CleanUp)
   208  	hpa := e2eautoscaling.CreateResourceHorizontalPodAutoscaler(ctx, rc, st.resourceType, st.metricTargetType, st.targetValue, st.minPods, st.maxPods)
   209  	ginkgo.DeferCleanup(e2eautoscaling.DeleteHorizontalPodAutoscaler, rc, hpa.Name)
   210  
   211  	rc.WaitForReplicas(ctx, st.firstScale, timeToWait)
   212  	if st.firstScaleStasis > 0 {
   213  		rc.EnsureDesiredReplicasInRange(ctx, st.firstScale, st.firstScale+1, st.firstScaleStasis, hpa.Name)
   214  	}
   215  	if st.resourceType == cpuResource && st.cpuBurst > 0 && st.secondScale > 0 {
   216  		rc.ConsumeCPU(st.cpuBurst)
   217  		rc.WaitForReplicas(ctx, int(st.secondScale), timeToWait)
   218  	}
   219  	if st.resourceType == memResource && st.memBurst > 0 && st.secondScale > 0 {
   220  		rc.ConsumeMem(st.memBurst)
   221  		rc.WaitForReplicas(ctx, int(st.secondScale), timeToWait)
   222  	}
   223  }
   224  
   225  func scaleUp(ctx context.Context, name string, kind schema.GroupVersionKind, resourceType v1.ResourceName, metricTargetType autoscalingv2.MetricTargetType, checkStability bool, f *framework.Framework) {
   226  	stasis := 0 * time.Minute
   227  	if checkStability {
   228  		stasis = 10 * time.Minute
   229  	}
   230  	st := &HPAScaleTest{
   231  		initPods:         1,
   232  		perPodCPURequest: 500,
   233  		perPodMemRequest: 500,
   234  		targetValue:      getTargetValueByType(100, 20, metricTargetType),
   235  		minPods:          1,
   236  		maxPods:          5,
   237  		firstScale:       3,
   238  		firstScaleStasis: stasis,
   239  		secondScale:      5,
   240  		resourceType:     resourceType,
   241  		metricTargetType: metricTargetType,
   242  	}
   243  	if resourceType == cpuResource {
   244  		st.initCPUTotal = 250
   245  		st.cpuBurst = 700
   246  	}
   247  	if resourceType == memResource {
   248  		st.initMemTotal = 250
   249  		st.memBurst = 700
   250  	}
   251  	st.run(ctx, name, kind, f)
   252  }
   253  
   254  func scaleDown(ctx context.Context, name string, kind schema.GroupVersionKind, resourceType v1.ResourceName, metricTargetType autoscalingv2.MetricTargetType, checkStability bool, f *framework.Framework) {
   255  	stasis := 0 * time.Minute
   256  	if checkStability {
   257  		stasis = 10 * time.Minute
   258  	}
   259  	st := &HPAScaleTest{
   260  		initPods:         5,
   261  		perPodCPURequest: 500,
   262  		perPodMemRequest: 500,
   263  		targetValue:      getTargetValueByType(150, 30, metricTargetType),
   264  		minPods:          1,
   265  		maxPods:          5,
   266  		firstScale:       3,
   267  		firstScaleStasis: stasis,
   268  		cpuBurst:         10,
   269  		secondScale:      1,
   270  		resourceType:     resourceType,
   271  		metricTargetType: metricTargetType,
   272  	}
   273  	if resourceType == cpuResource {
   274  		st.initCPUTotal = 325
   275  		st.cpuBurst = 10
   276  	}
   277  	if resourceType == memResource {
   278  		st.initMemTotal = 325
   279  		st.memBurst = 10
   280  	}
   281  	st.run(ctx, name, kind, f)
   282  }
   283  
   284  type HPAContainerResourceScaleTest struct {
   285  	initPods               int
   286  	initCPUTotal           int
   287  	initMemTotal           int
   288  	perContainerCPURequest int64
   289  	perContainerMemRequest int64
   290  	targetValue            int32
   291  	minPods                int32
   292  	maxPods                int32
   293  	noScale                bool
   294  	noScaleStasis          time.Duration
   295  	firstScale             int
   296  	firstScaleStasis       time.Duration
   297  	cpuBurst               int
   298  	memBurst               int
   299  	secondScale            int32
   300  	sidecarStatus          e2eautoscaling.SidecarStatusType
   301  	sidecarType            e2eautoscaling.SidecarWorkloadType
   302  	resourceType           v1.ResourceName
   303  	metricTargetType       autoscalingv2.MetricTargetType
   304  }
   305  
   306  func (st *HPAContainerResourceScaleTest) run(ctx context.Context, name string, kind schema.GroupVersionKind, f *framework.Framework) {
   307  	const timeToWait = 15 * time.Minute
   308  	initCPUTotal, initMemTotal := 0, 0
   309  	if st.resourceType == cpuResource {
   310  		initCPUTotal = st.initCPUTotal
   311  	} else if st.resourceType == memResource {
   312  		initMemTotal = st.initMemTotal
   313  	}
   314  	rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, name, f.Namespace.Name, kind, st.initPods, initCPUTotal, initMemTotal, 0, st.perContainerCPURequest, st.perContainerMemRequest, f.ClientSet, f.ScalesGetter, st.sidecarStatus, st.sidecarType)
   315  	ginkgo.DeferCleanup(rc.CleanUp)
   316  	hpa := e2eautoscaling.CreateContainerResourceHorizontalPodAutoscaler(ctx, rc, st.resourceType, st.metricTargetType, st.targetValue, st.minPods, st.maxPods)
   317  	ginkgo.DeferCleanup(e2eautoscaling.DeleteContainerResourceHPA, rc, hpa.Name)
   318  
   319  	if st.noScale {
   320  		if st.noScaleStasis > 0 {
   321  			rc.EnsureDesiredReplicasInRange(ctx, st.initPods, st.initPods, st.noScaleStasis, hpa.Name)
   322  		}
   323  	} else {
   324  		rc.WaitForReplicas(ctx, st.firstScale, timeToWait)
   325  		if st.firstScaleStasis > 0 {
   326  			rc.EnsureDesiredReplicasInRange(ctx, st.firstScale, st.firstScale+1, st.firstScaleStasis, hpa.Name)
   327  		}
   328  		if st.resourceType == cpuResource && st.cpuBurst > 0 && st.secondScale > 0 {
   329  			rc.ConsumeCPU(st.cpuBurst)
   330  			rc.WaitForReplicas(ctx, int(st.secondScale), timeToWait)
   331  		}
   332  		if st.resourceType == memResource && st.memBurst > 0 && st.secondScale > 0 {
   333  			rc.ConsumeMem(st.memBurst)
   334  			rc.WaitForReplicas(ctx, int(st.secondScale), timeToWait)
   335  		}
   336  	}
   337  }
   338  
   339  func scaleUpContainerResource(ctx context.Context, name string, kind schema.GroupVersionKind, resourceType v1.ResourceName, metricTargetType autoscalingv2.MetricTargetType, f *framework.Framework) {
   340  	st := &HPAContainerResourceScaleTest{
   341  		initPods:               1,
   342  		perContainerCPURequest: 500,
   343  		perContainerMemRequest: 500,
   344  		targetValue:            getTargetValueByType(100, 20, metricTargetType),
   345  		minPods:                1,
   346  		maxPods:                5,
   347  		firstScale:             3,
   348  		firstScaleStasis:       0,
   349  		secondScale:            5,
   350  		resourceType:           resourceType,
   351  		metricTargetType:       metricTargetType,
   352  		sidecarStatus:          e2eautoscaling.Disable,
   353  		sidecarType:            e2eautoscaling.Idle,
   354  	}
   355  	if resourceType == cpuResource {
   356  		st.initCPUTotal = 250
   357  		st.cpuBurst = 700
   358  	}
   359  	if resourceType == memResource {
   360  		st.initMemTotal = 250
   361  		st.memBurst = 700
   362  	}
   363  	st.run(ctx, name, kind, f)
   364  }
   365  
   366  func scaleOnIdleSideCar(ctx context.Context, name string, kind schema.GroupVersionKind, resourceType v1.ResourceName, metricTargetType autoscalingv2.MetricTargetType, checkStability bool, f *framework.Framework) {
   367  	// Scale up on a busy application with an idle sidecar container
   368  	stasis := 0 * time.Minute
   369  	if checkStability {
   370  		stasis = 10 * time.Minute
   371  	}
   372  	st := &HPAContainerResourceScaleTest{
   373  		initPods:               1,
   374  		initCPUTotal:           125,
   375  		perContainerCPURequest: 250,
   376  		targetValue:            20,
   377  		minPods:                1,
   378  		maxPods:                5,
   379  		firstScale:             3,
   380  		firstScaleStasis:       stasis,
   381  		cpuBurst:               500,
   382  		secondScale:            5,
   383  		resourceType:           resourceType,
   384  		metricTargetType:       metricTargetType,
   385  		sidecarStatus:          e2eautoscaling.Enable,
   386  		sidecarType:            e2eautoscaling.Idle,
   387  	}
   388  	st.run(ctx, name, kind, f)
   389  }
   390  
   391  func doNotScaleOnBusySidecar(ctx context.Context, name string, kind schema.GroupVersionKind, resourceType v1.ResourceName, metricTargetType autoscalingv2.MetricTargetType, checkStability bool, f *framework.Framework) {
   392  	// Do not scale up on a busy sidecar with an idle application
   393  	stasis := 0 * time.Minute
   394  	if checkStability {
   395  		stasis = 1 * time.Minute
   396  	}
   397  	st := &HPAContainerResourceScaleTest{
   398  		initPods:               1,
   399  		initCPUTotal:           250,
   400  		perContainerCPURequest: 500,
   401  		targetValue:            20,
   402  		minPods:                1,
   403  		maxPods:                5,
   404  		cpuBurst:               700,
   405  		sidecarStatus:          e2eautoscaling.Enable,
   406  		sidecarType:            e2eautoscaling.Busy,
   407  		resourceType:           resourceType,
   408  		metricTargetType:       metricTargetType,
   409  		noScale:                true,
   410  		noScaleStasis:          stasis,
   411  	}
   412  	st.run(ctx, name, kind, f)
   413  }
   414  
   415  func getTargetValueByType(averageValueTarget, averageUtilizationTarget int, targetType autoscalingv2.MetricTargetType) int32 {
   416  	if targetType == utilizationMetricType {
   417  		return int32(averageUtilizationTarget)
   418  	}
   419  	return int32(averageValueTarget)
   420  }
   421
View as plain text