Source file src/k8s.io/kubernetes/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go

Documentation: k8s.io/kubernetes/test/e2e/autoscaling

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package autoscaling
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"math"
    23  	"strings"
    24  	"time"
    25  
    26  	gcm "google.golang.org/api/monitoring/v3"
    27  	"google.golang.org/api/option"
    28  	appsv1 "k8s.io/api/apps/v1"
    29  	autoscalingv2 "k8s.io/api/autoscaling/v2"
    30  	v1 "k8s.io/api/core/v1"
    31  	"k8s.io/apimachinery/pkg/api/resource"
    32  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    33  	"k8s.io/apimachinery/pkg/util/wait"
    34  	clientset "k8s.io/client-go/kubernetes"
    35  	"k8s.io/kubernetes/test/e2e/feature"
    36  	"k8s.io/kubernetes/test/e2e/framework"
    37  	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
    38  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    39  	"k8s.io/kubernetes/test/e2e/instrumentation/monitoring"
    40  	admissionapi "k8s.io/pod-security-admission/api"
    41  
    42  	"github.com/onsi/ginkgo/v2"
    43  	"golang.org/x/oauth2/google"
    44  )
    45  
    46  const (
    47  	stackdriverExporterDeployment = "stackdriver-exporter-deployment"
    48  	dummyDeploymentName           = "dummy-deployment"
    49  	stackdriverExporterPod        = "stackdriver-exporter-pod"
    50  	externalMetricValue           = int64(85)
    51  )
    52  
    53  type externalMetricTarget struct {
    54  	value     int64
    55  	isAverage bool
    56  }
    57  
    58  var _ = SIGDescribe("[HPA]", feature.CustomMetricsAutoscaling, "Horizontal pod autoscaling (scale resource: Custom Metrics from Stackdriver)", func() {
    59  	ginkgo.BeforeEach(func() {
    60  		e2eskipper.SkipUnlessProviderIs("gce", "gke")
    61  	})
    62  
    63  	f := framework.NewDefaultFramework("horizontal-pod-autoscaling")
    64  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    65  
    66  	ginkgo.Describe("with Custom Metric of type Pod from Stackdriver", func() {
    67  		ginkgo.It("should scale down", func(ctx context.Context) {
    68  			initialReplicas := 2
    69  			// metric should cause scale down
    70  			metricValue := int64(100)
    71  			metricTarget := 2 * metricValue
    72  			metricSpecs := []autoscalingv2.MetricSpec{
    73  				podMetricSpecWithAverageValueTarget(monitoring.CustomMetricName, metricTarget),
    74  			}
    75  			tc := CustomMetricTestCase{
    76  				framework:       f,
    77  				kubeClient:      f.ClientSet,
    78  				initialReplicas: initialReplicas,
    79  				scaledReplicas:  1,
    80  				deployment:      monitoring.SimpleStackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue),
    81  				hpa:             hpa("custom-metrics-pods-hpa", f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, 1, 3, metricSpecs),
    82  			}
    83  			tc.Run(ctx)
    84  		})
    85  
    86  		ginkgo.It("should scale up with two metrics", func(ctx context.Context) {
    87  			initialReplicas := 1
    88  			// metric 1 would cause a scale down, if not for metric 2
    89  			metric1Value := int64(100)
    90  			metric1Target := 2 * metric1Value
    91  			// metric2 should cause a scale up
    92  			metric2Value := int64(200)
    93  			metric2Target := int64(0.5 * float64(metric2Value))
    94  			metricSpecs := []autoscalingv2.MetricSpec{
    95  				podMetricSpecWithAverageValueTarget("metric1", metric1Target),
    96  				podMetricSpecWithAverageValueTarget("metric2", metric2Target),
    97  			}
    98  			containers := []monitoring.CustomMetricContainerSpec{
    99  				{
   100  					Name:        "stackdriver-exporter-metric1",
   101  					MetricName:  "metric1",
   102  					MetricValue: metric1Value,
   103  				},
   104  				{
   105  					Name:        "stackdriver-exporter-metric2",
   106  					MetricName:  "metric2",
   107  					MetricValue: metric2Value,
   108  				},
   109  			}
   110  			tc := CustomMetricTestCase{
   111  				framework:       f,
   112  				kubeClient:      f.ClientSet,
   113  				initialReplicas: initialReplicas,
   114  				scaledReplicas:  3,
   115  				deployment:      monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers),
   116  				hpa:             hpa("custom-metrics-pods-hpa", f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, 1, 3, metricSpecs),
   117  			}
   118  			tc.Run(ctx)
   119  		})
   120  
   121  		ginkgo.It("should scale down with Prometheus", func(ctx context.Context) {
   122  			initialReplicas := 2
   123  			// metric should cause scale down
   124  			metricValue := int64(100)
   125  			metricTarget := 2 * metricValue
   126  			metricSpecs := []autoscalingv2.MetricSpec{
   127  				podMetricSpecWithAverageValueTarget(monitoring.CustomMetricName, metricTarget),
   128  			}
   129  			tc := CustomMetricTestCase{
   130  				framework:       f,
   131  				kubeClient:      f.ClientSet,
   132  				initialReplicas: initialReplicas,
   133  				scaledReplicas:  1,
   134  				deployment:      monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue),
   135  				hpa:             hpa("custom-metrics-pods-hpa", f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, 1, 3, metricSpecs),
   136  			}
   137  			tc.Run(ctx)
   138  		})
   139  	})
   140  
   141  	ginkgo.Describe("with Custom Metric of type Object from Stackdriver", func() {
   142  		ginkgo.It("should scale down", func(ctx context.Context) {
   143  			initialReplicas := 2
   144  			// metric should cause scale down
   145  			metricValue := int64(100)
   146  			metricTarget := 2 * metricValue
   147  			metricSpecs := []autoscalingv2.MetricSpec{
   148  				objectMetricSpecWithValueTarget(metricTarget),
   149  			}
   150  			tc := CustomMetricTestCase{
   151  				framework:       f,
   152  				kubeClient:      f.ClientSet,
   153  				initialReplicas: initialReplicas,
   154  				scaledReplicas:  1,
   155  				deployment:      noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)),
   156  				pod:             monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue),
   157  				hpa:             hpa("custom-metrics-objects-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs),
   158  			}
   159  			tc.Run(ctx)
   160  		})
   161  
   162  		ginkgo.It("should scale down to 0", func(ctx context.Context) {
   163  			initialReplicas := 2
   164  			// metric should cause scale down
   165  			metricValue := int64(0)
   166  			metricTarget := int64(200)
   167  			metricSpecs := []autoscalingv2.MetricSpec{
   168  				objectMetricSpecWithValueTarget(metricTarget),
   169  			}
   170  			tc := CustomMetricTestCase{
   171  				framework:       f,
   172  				kubeClient:      f.ClientSet,
   173  				initialReplicas: initialReplicas,
   174  				scaledReplicas:  0,
   175  				deployment:      noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)),
   176  				pod:             monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue),
   177  				hpa:             hpa("custom-metrics-objects-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 0, 3, metricSpecs),
   178  			}
   179  			tc.Run(ctx)
   180  		})
   181  	})
   182  
   183  	ginkgo.Describe("with External Metric from Stackdriver", func() {
   184  		ginkgo.It("should scale down with target value", func(ctx context.Context) {
   185  			initialReplicas := 2
   186  			// metric should cause scale down
   187  			metricValue := externalMetricValue
   188  			metricTarget := 3 * metricValue
   189  			metricSpecs := []autoscalingv2.MetricSpec{
   190  				externalMetricSpecWithTarget("target", f.Namespace.ObjectMeta.Name, externalMetricTarget{
   191  					value:     metricTarget,
   192  					isAverage: false,
   193  				}),
   194  			}
   195  			tc := CustomMetricTestCase{
   196  				framework:       f,
   197  				kubeClient:      f.ClientSet,
   198  				initialReplicas: initialReplicas,
   199  				scaledReplicas:  1,
   200  				deployment:      noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)),
   201  				pod:             monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, "target", metricValue),
   202  				hpa:             hpa("custom-metrics-external-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs),
   203  			}
   204  			tc.Run(ctx)
   205  		})
   206  
   207  		ginkgo.It("should scale down with target average value", func(ctx context.Context) {
   208  			initialReplicas := 2
   209  			// metric should cause scale down
   210  			metricValue := externalMetricValue
   211  			metricAverageTarget := 3 * metricValue
   212  			metricSpecs := []autoscalingv2.MetricSpec{
   213  				externalMetricSpecWithTarget("target_average", f.Namespace.ObjectMeta.Name, externalMetricTarget{
   214  					value:     metricAverageTarget,
   215  					isAverage: true,
   216  				}),
   217  			}
   218  			tc := CustomMetricTestCase{
   219  				framework:       f,
   220  				kubeClient:      f.ClientSet,
   221  				initialReplicas: initialReplicas,
   222  				scaledReplicas:  1,
   223  				deployment:      noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)),
   224  				pod:             monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, "target_average", externalMetricValue),
   225  				hpa:             hpa("custom-metrics-external-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs),
   226  			}
   227  			tc.Run(ctx)
   228  		})
   229  
   230  		ginkgo.It("should scale up with two metrics", func(ctx context.Context) {
   231  			initialReplicas := 1
   232  			// metric 1 would cause a scale down, if not for metric 2
   233  			metric1Value := externalMetricValue
   234  			metric1Target := 2 * metric1Value
   235  			// metric2 should cause a scale up
   236  			metric2Value := externalMetricValue
   237  			metric2Target := int64(math.Ceil(0.5 * float64(metric2Value)))
   238  			metricSpecs := []autoscalingv2.MetricSpec{
   239  				externalMetricSpecWithTarget("external_metric_1", f.Namespace.ObjectMeta.Name, externalMetricTarget{
   240  					value:     metric1Target,
   241  					isAverage: true,
   242  				}),
   243  				externalMetricSpecWithTarget("external_metric_2", f.Namespace.ObjectMeta.Name, externalMetricTarget{
   244  					value:     metric2Target,
   245  					isAverage: true,
   246  				}),
   247  			}
   248  			containers := []monitoring.CustomMetricContainerSpec{
   249  				{
   250  					Name:        "stackdriver-exporter-metric1",
   251  					MetricName:  "external_metric_1",
   252  					MetricValue: metric1Value,
   253  				},
   254  				{
   255  					Name:        "stackdriver-exporter-metric2",
   256  					MetricName:  "external_metric_2",
   257  					MetricValue: metric2Value,
   258  				},
   259  			}
   260  			tc := CustomMetricTestCase{
   261  				framework:       f,
   262  				kubeClient:      f.ClientSet,
   263  				initialReplicas: initialReplicas,
   264  				scaledReplicas:  3,
   265  				deployment:      monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers),
   266  				hpa:             hpa("custom-metrics-external-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs),
   267  			}
   268  			tc.Run(ctx)
   269  		})
   270  	})
   271  
   272  	ginkgo.Describe("with multiple metrics of different types", func() {
   273  		ginkgo.It("should scale up when one metric is missing (Pod and External metrics)", func(ctx context.Context) {
   274  			initialReplicas := 1
   275  			// First metric a pod metric which is missing.
   276  			// Second metric is external metric which is present, it should cause scale up.
   277  			metricSpecs := []autoscalingv2.MetricSpec{
   278  				podMetricSpecWithAverageValueTarget(monitoring.CustomMetricName, 2*externalMetricValue),
   279  				externalMetricSpecWithTarget("external_metric", f.Namespace.ObjectMeta.Name, externalMetricTarget{
   280  					value:     int64(math.Ceil(0.5 * float64(externalMetricValue))),
   281  					isAverage: true,
   282  				}),
   283  			}
   284  			containers := []monitoring.CustomMetricContainerSpec{
   285  				{
   286  					Name:        "stackdriver-exporter-metric",
   287  					MetricName:  "external_metric",
   288  					MetricValue: externalMetricValue,
   289  				},
   290  				// Pod Resource metric is missing from here.
   291  			}
   292  			tc := CustomMetricTestCase{
   293  				framework:       f,
   294  				kubeClient:      f.ClientSet,
   295  				initialReplicas: initialReplicas,
   296  				scaledReplicas:  3,
   297  				deployment:      monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers),
   298  				hpa:             hpa("multiple-metrics", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs)}
   299  			tc.Run(ctx)
   300  		})
   301  
   302  		ginkgo.It("should scale up when one metric is missing (Resource and Object metrics)", func(ctx context.Context) {
   303  			initialReplicas := 1
   304  			metricValue := int64(100)
   305  			// First metric a resource metric which is missing (no consumption).
   306  			// Second metric is object metric which is present, it should cause scale up.
   307  			metricSpecs := []autoscalingv2.MetricSpec{
   308  				resourceMetricSpecWithAverageUtilizationTarget(50),
   309  				objectMetricSpecWithValueTarget(int64(math.Ceil(0.5 * float64(metricValue)))),
   310  			}
   311  			tc := CustomMetricTestCase{
   312  				framework:       f,
   313  				kubeClient:      f.ClientSet,
   314  				initialReplicas: initialReplicas,
   315  				scaledReplicas:  3,
   316  				deployment:      monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 0),
   317  				pod:             monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue),
   318  				hpa:             hpa("multiple-metrics", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs)}
   319  			tc.Run(ctx)
   320  		})
   321  
   322  		ginkgo.It("should not scale down when one metric is missing (Container Resource and External Metrics)", func(ctx context.Context) {
   323  			initialReplicas := 2
   324  			// First metric a container resource metric which is missing.
   325  			// Second metric is external metric which is present, it should cause scale down if the first metric wasn't missing.
   326  			metricSpecs := []autoscalingv2.MetricSpec{
   327  				containerResourceMetricSpecWithAverageUtilizationTarget("container-resource-metric", 50),
   328  				externalMetricSpecWithTarget("external_metric", f.Namespace.ObjectMeta.Name, externalMetricTarget{
   329  					value:     2 * externalMetricValue,
   330  					isAverage: true,
   331  				}),
   332  			}
   333  			containers := []monitoring.CustomMetricContainerSpec{
   334  				{
   335  					Name:        "stackdriver-exporter-metric",
   336  					MetricName:  "external_metric",
   337  					MetricValue: externalMetricValue,
   338  				},
   339  				// Container Resource metric is missing from here.
   340  			}
   341  			tc := CustomMetricTestCase{
   342  				framework:       f,
   343  				kubeClient:      f.ClientSet,
   344  				initialReplicas: initialReplicas,
   345  				scaledReplicas:  initialReplicas,
   346  				verifyStability: true,
   347  				deployment:      monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers),
   348  				hpa:             hpa("multiple-metrics", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs)}
   349  			tc.Run(ctx)
   350  		})
   351  
   352  		ginkgo.It("should not scale down when one metric is missing (Pod and Object Metrics)", func(ctx context.Context) {
   353  			initialReplicas := 2
   354  			metricValue := int64(100)
   355  			// First metric an object metric which is missing.
   356  			// Second metric is pod metric which is present, it should cause scale down if the first metric wasn't missing.
   357  			metricSpecs := []autoscalingv2.MetricSpec{
   358  				objectMetricSpecWithValueTarget(int64(math.Ceil(0.5 * float64(metricValue)))),
   359  				podMetricSpecWithAverageValueTarget("pod_metric", 2*metricValue),
   360  			}
   361  			containers := []monitoring.CustomMetricContainerSpec{
   362  				{
   363  					Name:        "stackdriver-exporter-metric",
   364  					MetricName:  "pod_metric",
   365  					MetricValue: metricValue,
   366  				},
   367  			}
   368  			tc := CustomMetricTestCase{
   369  				framework:       f,
   370  				kubeClient:      f.ClientSet,
   371  				initialReplicas: initialReplicas,
   372  				scaledReplicas:  initialReplicas,
   373  				verifyStability: true,
   374  				deployment:      monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers),
   375  				hpa:             hpa("multiple-metrics", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs)}
   376  			tc.Run(ctx)
   377  		})
   378  	})
   379  
   380  })
   381  
   382  // CustomMetricTestCase is a struct for test cases.
   383  type CustomMetricTestCase struct {
   384  	framework       *framework.Framework
   385  	hpa             *autoscalingv2.HorizontalPodAutoscaler
   386  	kubeClient      clientset.Interface
   387  	deployment      *appsv1.Deployment
   388  	pod             *v1.Pod
   389  	initialReplicas int
   390  	scaledReplicas  int
   391  	verifyStability bool
   392  }
   393  
   394  // Run starts test case.
   395  func (tc *CustomMetricTestCase) Run(ctx context.Context) {
   396  	projectID := framework.TestContext.CloudConfig.ProjectID
   397  
   398  	client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
   399  	if err != nil {
   400  		framework.Failf("Failed to initialize gcm default client, %v", err)
   401  	}
   402  
   403  	// Hack for running tests locally, needed to authenticate in Stackdriver
   404  	// If this is your use case, create application default credentials:
   405  	// $ gcloud auth application-default login
   406  	// and uncomment following lines:
   407  
   408  	// ts, err := google.DefaultTokenSource(oauth2.NoContext)
   409  	// framework.Logf("Couldn't get application default credentials, %v", err)
   410  	// if err != nil {
   411  	// 	framework.Failf("Error accessing application default credentials, %v", err)
   412  	// }
   413  	// client = oauth2.NewClient(oauth2.NoContext, ts)
   414  
   415  	gcmService, err := gcm.NewService(ctx, option.WithHTTPClient(client))
   416  	if err != nil {
   417  		framework.Failf("Failed to create gcm service, %v", err)
   418  	}
   419  
   420  	// Set up a cluster: create a custom metric and set up k8s-sd adapter
   421  	err = monitoring.CreateDescriptors(gcmService, projectID)
   422  	if err != nil {
   423  		if strings.Contains(err.Error(), "Request throttled") {
   424  			e2eskipper.Skipf("Skipping...hitting rate limits on creating and updating metrics/labels")
   425  		}
   426  		framework.Failf("Failed to create metric descriptor: %v", err)
   427  	}
   428  	defer monitoring.CleanupDescriptors(gcmService, projectID)
   429  
   430  	err = monitoring.CreateAdapter(monitoring.AdapterDefault)
   431  	defer monitoring.CleanupAdapter(monitoring.AdapterDefault)
   432  	if err != nil {
   433  		framework.Failf("Failed to set up: %v", err)
   434  	}
   435  
   436  	// Run application that exports the metric
   437  	err = createDeploymentToScale(ctx, tc.framework, tc.kubeClient, tc.deployment, tc.pod)
   438  	if err != nil {
   439  		framework.Failf("Failed to create stackdriver-exporter pod: %v", err)
   440  	}
   441  	ginkgo.DeferCleanup(cleanupDeploymentsToScale, tc.framework, tc.kubeClient, tc.deployment, tc.pod)
   442  
   443  	// Wait for the deployment to run
   444  	waitForReplicas(ctx, tc.deployment.ObjectMeta.Name, tc.framework.Namespace.ObjectMeta.Name, tc.kubeClient, 15*time.Minute, tc.initialReplicas)
   445  
   446  	// Autoscale the deployment
   447  	_, err = tc.kubeClient.AutoscalingV2().HorizontalPodAutoscalers(tc.framework.Namespace.ObjectMeta.Name).Create(ctx, tc.hpa, metav1.CreateOptions{})
   448  	if err != nil {
   449  		framework.Failf("Failed to create HPA: %v", err)
   450  	}
   451  	ginkgo.DeferCleanup(framework.IgnoreNotFound(tc.kubeClient.AutoscalingV2().HorizontalPodAutoscalers(tc.framework.Namespace.ObjectMeta.Name).Delete), tc.hpa.ObjectMeta.Name, metav1.DeleteOptions{})
   452  
   453  	waitForReplicas(ctx, tc.deployment.ObjectMeta.Name, tc.framework.Namespace.ObjectMeta.Name, tc.kubeClient, 15*time.Minute, tc.scaledReplicas)
   454  
   455  	if tc.verifyStability {
   456  		ensureDesiredReplicasInRange(ctx, tc.deployment.ObjectMeta.Name, tc.framework.Namespace.ObjectMeta.Name, tc.kubeClient, tc.scaledReplicas, tc.scaledReplicas, 10*time.Minute)
   457  	}
   458  }
   459  
   460  func createDeploymentToScale(ctx context.Context, f *framework.Framework, cs clientset.Interface, deployment *appsv1.Deployment, pod *v1.Pod) error {
   461  	if deployment != nil {
   462  		_, err := cs.AppsV1().Deployments(f.Namespace.ObjectMeta.Name).Create(ctx, deployment, metav1.CreateOptions{})
   463  		if err != nil {
   464  			return err
   465  		}
   466  	}
   467  	if pod != nil {
   468  		_, err := cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Create(ctx, pod, metav1.CreateOptions{})
   469  		if err != nil {
   470  			return err
   471  		}
   472  	}
   473  	return nil
   474  }
   475  
   476  func cleanupDeploymentsToScale(ctx context.Context, f *framework.Framework, cs clientset.Interface, deployment *appsv1.Deployment, pod *v1.Pod) {
   477  	if deployment != nil {
   478  		_ = cs.AppsV1().Deployments(f.Namespace.ObjectMeta.Name).Delete(ctx, deployment.ObjectMeta.Name, metav1.DeleteOptions{})
   479  	}
   480  	if pod != nil {
   481  		_ = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Delete(ctx, pod.ObjectMeta.Name, metav1.DeleteOptions{})
   482  	}
   483  }
   484  
   485  func podMetricSpecWithAverageValueTarget(metric string, targetValue int64) autoscalingv2.MetricSpec {
   486  	return autoscalingv2.MetricSpec{
   487  		Type: autoscalingv2.PodsMetricSourceType,
   488  		Pods: &autoscalingv2.PodsMetricSource{
   489  			Metric: autoscalingv2.MetricIdentifier{
   490  				Name: metric,
   491  			},
   492  			Target: autoscalingv2.MetricTarget{
   493  				Type:         autoscalingv2.AverageValueMetricType,
   494  				AverageValue: resource.NewQuantity(targetValue, resource.DecimalSI),
   495  			},
   496  		},
   497  	}
   498  }
   499  
   500  func objectMetricSpecWithValueTarget(targetValue int64) autoscalingv2.MetricSpec {
   501  	return autoscalingv2.MetricSpec{
   502  		Type: autoscalingv2.ObjectMetricSourceType,
   503  		Object: &autoscalingv2.ObjectMetricSource{
   504  			Metric: autoscalingv2.MetricIdentifier{
   505  				Name: monitoring.CustomMetricName,
   506  			},
   507  			DescribedObject: autoscalingv2.CrossVersionObjectReference{
   508  				Kind: "Pod",
   509  				Name: stackdriverExporterPod,
   510  			},
   511  			Target: autoscalingv2.MetricTarget{
   512  				Type:  autoscalingv2.ValueMetricType,
   513  				Value: resource.NewQuantity(targetValue, resource.DecimalSI),
   514  			},
   515  		},
   516  	}
   517  }
   518  
   519  func resourceMetricSpecWithAverageUtilizationTarget(targetValue int32) autoscalingv2.MetricSpec {
   520  	return autoscalingv2.MetricSpec{
   521  		Type: autoscalingv2.ResourceMetricSourceType,
   522  		Resource: &autoscalingv2.ResourceMetricSource{
   523  			Name: v1.ResourceCPU,
   524  			Target: autoscalingv2.MetricTarget{
   525  				Type:               autoscalingv2.UtilizationMetricType,
   526  				AverageUtilization: &targetValue,
   527  			},
   528  		},
   529  	}
   530  }
   531  
   532  func containerResourceMetricSpecWithAverageUtilizationTarget(containerName string, targetValue int32) autoscalingv2.MetricSpec {
   533  	return autoscalingv2.MetricSpec{
   534  		Type: autoscalingv2.ContainerResourceMetricSourceType,
   535  		ContainerResource: &autoscalingv2.ContainerResourceMetricSource{
   536  			Name:      v1.ResourceCPU,
   537  			Container: containerName,
   538  			Target: autoscalingv2.MetricTarget{
   539  				Type:               autoscalingv2.UtilizationMetricType,
   540  				AverageUtilization: &targetValue,
   541  			},
   542  		},
   543  	}
   544  }
   545  
   546  func externalMetricSpecWithTarget(metric string, namespace string, target externalMetricTarget) autoscalingv2.MetricSpec {
   547  	selector := &metav1.LabelSelector{
   548  		MatchLabels: map[string]string{"resource.type": "k8s_pod"},
   549  		MatchExpressions: []metav1.LabelSelectorRequirement{
   550  			{
   551  				Key:      "resource.labels.namespace_name",
   552  				Operator: metav1.LabelSelectorOpIn,
   553  				Values:   []string{namespace},
   554  			},
   555  			{
   556  				Key:      "resource.labels.pod_name",
   557  				Operator: metav1.LabelSelectorOpExists,
   558  				Values:   []string{},
   559  			},
   560  		},
   561  	}
   562  	metricSpec := autoscalingv2.MetricSpec{
   563  		Type: autoscalingv2.ExternalMetricSourceType,
   564  		External: &autoscalingv2.ExternalMetricSource{
   565  			Metric: autoscalingv2.MetricIdentifier{
   566  				Name:     "custom.googleapis.com|" + metric,
   567  				Selector: selector,
   568  			},
   569  		},
   570  	}
   571  	if target.isAverage {
   572  		metricSpec.External.Target.Type = autoscalingv2.AverageValueMetricType
   573  		metricSpec.External.Target.AverageValue = resource.NewQuantity(target.value, resource.DecimalSI)
   574  	} else {
   575  		metricSpec.External.Target.Type = autoscalingv2.ValueMetricType
   576  		metricSpec.External.Target.Value = resource.NewQuantity(target.value, resource.DecimalSI)
   577  	}
   578  	return metricSpec
   579  }
   580  
   581  func hpa(name, namespace, deploymentName string, minReplicas, maxReplicas int32, metricSpecs []autoscalingv2.MetricSpec) *autoscalingv2.HorizontalPodAutoscaler {
   582  	return &autoscalingv2.HorizontalPodAutoscaler{
   583  		ObjectMeta: metav1.ObjectMeta{
   584  			Name:      name,
   585  			Namespace: namespace,
   586  		},
   587  		Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
   588  			Metrics:     metricSpecs,
   589  			MinReplicas: &minReplicas,
   590  			MaxReplicas: maxReplicas,
   591  			ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{
   592  				APIVersion: "apps/v1",
   593  				Kind:       "Deployment",
   594  				Name:       deploymentName,
   595  			},
   596  		},
   597  	}
   598  }
   599  
   600  func waitForReplicas(ctx context.Context, deploymentName, namespace string, cs clientset.Interface, timeout time.Duration, desiredReplicas int) {
   601  	interval := 20 * time.Second
   602  	err := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
   603  		deployment, err := cs.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{})
   604  		if err != nil {
   605  			framework.Failf("Failed to get replication controller %s: %v", deployment, err)
   606  		}
   607  		replicas := int(deployment.Status.ReadyReplicas)
   608  		framework.Logf("waiting for %d replicas (current: %d)", desiredReplicas, replicas)
   609  		return replicas == desiredReplicas, nil // Expected number of replicas found. Exit.
   610  	})
   611  	if err != nil {
   612  		framework.Failf("Timeout waiting %v for %v replicas", timeout, desiredReplicas)
   613  	}
   614  }
   615  
   616  func ensureDesiredReplicasInRange(ctx context.Context, deploymentName, namespace string, cs clientset.Interface, minDesiredReplicas, maxDesiredReplicas int, timeout time.Duration) {
   617  	interval := 60 * time.Second
   618  	err := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
   619  		deployment, err := cs.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{})
   620  		if err != nil {
   621  			return true, err
   622  		}
   623  		replicas := int(deployment.Status.ReadyReplicas)
   624  		framework.Logf("expecting there to be in [%d, %d] replicas (are: %d)", minDesiredReplicas, maxDesiredReplicas, replicas)
   625  		if replicas < minDesiredReplicas {
   626  			return false, fmt.Errorf("number of replicas below target")
   627  		} else if replicas > maxDesiredReplicas {
   628  			return false, fmt.Errorf("number of replicas above target")
   629  		} else {
   630  			return false, nil // Expected number of replicas found. Continue polling until timeout.
   631  		}
   632  	})
   633  	// The call above always returns an error, but if it is timeout, it's OK (condition satisfied all the time).
   634  	if wait.Interrupted(err) || strings.Contains(err.Error(), "would exceed context deadline") {
   635  		framework.Logf("Number of replicas was stable over %v", timeout)
   636  		return
   637  	}
   638  	framework.ExpectNoErrorWithOffset(1, err)
   639  }
   640  
   641  func noExporterDeployment(name, namespace string, replicas int32) *appsv1.Deployment {
   642  	d := e2edeployment.NewDeployment(name, replicas, map[string]string{"name": name}, "", "", appsv1.RollingUpdateDeploymentStrategyType)
   643  	d.ObjectMeta.Namespace = namespace
   644  	d.Spec.Template.Spec = v1.PodSpec{Containers: []v1.Container{
   645  		{
   646  			Name:            "sleeper",
   647  			Image:           "registry.k8s.io/e2e-test-images/agnhost:2.40",
   648  			ImagePullPolicy: v1.PullAlways,
   649  			Command:         []string{"/agnhost"},
   650  			Args:            []string{"pause"}, // do nothing forever
   651  		},
   652  	}}
   653  	return d
   654  }
   655  

View as plain text