custom_metrics_deployments.go

Documentation: k8s.io/kubernetes/test/e2e/instrumentation/monitoring

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package monitoring
    18  
    19  import (
    20  	"fmt"
    21  	"os/exec"
    22  	"strings"
    23  
    24  	appsv1 "k8s.io/api/apps/v1"
    25  	v1 "k8s.io/api/core/v1"
    26  	rbacv1 "k8s.io/api/rbac/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/kubernetes/test/e2e/framework"
    29  	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
    30  	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
    31  	imageutils "k8s.io/kubernetes/test/utils/image"
    32  
    33  	gcm "google.golang.org/api/monitoring/v3"
    34  )
    35  
    36  var (
    37  	// CustomMetricName is the metrics name used in test cases.
    38  	CustomMetricName = "foo"
    39  	// UnusedMetricName is the unused metrics name used in test cases.
    40  	UnusedMetricName = "unused"
    41  	// CustomMetricValue is the value for CustomMetricName.
    42  	CustomMetricValue = int64(448)
    43  	// UnusedMetricValue is the value for UnusedMetricName.
    44  	UnusedMetricValue = int64(446)
    45  	// StackdriverExporter is exporter name.
    46  	StackdriverExporter = "stackdriver-exporter"
    47  	// HPAPermissions is a ClusterRoleBinding that grants unauthenticated user permissions granted for
    48  	// HPA for testing purposes, i.e. it should grant permission to read custom metrics.
    49  	HPAPermissions = &rbacv1.ClusterRoleBinding{
    50  		ObjectMeta: metav1.ObjectMeta{
    51  			Name: "custom-metrics-reader",
    52  		},
    53  		RoleRef: rbacv1.RoleRef{
    54  			APIGroup: "rbac.authorization.k8s.io",
    55  			Kind:     "ClusterRole",
    56  			Name:     "system:controller:horizontal-pod-autoscaler",
    57  		},
    58  		Subjects: []rbacv1.Subject{
    59  			{
    60  				APIGroup: "rbac.authorization.k8s.io",
    61  				Kind:     "Group",
    62  				Name:     "system:unauthenticated",
    63  			},
    64  		},
    65  	}
    66  	// StagingDeploymentsLocation is the location where the adapter deployment files are stored.
    67  	StagingDeploymentsLocation = "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/staging/"
    68  	// AdapterForOldResourceModel is file name for the old resource model.
    69  	AdapterForOldResourceModel = "adapter_old_resource_model.yaml"
    70  	// AdapterForNewResourceModel is file name for the new resource model.
    71  	AdapterForNewResourceModel = "adapter_new_resource_model.yaml"
    72  	// AdapterDefault is the default model.
    73  	AdapterDefault = AdapterForOldResourceModel
    74  	// ClusterAdminBinding is the cluster rolebinding name for test cases.
    75  	ClusterAdminBinding = "e2e-test-cluster-admin-binding"
    76  )
    77  
    78  // CustomMetricContainerSpec allows to specify a config for StackdriverExporterDeployment
    79  // with multiple containers exporting different metrics.
    80  type CustomMetricContainerSpec struct {
    81  	Name        string
    82  	MetricName  string
    83  	MetricValue int64
    84  }
    85  
    86  // SimpleStackdriverExporterDeployment is a Deployment of simple application that exports a metric of
    87  // fixed value to Stackdriver in a loop.
    88  func SimpleStackdriverExporterDeployment(name, namespace string, replicas int32, metricValue int64) *appsv1.Deployment {
    89  	return StackdriverExporterDeployment(name, namespace, replicas,
    90  		[]CustomMetricContainerSpec{
    91  			{
    92  				Name:        StackdriverExporter,
    93  				MetricName:  CustomMetricName,
    94  				MetricValue: metricValue,
    95  			},
    96  		})
    97  }
    98  
    99  // StackdriverExporterDeployment is a Deployment of an application that can expose
   100  // an arbitrary amount of metrics of fixed value to Stackdriver in a loop. Each metric
   101  // is exposed by a different container in one pod.
   102  // The metric names and values are configured via the containers parameter.
   103  func StackdriverExporterDeployment(name, namespace string, replicas int32, containers []CustomMetricContainerSpec) *appsv1.Deployment {
   104  	podSpec := v1.PodSpec{Containers: []v1.Container{}}
   105  	for _, containerSpec := range containers {
   106  		podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, namespace, containerSpec.MetricName, containerSpec.MetricValue))
   107  	}
   108  
   109  	d := e2edeployment.NewDeployment(name, replicas, map[string]string{"name": name}, "", "", appsv1.RollingUpdateDeploymentStrategyType)
   110  	d.ObjectMeta.Namespace = namespace
   111  	d.Spec.Template.Spec = podSpec
   112  	return d
   113  }
   114  
   115  // StackdriverExporterPod is a Pod of simple application that exports a metric of fixed value to
   116  // Stackdriver in a loop.
   117  func StackdriverExporterPod(podName, namespace, podLabel, metricName string, metricValue int64) *v1.Pod {
   118  	return &v1.Pod{
   119  		ObjectMeta: metav1.ObjectMeta{
   120  			Name:      podName,
   121  			Namespace: namespace,
   122  			Labels: map[string]string{
   123  				"name": podLabel,
   124  			},
   125  		},
   126  		Spec: v1.PodSpec{
   127  			Containers: []v1.Container{stackdriverExporterContainerSpec(StackdriverExporter, namespace, metricName, metricValue)},
   128  		},
   129  	}
   130  }
   131  
   132  func stackdriverExporterContainerSpec(name string, namespace string, metricName string, metricValue int64) v1.Container {
   133  	return v1.Container{
   134  		Name:            name,
   135  		Image:           imageutils.GetE2EImage(imageutils.SdDummyExporter),
   136  		ImagePullPolicy: v1.PullPolicy("Always"),
   137  		Command: []string{
   138  			"/bin/sh",
   139  			"-c",
   140  			strings.Join([]string{
   141  				"./sd_dummy_exporter",
   142  				"--pod-id=$(POD_ID)",
   143  				"--pod-name=$(POD_NAME)",
   144  				"--namespace=" + namespace,
   145  				"--metric-name=" + metricName,
   146  				fmt.Sprintf("--metric-value=%v", metricValue),
   147  				"--use-old-resource-model",
   148  				"--use-new-resource-model",
   149  			}, " "),
   150  		},
   151  		Env: []v1.EnvVar{
   152  			{
   153  				Name: "POD_ID",
   154  				ValueFrom: &v1.EnvVarSource{
   155  					FieldRef: &v1.ObjectFieldSelector{
   156  						FieldPath: "metadata.uid",
   157  					},
   158  				},
   159  			},
   160  			{
   161  				Name: "POD_NAME",
   162  				ValueFrom: &v1.EnvVarSource{
   163  					FieldRef: &v1.ObjectFieldSelector{
   164  						FieldPath: "metadata.name",
   165  					},
   166  				},
   167  			},
   168  		},
   169  		Ports: []v1.ContainerPort{{ContainerPort: 80}},
   170  	}
   171  }
   172  
   173  // PrometheusExporterDeployment is a Deployment of simple application with two containers
   174  // one exposing a metric in prometheus format and second a prometheus-to-sd container
   175  // that scrapes the metric and pushes it to stackdriver.
   176  func PrometheusExporterDeployment(name, namespace string, replicas int32, metricValue int64) *appsv1.Deployment {
   177  	d := e2edeployment.NewDeployment(name, replicas, map[string]string{"name": name}, "", "", appsv1.RollingUpdateDeploymentStrategyType)
   178  	d.ObjectMeta.Namespace = namespace
   179  	d.Spec.Template.Spec = prometheusExporterPodSpec(CustomMetricName, metricValue, 8080)
   180  	return d
   181  }
   182  
   183  func prometheusExporterPodSpec(metricName string, metricValue int64, port int32) v1.PodSpec {
   184  	return v1.PodSpec{
   185  		Containers: []v1.Container{
   186  			{
   187  				Name:            "prometheus-exporter",
   188  				Image:           imageutils.GetE2EImage(imageutils.PrometheusDummyExporter),
   189  				ImagePullPolicy: v1.PullPolicy("Always"),
   190  				Command: []string{"/prometheus_dummy_exporter", "--metric-name=" + metricName,
   191  					fmt.Sprintf("--metric-value=%v", metricValue), fmt.Sprintf("=--port=%d", port)},
   192  				Ports: []v1.ContainerPort{{ContainerPort: port}},
   193  			},
   194  			{
   195  				Name:            "prometheus-to-sd",
   196  				Image:           imageutils.GetE2EImage(imageutils.PrometheusToSd),
   197  				ImagePullPolicy: v1.PullPolicy("Always"),
   198  				Command: []string{"/monitor", fmt.Sprintf("--source=:http://localhost:%d", port),
   199  					"--stackdriver-prefix=custom.googleapis.com", "--pod-id=$(POD_ID)", "--namespace-id=$(POD_NAMESPACE)"},
   200  				Env: []v1.EnvVar{
   201  					{
   202  						Name: "POD_ID",
   203  						ValueFrom: &v1.EnvVarSource{
   204  							FieldRef: &v1.ObjectFieldSelector{
   205  								FieldPath: "metadata.uid",
   206  							},
   207  						},
   208  					},
   209  					{
   210  						Name: "POD_NAMESPACE",
   211  						ValueFrom: &v1.EnvVarSource{
   212  							FieldRef: &v1.ObjectFieldSelector{
   213  								FieldPath: "metadata.namespace",
   214  							},
   215  						},
   216  					},
   217  				},
   218  			},
   219  		},
   220  	}
   221  }
   222  
   223  // CreateAdapter creates Custom Metrics - Stackdriver adapter
   224  // adapterDeploymentFile should be a filename for adapter deployment located in StagingDeploymentLocation
   225  func CreateAdapter(adapterDeploymentFile string) error {
   226  	// A workaround to make the work on GKE. GKE doesn't normally allow to create cluster roles,
   227  	// which the adapter deployment does. The solution is to create cluster role binding for
   228  	// cluster-admin role and currently used service account.
   229  	err := createClusterAdminBinding()
   230  	if err != nil {
   231  		return err
   232  	}
   233  	adapterURL := StagingDeploymentsLocation + adapterDeploymentFile
   234  	err = exec.Command("wget", adapterURL).Run()
   235  	if err != nil {
   236  		return err
   237  	}
   238  	stat, err := e2ekubectl.RunKubectl("", "apply", "-f", adapterURL)
   239  	framework.Logf(stat)
   240  	return err
   241  }
   242  
   243  func createClusterAdminBinding() error {
   244  	stdout, stderr, err := framework.RunCmd("gcloud", "config", "get-value", "core/account")
   245  	if err != nil {
   246  		framework.Logf(stderr)
   247  		return err
   248  	}
   249  	serviceAccount := strings.TrimSpace(stdout)
   250  	framework.Logf("current service account: %q", serviceAccount)
   251  	stat, err := e2ekubectl.RunKubectl("", "create", "clusterrolebinding", ClusterAdminBinding, "--clusterrole=cluster-admin", "--user="+serviceAccount)
   252  	framework.Logf(stat)
   253  	return err
   254  }
   255  
   256  // CreateDescriptors creates descriptors for metrics: CustomMetricName and UnusedMetricName.
   257  func CreateDescriptors(service *gcm.Service, projectID string) error {
   258  	_, err := service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectID), &gcm.MetricDescriptor{
   259  		Name:       CustomMetricName,
   260  		ValueType:  "INT64",
   261  		Type:       "custom.googleapis.com/" + CustomMetricName,
   262  		MetricKind: "GAUGE",
   263  	}).Do()
   264  	if err != nil {
   265  		return err
   266  	}
   267  	_, err = service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectID), &gcm.MetricDescriptor{
   268  		Name:       UnusedMetricName,
   269  		ValueType:  "INT64",
   270  		Type:       "custom.googleapis.com/" + UnusedMetricName,
   271  		MetricKind: "GAUGE",
   272  	}).Do()
   273  	return err
   274  }
   275  
   276  // CleanupDescriptors deletes descriptors for metrics: CustomMetricName and UnusedMetricName.
   277  // TODO: Cleanup time series as well
   278  func CleanupDescriptors(service *gcm.Service, projectID string) {
   279  	_, err := service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectID, CustomMetricName)).Do()
   280  	if err != nil {
   281  		framework.Logf("Failed to delete descriptor for metric '%s': %v", CustomMetricName, err)
   282  	}
   283  	_, err = service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectID, UnusedMetricName)).Do()
   284  	if err != nil {
   285  		framework.Logf("Failed to delete descriptor for metric '%s': %v", CustomMetricName, err)
   286  	}
   287  }
   288  
   289  // CleanupAdapter deletes Custom Metrics - Stackdriver adapter deployments.
   290  func CleanupAdapter(adapterDeploymentFile string) {
   291  	stat, err := e2ekubectl.RunKubectl("", "delete", "-f", adapterDeploymentFile)
   292  	framework.Logf(stat)
   293  	if err != nil {
   294  		framework.Logf("Failed to delete adapter deployments: %s", err)
   295  	}
   296  	err = exec.Command("rm", adapterDeploymentFile).Run()
   297  	if err != nil {
   298  		framework.Logf("Failed to delete adapter deployment file: %s", err)
   299  	}
   300  	cleanupClusterAdminBinding()
   301  }
   302  
   303  func cleanupClusterAdminBinding() {
   304  	stat, err := e2ekubectl.RunKubectl("", "delete", "clusterrolebinding", ClusterAdminBinding)
   305  	framework.Logf(stat)
   306  	if err != nil {
   307  		framework.Logf("Failed to delete cluster admin binding: %s", err)
   308  	}
   309  }
   310
View as plain text