gce.go

Documentation: k8s.io/kubernetes/test/e2e/framework/providers/gce

     1  //go:build !providerless
     2  // +build !providerless
     3  
     4  /*
     5  Copyright 2018 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package gce
    21  
    22  import (
    23  	"context"
    24  	"fmt"
    25  	"math/rand"
    26  	"net/http"
    27  	"os/exec"
    28  	"regexp"
    29  	"strings"
    30  	"time"
    31  
    32  	compute "google.golang.org/api/compute/v1"
    33  	"google.golang.org/api/googleapi"
    34  	v1 "k8s.io/api/core/v1"
    35  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    36  	"k8s.io/apimachinery/pkg/util/uuid"
    37  	"k8s.io/apimachinery/pkg/util/wait"
    38  	clientset "k8s.io/client-go/kubernetes"
    39  	"k8s.io/kubernetes/test/e2e/framework"
    40  	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
    41  	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
    42  	gcecloud "k8s.io/legacy-cloud-providers/gce"
    43  )
    44  
    45  func init() {
    46  	framework.RegisterProvider("gce", factory)
    47  	framework.RegisterProvider("gke", factory)
    48  }
    49  
    50  func factory() (framework.ProviderInterface, error) {
    51  	framework.Logf("Fetching cloud provider for %q\r", framework.TestContext.Provider)
    52  	zone := framework.TestContext.CloudConfig.Zone
    53  	region := framework.TestContext.CloudConfig.Region
    54  	allowedZones := framework.TestContext.CloudConfig.Zones
    55  
    56  	// ensure users don't specify a zone outside of the requested zones
    57  	if len(zone) > 0 && len(allowedZones) > 0 {
    58  		var found bool
    59  		for _, allowedZone := range allowedZones {
    60  			if zone == allowedZone {
    61  				found = true
    62  				break
    63  			}
    64  		}
    65  		if !found {
    66  			return nil, fmt.Errorf("the provided zone %q must be included in the list of allowed zones %v", zone, allowedZones)
    67  		}
    68  	}
    69  
    70  	var err error
    71  	if region == "" {
    72  		region, err = gcecloud.GetGCERegion(zone)
    73  		if err != nil {
    74  			return nil, fmt.Errorf("error parsing GCE/GKE region from zone %q: %w", zone, err)
    75  		}
    76  	}
    77  	managedZones := []string{} // Manage all zones in the region
    78  	if !framework.TestContext.CloudConfig.MultiZone {
    79  		managedZones = []string{zone}
    80  	}
    81  	if len(allowedZones) > 0 {
    82  		managedZones = allowedZones
    83  	}
    84  
    85  	gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{
    86  		APIEndpoint:        framework.TestContext.CloudConfig.APIEndpoint,
    87  		ProjectID:          framework.TestContext.CloudConfig.ProjectID,
    88  		Region:             region,
    89  		Zone:               zone,
    90  		ManagedZones:       managedZones,
    91  		NetworkName:        "", // TODO: Change this to use framework.TestContext.CloudConfig.Network?
    92  		SubnetworkName:     "",
    93  		NodeTags:           nil,
    94  		NodeInstancePrefix: "",
    95  		TokenSource:        nil,
    96  		UseMetadataServer:  false,
    97  		AlphaFeatureGate:   gcecloud.NewAlphaFeatureGate([]string{}),
    98  	})
    99  
   100  	if err != nil {
   101  		return nil, fmt.Errorf("Error building GCE/GKE provider: %w", err)
   102  	}
   103  
   104  	// Arbitrarily pick one of the zones we have nodes in, looking at prepopulated zones first.
   105  	if framework.TestContext.CloudConfig.Zone == "" && len(managedZones) > 0 {
   106  		framework.TestContext.CloudConfig.Zone = managedZones[rand.Intn(len(managedZones))]
   107  	}
   108  	if framework.TestContext.CloudConfig.Zone == "" && framework.TestContext.CloudConfig.MultiZone {
   109  		zones, err := gceCloud.GetAllZonesFromCloudProvider()
   110  		if err != nil {
   111  			return nil, err
   112  		}
   113  
   114  		framework.TestContext.CloudConfig.Zone, _ = zones.PopAny()
   115  	}
   116  
   117  	return NewProvider(gceCloud), nil
   118  }
   119  
   120  // NewProvider returns a cloud provider interface for GCE
   121  func NewProvider(gceCloud *gcecloud.Cloud) framework.ProviderInterface {
   122  	return &Provider{
   123  		gceCloud: gceCloud,
   124  	}
   125  }
   126  
   127  // Provider is a structure to handle GCE clouds for e2e testing
   128  type Provider struct {
   129  	framework.NullProvider
   130  	gceCloud *gcecloud.Cloud
   131  }
   132  
   133  // ResizeGroup resizes an instance group
   134  func (p *Provider) ResizeGroup(group string, size int32) error {
   135  	// TODO: make this hit the compute API directly instead of shelling out to gcloud.
   136  	// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
   137  	zone, err := getGCEZoneForGroup(group)
   138  	if err != nil {
   139  		return err
   140  	}
   141  	output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "resize",
   142  		group, fmt.Sprintf("--size=%v", size),
   143  		"--project="+framework.TestContext.CloudConfig.ProjectID, "--zone="+zone).CombinedOutput()
   144  	if err != nil {
   145  		return fmt.Errorf("Failed to resize node instance group %s: %s", group, output)
   146  	}
   147  	return nil
   148  }
   149  
   150  // GetGroupNodes returns a node name for the specified node group
   151  func (p *Provider) GetGroupNodes(group string) ([]string, error) {
   152  	// TODO: make this hit the compute API directly instead of shelling out to gcloud.
   153  	// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
   154  	zone, err := getGCEZoneForGroup(group)
   155  	if err != nil {
   156  		return nil, err
   157  	}
   158  	output, err := exec.Command("gcloud", "compute", "instance-groups", "managed",
   159  		"list-instances", group, "--project="+framework.TestContext.CloudConfig.ProjectID,
   160  		"--zone="+zone).CombinedOutput()
   161  	if err != nil {
   162  		return nil, fmt.Errorf("Failed to get nodes in instance group %s: %s", group, output)
   163  	}
   164  	re := regexp.MustCompile(".*RUNNING")
   165  	lines := re.FindAllString(string(output), -1)
   166  	for i, line := range lines {
   167  		lines[i] = line[:strings.Index(line, " ")]
   168  	}
   169  	return lines, nil
   170  }
   171  
   172  // GroupSize returns the size of an instance group
   173  func (p *Provider) GroupSize(group string) (int, error) {
   174  	// TODO: make this hit the compute API directly instead of shelling out to gcloud.
   175  	// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
   176  	zone, err := getGCEZoneForGroup(group)
   177  	if err != nil {
   178  		return -1, err
   179  	}
   180  	output, err := exec.Command("gcloud", "compute", "instance-groups", "managed",
   181  		"list-instances", group, "--project="+framework.TestContext.CloudConfig.ProjectID,
   182  		"--zone="+zone).CombinedOutput()
   183  	if err != nil {
   184  		return -1, fmt.Errorf("Failed to get group size for group %s: %s", group, output)
   185  	}
   186  	re := regexp.MustCompile("RUNNING")
   187  	return len(re.FindAllString(string(output), -1)), nil
   188  }
   189  
   190  // EnsureLoadBalancerResourcesDeleted ensures that cloud load balancer resources that were created
   191  func (p *Provider) EnsureLoadBalancerResourcesDeleted(ctx context.Context, ip, portRange string) error {
   192  	project := framework.TestContext.CloudConfig.ProjectID
   193  	region, err := gcecloud.GetGCERegion(framework.TestContext.CloudConfig.Zone)
   194  	if err != nil {
   195  		return fmt.Errorf("could not get region for zone %q: %w", framework.TestContext.CloudConfig.Zone, err)
   196  	}
   197  
   198  	return wait.PollWithContext(ctx, 10*time.Second, 5*time.Minute, func(ctx context.Context) (bool, error) {
   199  		computeservice := p.gceCloud.ComputeServices().GA
   200  		list, err := computeservice.ForwardingRules.List(project, region).Do()
   201  		if err != nil {
   202  			return false, err
   203  		}
   204  		for _, item := range list.Items {
   205  			if item.PortRange == portRange && item.IPAddress == ip {
   206  				framework.Logf("found a load balancer: %v", item)
   207  				return false, nil
   208  			}
   209  		}
   210  		return true, nil
   211  	})
   212  }
   213  
   214  func getGCEZoneForGroup(group string) (string, error) {
   215  	output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "list",
   216  		"--project="+framework.TestContext.CloudConfig.ProjectID, "--format=value(zone)", "--filter=name="+group).Output()
   217  	if err != nil {
   218  		return "", fmt.Errorf("Failed to get zone for node group %s: %s", group, output)
   219  	}
   220  	return strings.TrimSpace(string(output)), nil
   221  }
   222  
   223  // DeleteNode deletes a node which is specified as the argument
   224  func (p *Provider) DeleteNode(node *v1.Node) error {
   225  	zone := framework.TestContext.CloudConfig.Zone
   226  	project := framework.TestContext.CloudConfig.ProjectID
   227  
   228  	return p.gceCloud.DeleteInstance(project, zone, node.Name)
   229  }
   230  
   231  func (p *Provider) CreateShare() (string, string, string, error) {
   232  	return "", "", "", nil
   233  }
   234  
   235  func (p *Provider) DeleteShare(accountName, shareName string) error {
   236  	return nil
   237  }
   238  
   239  // CreatePD creates a persistent volume
   240  func (p *Provider) CreatePD(zone string) (string, error) {
   241  	pdName := fmt.Sprintf("%s-%s", framework.TestContext.Prefix, string(uuid.NewUUID()))
   242  
   243  	if zone == "" && framework.TestContext.CloudConfig.MultiZone {
   244  		zones, err := p.gceCloud.GetAllZonesFromCloudProvider()
   245  		if err != nil {
   246  			return "", err
   247  		}
   248  		zone, _ = zones.PopAny()
   249  	}
   250  
   251  	tags := map[string]string{}
   252  	if _, err := p.gceCloud.CreateDisk(pdName, gcecloud.DiskTypeStandard, zone, 2 /* sizeGb */, tags); err != nil {
   253  		return "", err
   254  	}
   255  	return pdName, nil
   256  }
   257  
   258  // DeletePD deletes a persistent volume
   259  func (p *Provider) DeletePD(pdName string) error {
   260  	err := p.gceCloud.DeleteDisk(pdName)
   261  
   262  	if err != nil {
   263  		if gerr, ok := err.(*googleapi.Error); ok && len(gerr.Errors) > 0 && gerr.Errors[0].Reason == "notFound" {
   264  			// PD already exists, ignore error.
   265  			return nil
   266  		}
   267  
   268  		framework.Logf("error deleting PD %q: %v", pdName, err)
   269  	}
   270  	return err
   271  }
   272  
   273  // CreatePVSource creates a persistent volume source
   274  func (p *Provider) CreatePVSource(ctx context.Context, zone, diskName string) (*v1.PersistentVolumeSource, error) {
   275  	return &v1.PersistentVolumeSource{
   276  		GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
   277  			PDName:   diskName,
   278  			FSType:   "ext3",
   279  			ReadOnly: false,
   280  		},
   281  	}, nil
   282  }
   283  
   284  // DeletePVSource deletes a persistent volume source
   285  func (p *Provider) DeletePVSource(ctx context.Context, pvSource *v1.PersistentVolumeSource) error {
   286  	return e2epv.DeletePDWithRetry(ctx, pvSource.GCEPersistentDisk.PDName)
   287  }
   288  
   289  // CleanupServiceResources cleans up GCE Service Type=LoadBalancer resources with
   290  // the given name. The name is usually the UUID of the Service prefixed with an
   291  // alpha-numeric character ('a') to work around cloudprovider rules.
   292  func (p *Provider) CleanupServiceResources(ctx context.Context, c clientset.Interface, loadBalancerName, region, zone string) {
   293  	if pollErr := wait.PollWithContext(ctx, 5*time.Second, e2eservice.LoadBalancerCleanupTimeout, func(ctx context.Context) (bool, error) {
   294  		if err := p.cleanupGCEResources(ctx, c, loadBalancerName, region, zone); err != nil {
   295  			framework.Logf("Still waiting for glbc to cleanup: %v", err)
   296  			return false, nil
   297  		}
   298  		return true, nil
   299  	}); pollErr != nil {
   300  		framework.Failf("Failed to cleanup service GCE resources.")
   301  	}
   302  }
   303  
   304  func (p *Provider) cleanupGCEResources(ctx context.Context, c clientset.Interface, loadBalancerName, region, zone string) (retErr error) {
   305  	if region == "" {
   306  		// Attempt to parse region from zone if no region is given.
   307  		var err error
   308  		region, err = gcecloud.GetGCERegion(zone)
   309  		if err != nil {
   310  			return fmt.Errorf("error parsing GCE/GKE region from zone %q: %w", zone, err)
   311  		}
   312  	}
   313  	if err := p.gceCloud.DeleteFirewall(gcecloud.MakeFirewallName(loadBalancerName)); err != nil &&
   314  		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
   315  		retErr = err
   316  	}
   317  	if err := p.gceCloud.DeleteRegionForwardingRule(loadBalancerName, region); err != nil &&
   318  		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
   319  		retErr = fmt.Errorf("%v\n%v", retErr, err)
   320  
   321  	}
   322  	if err := p.gceCloud.DeleteRegionAddress(loadBalancerName, region); err != nil &&
   323  		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
   324  		retErr = fmt.Errorf("%v\n%v", retErr, err)
   325  	}
   326  	clusterID, err := GetClusterID(ctx, c)
   327  	if err != nil {
   328  		retErr = fmt.Errorf("%v\n%v", retErr, err)
   329  		return
   330  	}
   331  	hcNames := []string{gcecloud.MakeNodesHealthCheckName(clusterID)}
   332  	hc, getErr := p.gceCloud.GetHTTPHealthCheck(loadBalancerName)
   333  	if getErr != nil && !IsGoogleAPIHTTPErrorCode(getErr, http.StatusNotFound) {
   334  		retErr = fmt.Errorf("%v\n%v", retErr, getErr)
   335  		return
   336  	}
   337  	if hc != nil {
   338  		hcNames = append(hcNames, hc.Name)
   339  	}
   340  	if err := p.gceCloud.DeleteExternalTargetPoolAndChecks(&v1.Service{}, loadBalancerName, region, clusterID, hcNames...); err != nil &&
   341  		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
   342  		retErr = fmt.Errorf("%v\n%v", retErr, err)
   343  	}
   344  	return
   345  }
   346  
   347  // L4LoadBalancerSrcRanges contains the ranges of ips used by the GCE L4 load
   348  // balancers for proxying client requests and performing health checks.
   349  func (p *Provider) L4LoadBalancerSrcRanges() []string {
   350  	return gcecloud.L4LoadBalancerSrcRanges()
   351  }
   352  
   353  // EnableAndDisableInternalLB returns functions for both enabling and disabling internal Load Balancer
   354  func (p *Provider) EnableAndDisableInternalLB() (enable, disable func(svc *v1.Service)) {
   355  	enable = func(svc *v1.Service) {
   356  		svc.ObjectMeta.Annotations = map[string]string{gcecloud.ServiceAnnotationLoadBalancerType: string(gcecloud.LBTypeInternal)}
   357  	}
   358  	disable = func(svc *v1.Service) {
   359  		delete(svc.ObjectMeta.Annotations, gcecloud.ServiceAnnotationLoadBalancerType)
   360  	}
   361  	return
   362  }
   363  
   364  // GetInstanceTags gets tags from GCE instance with given name.
   365  func GetInstanceTags(cloudConfig framework.CloudConfig, instanceName string) *compute.Tags {
   366  	gceCloud := cloudConfig.Provider.(*Provider).gceCloud
   367  	res, err := gceCloud.ComputeServices().GA.Instances.Get(cloudConfig.ProjectID, cloudConfig.Zone,
   368  		instanceName).Do()
   369  	if err != nil {
   370  		framework.Failf("Failed to get instance tags for %v: %v", instanceName, err)
   371  	}
   372  	return res.Tags
   373  }
   374  
   375  // SetInstanceTags sets tags on GCE instance with given name.
   376  func SetInstanceTags(cloudConfig framework.CloudConfig, instanceName, zone string, tags []string) []string {
   377  	gceCloud := cloudConfig.Provider.(*Provider).gceCloud
   378  	// Re-get instance everytime because we need the latest fingerprint for updating metadata
   379  	resTags := GetInstanceTags(cloudConfig, instanceName)
   380  	_, err := gceCloud.ComputeServices().GA.Instances.SetTags(
   381  		cloudConfig.ProjectID, zone, instanceName,
   382  		&compute.Tags{Fingerprint: resTags.Fingerprint, Items: tags}).Do()
   383  	if err != nil {
   384  		framework.Failf("failed to set instance tags: %v", err)
   385  	}
   386  	framework.Logf("Sent request to set tags %v on instance: %v", tags, instanceName)
   387  	return resTags.Items
   388  }
   389  
   390  // IsGoogleAPIHTTPErrorCode returns true if the error is a google api
   391  // error matching the corresponding HTTP error code.
   392  func IsGoogleAPIHTTPErrorCode(err error, code int) bool {
   393  	apiErr, ok := err.(*googleapi.Error)
   394  	return ok && apiErr.Code == code
   395  }
   396  
   397  // GetGCECloud returns GCE cloud provider
   398  func GetGCECloud() (*gcecloud.Cloud, error) {
   399  	p, ok := framework.TestContext.CloudConfig.Provider.(*Provider)
   400  	if !ok {
   401  		return nil, fmt.Errorf("failed to convert CloudConfig.Provider to GCE provider: %#v", framework.TestContext.CloudConfig.Provider)
   402  	}
   403  	return p.gceCloud, nil
   404  }
   405  
   406  // GetClusterID returns cluster ID
   407  func GetClusterID(ctx context.Context, c clientset.Interface) (string, error) {
   408  	cm, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, gcecloud.UIDConfigMapName, metav1.GetOptions{})
   409  	if err != nil || cm == nil {
   410  		return "", fmt.Errorf("error getting cluster ID: %w", err)
   411  	}
   412  	clusterID, clusterIDExists := cm.Data[gcecloud.UIDCluster]
   413  	providerID, providerIDExists := cm.Data[gcecloud.UIDProvider]
   414  	if !clusterIDExists {
   415  		return "", fmt.Errorf("cluster ID not set")
   416  	}
   417  	if providerIDExists {
   418  		return providerID, nil
   419  	}
   420  	return clusterID, nil
   421  }
   422
View as plain text