ingress.go

Documentation: k8s.io/kubernetes/test/e2e/framework/providers/gce

     1  //go:build !providerless
     2  // +build !providerless
     3  
     4  /*
     5  Copyright 2015 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package gce
    21  
    22  import (
    23  	"context"
    24  	"crypto/sha256"
    25  	"encoding/json"
    26  	"fmt"
    27  	"net/http"
    28  	"os/exec"
    29  	"strconv"
    30  	"strings"
    31  	"time"
    32  
    33  	"github.com/onsi/ginkgo/v2"
    34  	compute "google.golang.org/api/compute/v1"
    35  	"google.golang.org/api/googleapi"
    36  	v1 "k8s.io/api/core/v1"
    37  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    38  	"k8s.io/apimachinery/pkg/util/wait"
    39  	clientset "k8s.io/client-go/kubernetes"
    40  	"k8s.io/kubernetes/test/e2e/framework"
    41  	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
    42  	utilexec "k8s.io/utils/exec"
    43  )
    44  
    45  const (
    46  	// Name of the config-map and key the ingress controller stores its uid in.
    47  	uidConfigMap = "ingress-uid"
    48  	uidKey       = "uid"
    49  
    50  	// all cloud resources created by the ingress controller start with this
    51  	// prefix.
    52  	k8sPrefix = "k8s-"
    53  
    54  	// clusterDelimiter is the delimiter used by the ingress controller
    55  	// to split uid from other naming/metadata.
    56  	clusterDelimiter = "--"
    57  
    58  	// Cloud resources created by the ingress controller older than this
    59  	// are automatically purged to prevent running out of quota.
    60  	// TODO(37335): write soak tests and bump this up to a week.
    61  	maxAge = 48 * time.Hour
    62  
    63  	// GCE only allows names < 64 characters, and the loadbalancer controller inserts
    64  	// a single character of padding.
    65  	nameLenLimit = 62
    66  
    67  	negBackend = backendType("networkEndpointGroup")
    68  	igBackend  = backendType("instanceGroup")
    69  )
    70  
    71  type backendType string
    72  
    73  // IngressController manages implementation details of Ingress on GCE/GKE.
    74  type IngressController struct {
    75  	Ns           string
    76  	UID          string
    77  	staticIPName string
    78  	Client       clientset.Interface
    79  	Cloud        framework.CloudConfig
    80  }
    81  
    82  // CleanupIngressController calls cont.CleanupIngressControllerWithTimeout with hard-coded timeout
    83  func (cont *IngressController) CleanupIngressController(ctx context.Context) error {
    84  	return cont.CleanupIngressControllerWithTimeout(ctx, e2eservice.LoadBalancerCleanupTimeout)
    85  }
    86  
    87  // CleanupIngressControllerWithTimeout calls the IngressController.Cleanup(false)
    88  // followed with deleting the static ip, and then a final IngressController.Cleanup(true)
    89  func (cont *IngressController) CleanupIngressControllerWithTimeout(ctx context.Context, timeout time.Duration) error {
    90  	pollErr := wait.PollWithContext(ctx, 5*time.Second, timeout, func(ctx context.Context) (bool, error) {
    91  		if err := cont.Cleanup(false); err != nil {
    92  			framework.Logf("Monitoring glbc's cleanup of gce resources:\n%v", err)
    93  			return false, nil
    94  		}
    95  		return true, nil
    96  	})
    97  
    98  	// Always try to cleanup even if pollErr == nil, because the cleanup
    99  	// routine also purges old leaked resources based on creation timestamp.
   100  	ginkgo.By("Performing final delete of any remaining resources")
   101  	if cleanupErr := cont.Cleanup(true); cleanupErr != nil {
   102  		ginkgo.By(fmt.Sprintf("WARNING: possibly leaked resources: %v\n", cleanupErr))
   103  	} else {
   104  		ginkgo.By("No resources leaked.")
   105  	}
   106  
   107  	// Static-IP allocated on behalf of the test, never deleted by the
   108  	// controller. Delete this IP only after the controller has had a chance
   109  	// to cleanup or it might interfere with the controller, causing it to
   110  	// throw out confusing events.
   111  	if ipErr := wait.PollWithContext(ctx, 5*time.Second, 1*time.Minute, func(ctx context.Context) (bool, error) {
   112  		if err := cont.deleteStaticIPs(); err != nil {
   113  			framework.Logf("Failed to delete static-ip: %v\n", err)
   114  			return false, nil
   115  		}
   116  		return true, nil
   117  	}); ipErr != nil {
   118  		// If this is a persistent error, the suite will fail when we run out
   119  		// of quota anyway.
   120  		ginkgo.By(fmt.Sprintf("WARNING: possibly leaked static IP: %v\n", ipErr))
   121  	}
   122  
   123  	// Logging that the GLBC failed to cleanup GCE resources on ingress deletion
   124  	// See kubernetes/ingress#431
   125  	if pollErr != nil {
   126  		return fmt.Errorf("error: L7 controller failed to delete all cloud resources on time. %v", pollErr)
   127  	}
   128  	return nil
   129  }
   130  
   131  func (cont *IngressController) getL7AddonUID(ctx context.Context) (string, error) {
   132  	framework.Logf("Retrieving UID from config map: %v/%v", metav1.NamespaceSystem, uidConfigMap)
   133  	cm, err := cont.Client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, uidConfigMap, metav1.GetOptions{})
   134  	if err != nil {
   135  		return "", err
   136  	}
   137  	if uid, ok := cm.Data[uidKey]; ok {
   138  		return uid, nil
   139  	}
   140  	return "", fmt.Errorf("Could not find cluster UID for L7 addon pod")
   141  }
   142  
   143  func (cont *IngressController) deleteForwardingRule(del bool) string {
   144  	msg := ""
   145  	fwList := []compute.ForwardingRule{}
   146  	for _, regex := range []string{fmt.Sprintf("%vfw-.*%v.*", k8sPrefix, clusterDelimiter), fmt.Sprintf("%vfws-.*%v.*", k8sPrefix, clusterDelimiter)} {
   147  		gcloudComputeResourceList("forwarding-rules", regex, cont.Cloud.ProjectID, &fwList)
   148  		if len(fwList) == 0 {
   149  			continue
   150  		}
   151  		for _, f := range fwList {
   152  			if !cont.canDelete(f.Name, f.CreationTimestamp, del) {
   153  				continue
   154  			}
   155  			if del {
   156  				GcloudComputeResourceDelete("forwarding-rules", f.Name, cont.Cloud.ProjectID, "--global")
   157  			} else {
   158  				msg += fmt.Sprintf("%v (forwarding rule)\n", f.Name)
   159  			}
   160  		}
   161  	}
   162  	return msg
   163  }
   164  
   165  func (cont *IngressController) deleteAddresses(del bool) string {
   166  	msg := ""
   167  	ipList := []compute.Address{}
   168  	regex := fmt.Sprintf("%vfw-.*%v.*", k8sPrefix, clusterDelimiter)
   169  	gcloudComputeResourceList("addresses", regex, cont.Cloud.ProjectID, &ipList)
   170  	if len(ipList) != 0 {
   171  		for _, ip := range ipList {
   172  			if !cont.canDelete(ip.Name, ip.CreationTimestamp, del) {
   173  				continue
   174  			}
   175  			if del {
   176  				GcloudComputeResourceDelete("addresses", ip.Name, cont.Cloud.ProjectID, "--global")
   177  			} else {
   178  				msg += fmt.Sprintf("%v (static-ip)\n", ip.Name)
   179  			}
   180  		}
   181  	}
   182  	return msg
   183  }
   184  
   185  func (cont *IngressController) deleteTargetProxy(del bool) string {
   186  	msg := ""
   187  	tpList := []compute.TargetHttpProxy{}
   188  	regex := fmt.Sprintf("%vtp-.*%v.*", k8sPrefix, clusterDelimiter)
   189  	gcloudComputeResourceList("target-http-proxies", regex, cont.Cloud.ProjectID, &tpList)
   190  	if len(tpList) != 0 {
   191  		for _, t := range tpList {
   192  			if !cont.canDelete(t.Name, t.CreationTimestamp, del) {
   193  				continue
   194  			}
   195  			if del {
   196  				GcloudComputeResourceDelete("target-http-proxies", t.Name, cont.Cloud.ProjectID)
   197  			} else {
   198  				msg += fmt.Sprintf("%v (target-http-proxy)\n", t.Name)
   199  			}
   200  		}
   201  	}
   202  	tpsList := []compute.TargetHttpsProxy{}
   203  	regex = fmt.Sprintf("%vtps-.*%v.*", k8sPrefix, clusterDelimiter)
   204  	gcloudComputeResourceList("target-https-proxies", regex, cont.Cloud.ProjectID, &tpsList)
   205  	if len(tpsList) != 0 {
   206  		for _, t := range tpsList {
   207  			if !cont.canDelete(t.Name, t.CreationTimestamp, del) {
   208  				continue
   209  			}
   210  			if del {
   211  				GcloudComputeResourceDelete("target-https-proxies", t.Name, cont.Cloud.ProjectID)
   212  			} else {
   213  				msg += fmt.Sprintf("%v (target-https-proxy)\n", t.Name)
   214  			}
   215  		}
   216  	}
   217  	return msg
   218  }
   219  
   220  func (cont *IngressController) deleteURLMap(del bool) (msg string) {
   221  	gceCloud := cont.Cloud.Provider.(*Provider).gceCloud
   222  	umList, err := gceCloud.ListURLMaps()
   223  	if err != nil {
   224  		if cont.isHTTPErrorCode(err, http.StatusNotFound) {
   225  			return msg
   226  		}
   227  		return fmt.Sprintf("Failed to list url maps: %v", err)
   228  	}
   229  	if len(umList) == 0 {
   230  		return msg
   231  	}
   232  	for _, um := range umList {
   233  		if !cont.canDelete(um.Name, um.CreationTimestamp, del) {
   234  			continue
   235  		}
   236  		if del {
   237  			framework.Logf("Deleting url-map: %s", um.Name)
   238  			if err := gceCloud.DeleteURLMap(um.Name); err != nil &&
   239  				!cont.isHTTPErrorCode(err, http.StatusNotFound) {
   240  				msg += fmt.Sprintf("Failed to delete url map %v\n", um.Name)
   241  			}
   242  		} else {
   243  			msg += fmt.Sprintf("%v (url-map)\n", um.Name)
   244  		}
   245  	}
   246  	return msg
   247  }
   248  
   249  func (cont *IngressController) deleteBackendService(del bool) (msg string) {
   250  	gceCloud := cont.Cloud.Provider.(*Provider).gceCloud
   251  	beList, err := gceCloud.ListGlobalBackendServices()
   252  	if err != nil {
   253  		if cont.isHTTPErrorCode(err, http.StatusNotFound) {
   254  			return msg
   255  		}
   256  		return fmt.Sprintf("Failed to list backend services: %v", err)
   257  	}
   258  	if len(beList) == 0 {
   259  		framework.Logf("No backend services found")
   260  		return msg
   261  	}
   262  	for _, be := range beList {
   263  		if !cont.canDelete(be.Name, be.CreationTimestamp, del) {
   264  			continue
   265  		}
   266  		if del {
   267  			framework.Logf("Deleting backed-service: %s", be.Name)
   268  			if err := gceCloud.DeleteGlobalBackendService(be.Name); err != nil &&
   269  				!cont.isHTTPErrorCode(err, http.StatusNotFound) {
   270  				msg += fmt.Sprintf("Failed to delete backend service %v: %v\n", be.Name, err)
   271  			}
   272  		} else {
   273  			msg += fmt.Sprintf("%v (backend-service)\n", be.Name)
   274  		}
   275  	}
   276  	return msg
   277  }
   278  
   279  func (cont *IngressController) deleteHTTPHealthCheck(del bool) (msg string) {
   280  	gceCloud := cont.Cloud.Provider.(*Provider).gceCloud
   281  	hcList, err := gceCloud.ListHTTPHealthChecks()
   282  	if err != nil {
   283  		if cont.isHTTPErrorCode(err, http.StatusNotFound) {
   284  			return msg
   285  		}
   286  		return fmt.Sprintf("Failed to list HTTP health checks: %v", err)
   287  	}
   288  	if len(hcList) == 0 {
   289  		return msg
   290  	}
   291  	for _, hc := range hcList {
   292  		if !cont.canDelete(hc.Name, hc.CreationTimestamp, del) {
   293  			continue
   294  		}
   295  		if del {
   296  			framework.Logf("Deleting http-health-check: %s", hc.Name)
   297  			if err := gceCloud.DeleteHTTPHealthCheck(hc.Name); err != nil &&
   298  				!cont.isHTTPErrorCode(err, http.StatusNotFound) {
   299  				msg += fmt.Sprintf("Failed to delete HTTP health check %v\n", hc.Name)
   300  			}
   301  		} else {
   302  			msg += fmt.Sprintf("%v (http-health-check)\n", hc.Name)
   303  		}
   304  	}
   305  	return msg
   306  }
   307  
   308  func (cont *IngressController) deleteSSLCertificate(del bool) (msg string) {
   309  	gceCloud := cont.Cloud.Provider.(*Provider).gceCloud
   310  	sslList, err := gceCloud.ListSslCertificates()
   311  	if err != nil {
   312  		if cont.isHTTPErrorCode(err, http.StatusNotFound) {
   313  			return msg
   314  		}
   315  		return fmt.Sprintf("Failed to list ssl certificates: %v", err)
   316  	}
   317  	if len(sslList) != 0 {
   318  		for _, s := range sslList {
   319  			if !cont.canDelete(s.Name, s.CreationTimestamp, del) {
   320  				continue
   321  			}
   322  			if del {
   323  				framework.Logf("Deleting ssl-certificate: %s", s.Name)
   324  				if err := gceCloud.DeleteSslCertificate(s.Name); err != nil &&
   325  					!cont.isHTTPErrorCode(err, http.StatusNotFound) {
   326  					msg += fmt.Sprintf("Failed to delete ssl certificates: %v\n", s.Name)
   327  				}
   328  			} else {
   329  				msg += fmt.Sprintf("%v (ssl-certificate)\n", s.Name)
   330  			}
   331  		}
   332  	}
   333  	return msg
   334  }
   335  
   336  func (cont *IngressController) deleteInstanceGroup(del bool) (msg string) {
   337  	gceCloud := cont.Cloud.Provider.(*Provider).gceCloud
   338  	// TODO: E2E cloudprovider has only 1 zone, but the cluster can have many.
   339  	// We need to poll on all IGs across all zones.
   340  	igList, err := gceCloud.ListInstanceGroups(cont.Cloud.Zone)
   341  	if err != nil {
   342  		if cont.isHTTPErrorCode(err, http.StatusNotFound) {
   343  			return msg
   344  		}
   345  		return fmt.Sprintf("Failed to list instance groups: %v", err)
   346  	}
   347  	if len(igList) == 0 {
   348  		return msg
   349  	}
   350  	for _, ig := range igList {
   351  		if !cont.canDelete(ig.Name, ig.CreationTimestamp, del) {
   352  			continue
   353  		}
   354  		if del {
   355  			framework.Logf("Deleting instance-group: %s", ig.Name)
   356  			if err := gceCloud.DeleteInstanceGroup(ig.Name, cont.Cloud.Zone); err != nil &&
   357  				!cont.isHTTPErrorCode(err, http.StatusNotFound) {
   358  				msg += fmt.Sprintf("Failed to delete instance group %v\n", ig.Name)
   359  			}
   360  		} else {
   361  			msg += fmt.Sprintf("%v (instance-group)\n", ig.Name)
   362  		}
   363  	}
   364  	return msg
   365  }
   366  
   367  func (cont *IngressController) deleteNetworkEndpointGroup(del bool) (msg string) {
   368  	gceCloud := cont.Cloud.Provider.(*Provider).gceCloud
   369  	// TODO: E2E cloudprovider has only 1 zone, but the cluster can have many.
   370  	// We need to poll on all NEGs across all zones.
   371  	negList, err := gceCloud.ListNetworkEndpointGroup(cont.Cloud.Zone)
   372  	if err != nil {
   373  		if cont.isHTTPErrorCode(err, http.StatusNotFound) {
   374  			return msg
   375  		}
   376  		// Do not return error as NEG is still alpha.
   377  		framework.Logf("Failed to list network endpoint group: %v", err)
   378  		return msg
   379  	}
   380  	if len(negList) == 0 {
   381  		return msg
   382  	}
   383  	for _, neg := range negList {
   384  		if !cont.canDeleteNEG(neg.Name, neg.CreationTimestamp, del) {
   385  			continue
   386  		}
   387  		if del {
   388  			framework.Logf("Deleting network-endpoint-group: %s", neg.Name)
   389  			if err := gceCloud.DeleteNetworkEndpointGroup(neg.Name, cont.Cloud.Zone); err != nil &&
   390  				!cont.isHTTPErrorCode(err, http.StatusNotFound) {
   391  				msg += fmt.Sprintf("Failed to delete network endpoint group %v\n", neg.Name)
   392  			}
   393  		} else {
   394  			msg += fmt.Sprintf("%v (network-endpoint-group)\n", neg.Name)
   395  		}
   396  	}
   397  	return msg
   398  }
   399  
   400  // canDelete returns true if either the name ends in a suffix matching this
   401  // controller's UID, or the creationTimestamp exceeds the maxAge and del is set
   402  // to true. Always returns false if the name doesn't match that we expect for
   403  // Ingress cloud resources.
   404  func (cont *IngressController) canDelete(resourceName, creationTimestamp string, delOldResources bool) bool {
   405  	// ignore everything not created by an ingress controller.
   406  	splitName := strings.Split(resourceName, clusterDelimiter)
   407  	if !strings.HasPrefix(resourceName, k8sPrefix) || len(splitName) != 2 {
   408  		return false
   409  	}
   410  
   411  	// Resources created by the GLBC have a "0"" appended to the end if truncation
   412  	// occurred. Removing the zero allows the following match.
   413  	truncatedClusterUID := splitName[1]
   414  	if len(truncatedClusterUID) >= 1 && strings.HasSuffix(truncatedClusterUID, "0") {
   415  		truncatedClusterUID = truncatedClusterUID[:len(truncatedClusterUID)-1]
   416  	}
   417  
   418  	// always delete things that are created by the current ingress controller.
   419  	// Because of resource name truncation, this looks for a common prefix
   420  	if strings.HasPrefix(cont.UID, truncatedClusterUID) {
   421  		return true
   422  	}
   423  	if !delOldResources {
   424  		return false
   425  	}
   426  	return canDeleteWithTimestamp(resourceName, creationTimestamp)
   427  }
   428  
   429  // canDeleteNEG returns true if either the name contains this controller's UID,
   430  // or the creationTimestamp exceeds the maxAge and del is set to true.
   431  func (cont *IngressController) canDeleteNEG(resourceName, creationTimestamp string, delOldResources bool) bool {
   432  	if !strings.HasPrefix(resourceName, "k8s") {
   433  		return false
   434  	}
   435  
   436  	if strings.Contains(resourceName, cont.UID) {
   437  		return true
   438  	}
   439  
   440  	if !delOldResources {
   441  		return false
   442  	}
   443  
   444  	return canDeleteWithTimestamp(resourceName, creationTimestamp)
   445  }
   446  
   447  func canDeleteWithTimestamp(resourceName, creationTimestamp string) bool {
   448  	createdTime, err := time.Parse(time.RFC3339, creationTimestamp)
   449  	if err != nil {
   450  		framework.Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err)
   451  		return false
   452  	}
   453  	if time.Since(createdTime) > maxAge {
   454  		framework.Logf("%v created on %v IS too old", resourceName, creationTimestamp)
   455  		return true
   456  	}
   457  	return false
   458  }
   459  
   460  func (cont *IngressController) deleteFirewallRule(del bool) (msg string) {
   461  	fwList := []compute.Firewall{}
   462  	regex := fmt.Sprintf("%vfw-l7%v.*", k8sPrefix, clusterDelimiter)
   463  	gcloudComputeResourceList("firewall-rules", regex, cont.Cloud.ProjectID, &fwList)
   464  	if len(fwList) != 0 {
   465  		for _, f := range fwList {
   466  			if !cont.canDelete(f.Name, f.CreationTimestamp, del) {
   467  				continue
   468  			}
   469  			if del {
   470  				GcloudComputeResourceDelete("firewall-rules", f.Name, cont.Cloud.ProjectID)
   471  			} else {
   472  				msg += fmt.Sprintf("%v (firewall rule)\n", f.Name)
   473  			}
   474  		}
   475  	}
   476  	return msg
   477  }
   478  
   479  func (cont *IngressController) isHTTPErrorCode(err error, code int) bool {
   480  	apiErr, ok := err.(*googleapi.Error)
   481  	return ok && apiErr.Code == code
   482  }
   483  
   484  // WaitForNegBackendService waits for the expected backend service to become
   485  func (cont *IngressController) WaitForNegBackendService(ctx context.Context, svcPorts map[string]v1.ServicePort) error {
   486  	return wait.PollWithContext(ctx, 5*time.Second, 1*time.Minute, func(ctx context.Context) (bool, error) {
   487  		err := cont.verifyBackendMode(svcPorts, negBackend)
   488  		if err != nil {
   489  			framework.Logf("Err while checking if backend service is using NEG: %v", err)
   490  			return false, nil
   491  		}
   492  		return true, nil
   493  	})
   494  }
   495  
   496  // BackendServiceUsingNEG returns true only if all global backend service with matching svcPorts pointing to NEG as backend
   497  func (cont *IngressController) BackendServiceUsingNEG(svcPorts map[string]v1.ServicePort) error {
   498  	return cont.verifyBackendMode(svcPorts, negBackend)
   499  }
   500  
   501  // BackendServiceUsingIG returns true only if all global backend service with matching svcPorts pointing to IG as backend
   502  func (cont *IngressController) BackendServiceUsingIG(svcPorts map[string]v1.ServicePort) error {
   503  	return cont.verifyBackendMode(svcPorts, igBackend)
   504  }
   505  
   506  func (cont *IngressController) verifyBackendMode(svcPorts map[string]v1.ServicePort, backendType backendType) error {
   507  	gceCloud := cont.Cloud.Provider.(*Provider).gceCloud
   508  	beList, err := gceCloud.ListGlobalBackendServices()
   509  	if err != nil {
   510  		return fmt.Errorf("failed to list backend services: %w", err)
   511  	}
   512  
   513  	hcList, err := gceCloud.ListHealthChecks()
   514  	if err != nil {
   515  		return fmt.Errorf("failed to list health checks: %w", err)
   516  	}
   517  
   518  	// Generate short UID
   519  	uid := cont.UID
   520  	if len(uid) > 8 {
   521  		uid = uid[:8]
   522  	}
   523  
   524  	matchingBackendService := 0
   525  	for svcName, sp := range svcPorts {
   526  		match := false
   527  		bsMatch := &compute.BackendService{}
   528  		// NEG BackendServices' names contain the a sha256 hash of a string.
   529  		// This logic is copied from the ingress-gce namer.
   530  		// WARNING: This needs to adapt if the naming convention changed.
   531  		negString := strings.Join([]string{uid, cont.Ns, svcName, fmt.Sprintf("%v", sp.Port)}, ";")
   532  		negHash := fmt.Sprintf("%x", sha256.Sum256([]byte(negString)))[:8]
   533  		for _, bs := range beList {
   534  			// Non-NEG BackendServices are named with the Nodeport in the name.
   535  			if backendType == igBackend && strings.Contains(bs.Name, strconv.Itoa(int(sp.NodePort))) {
   536  				match = true
   537  				bsMatch = bs
   538  				matchingBackendService++
   539  				break
   540  			}
   541  
   542  			// NEG BackendServices' names contain the a sha256 hash of a string.
   543  			if backendType == negBackend && strings.Contains(bs.Name, negHash) {
   544  				match = true
   545  				bsMatch = bs
   546  				matchingBackendService++
   547  				break
   548  			}
   549  		}
   550  
   551  		if match {
   552  			for _, be := range bsMatch.Backends {
   553  				if !strings.Contains(be.Group, string(backendType)) {
   554  					return fmt.Errorf("expect to find backends with type %q, but got backend group: %v", backendType, be.Group)
   555  				}
   556  			}
   557  
   558  			// Check that the correct HealthCheck exists for the BackendService
   559  			hcMatch := false
   560  			for _, hc := range hcList {
   561  				if hc.Name == bsMatch.Name {
   562  					hcMatch = true
   563  					break
   564  				}
   565  			}
   566  
   567  			if !hcMatch {
   568  				return fmt.Errorf("missing healthcheck for backendservice: %v", bsMatch.Name)
   569  			}
   570  		}
   571  	}
   572  
   573  	if matchingBackendService != len(svcPorts) {
   574  		beNames := []string{}
   575  		for _, be := range beList {
   576  			beNames = append(beNames, be.Name)
   577  		}
   578  		return fmt.Errorf("expect %d backend service with backend type: %v, but got %d matching backend service. Expect backend services for service ports: %v, but got backend services: %v", len(svcPorts), backendType, matchingBackendService, svcPorts, beNames)
   579  	}
   580  
   581  	return nil
   582  }
   583  
   584  // Cleanup cleans up cloud resources.
   585  // If del is false, it simply reports existing resources without deleting them.
   586  // If dle is true, it deletes resources it finds acceptable (see canDelete func).
   587  func (cont *IngressController) Cleanup(del bool) error {
   588  	// Ordering is important here because we cannot delete resources that other
   589  	// resources hold references to.
   590  	errMsg := cont.deleteForwardingRule(del)
   591  	// Static IPs are named after forwarding rules.
   592  	errMsg += cont.deleteAddresses(del)
   593  
   594  	errMsg += cont.deleteTargetProxy(del)
   595  	errMsg += cont.deleteURLMap(del)
   596  	errMsg += cont.deleteBackendService(del)
   597  	errMsg += cont.deleteHTTPHealthCheck(del)
   598  
   599  	errMsg += cont.deleteInstanceGroup(del)
   600  	errMsg += cont.deleteNetworkEndpointGroup(del)
   601  	errMsg += cont.deleteFirewallRule(del)
   602  	errMsg += cont.deleteSSLCertificate(del)
   603  
   604  	// TODO: Verify instance-groups, issue #16636. Gcloud mysteriously barfs when told
   605  	// to unmarshal instance groups into the current vendored gce-client's understanding
   606  	// of the struct.
   607  	if errMsg == "" {
   608  		return nil
   609  	}
   610  	return fmt.Errorf(errMsg)
   611  }
   612  
   613  // Init initializes the IngressController with an UID
   614  func (cont *IngressController) Init(ctx context.Context) error {
   615  	uid, err := cont.getL7AddonUID(ctx)
   616  	if err != nil {
   617  		return err
   618  	}
   619  	cont.UID = uid
   620  	// There's a name limit imposed by GCE. The controller will truncate.
   621  	testName := fmt.Sprintf("k8s-fw-foo-app-X-%v--%v", cont.Ns, cont.UID)
   622  	if len(testName) > nameLenLimit {
   623  		framework.Logf("WARNING: test name including cluster UID: %v is over the GCE limit of %v", testName, nameLenLimit)
   624  	} else {
   625  		framework.Logf("Detected cluster UID %v", cont.UID)
   626  	}
   627  	return nil
   628  }
   629  
   630  // deleteStaticIPs deletes all static-ips allocated through calls to
   631  // CreateStaticIP.
   632  func (cont *IngressController) deleteStaticIPs() error {
   633  	if cont.staticIPName != "" {
   634  		if err := GcloudComputeResourceDelete("addresses", cont.staticIPName, cont.Cloud.ProjectID, "--global"); err == nil {
   635  			cont.staticIPName = ""
   636  		} else {
   637  			return err
   638  		}
   639  	} else {
   640  		e2eIPs := []compute.Address{}
   641  		gcloudComputeResourceList("addresses", "e2e-.*", cont.Cloud.ProjectID, &e2eIPs)
   642  		ips := []string{}
   643  		for _, ip := range e2eIPs {
   644  			ips = append(ips, ip.Name)
   645  		}
   646  		framework.Logf("None of the remaining %d static-ips were created by this e2e: %v", len(ips), strings.Join(ips, ", "))
   647  	}
   648  	return nil
   649  }
   650  
   651  // gcloudComputeResourceList unmarshals json output of gcloud into given out interface.
   652  func gcloudComputeResourceList(resource, regex, project string, out interface{}) {
   653  	// gcloud prints a message to stderr if it has an available update
   654  	// so we only look at stdout.
   655  	command := []string{
   656  		"compute", resource, "list",
   657  		fmt.Sprintf("--filter='name ~ \"%q\"'", regex),
   658  		fmt.Sprintf("--project=%v", project),
   659  		"-q", "--format=json",
   660  	}
   661  	output, err := exec.Command("gcloud", command...).Output()
   662  	if err != nil {
   663  		errCode := -1
   664  		errMsg := ""
   665  		if exitErr, ok := err.(utilexec.ExitError); ok {
   666  			errCode = exitErr.ExitStatus()
   667  			errMsg = exitErr.Error()
   668  			if osExitErr, ok := err.(*exec.ExitError); ok {
   669  				errMsg = fmt.Sprintf("%v, stderr %v", errMsg, string(osExitErr.Stderr))
   670  			}
   671  		}
   672  		framework.Logf("Error running gcloud command 'gcloud %s': err: %v, output: %v, status: %d, msg: %v", strings.Join(command, " "), err, string(output), errCode, errMsg)
   673  	}
   674  	if err := json.Unmarshal([]byte(output), out); err != nil {
   675  		framework.Logf("Error unmarshalling gcloud output for %v: %v, output: %v", resource, err, string(output))
   676  	}
   677  }
   678  
   679  // GcloudComputeResourceDelete deletes the specified compute resource by name and project.
   680  func GcloudComputeResourceDelete(resource, name, project string, args ...string) error {
   681  	framework.Logf("Deleting %v: %v", resource, name)
   682  	argList := append([]string{"compute", resource, "delete", name, fmt.Sprintf("--project=%v", project), "-q"}, args...)
   683  	output, err := exec.Command("gcloud", argList...).CombinedOutput()
   684  	if err != nil {
   685  		framework.Logf("Error deleting %v, output: %v\nerror: %+v", resource, string(output), err)
   686  	}
   687  	return err
   688  }
   689
View as plain text