network_tiers.go

Documentation: k8s.io/kubernetes/test/e2e/network

     1  //go:build !providerless
     2  // +build !providerless
     3  
     4  /*
     5  Copyright 2017 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package network
    21  
    22  import (
    23  	"context"
    24  	"fmt"
    25  	"net/http"
    26  	"time"
    27  
    28  	compute "google.golang.org/api/compute/v1"
    29  
    30  	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
    31  	v1 "k8s.io/api/core/v1"
    32  	"k8s.io/apimachinery/pkg/util/wait"
    33  	clientset "k8s.io/client-go/kubernetes"
    34  	cloudprovider "k8s.io/cloud-provider"
    35  	"k8s.io/kubernetes/test/e2e/framework"
    36  	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
    37  	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
    38  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    39  	"k8s.io/kubernetes/test/e2e/network/common"
    40  	gcecloud "k8s.io/legacy-cloud-providers/gce"
    41  	admissionapi "k8s.io/pod-security-admission/api"
    42  
    43  	"github.com/onsi/ginkgo/v2"
    44  	"github.com/onsi/gomega"
    45  )
    46  
    47  var _ = common.SIGDescribe("Services GCE", framework.WithSlow(), func() {
    48  	f := framework.NewDefaultFramework("services")
    49  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    50  
    51  	var cs clientset.Interface
    52  	serviceLBNames := []string{}
    53  
    54  	ginkgo.BeforeEach(func() {
    55  		// This test suite requires the GCE environment.
    56  		e2eskipper.SkipUnlessProviderIs("gce")
    57  		cs = f.ClientSet
    58  	})
    59  
    60  	ginkgo.AfterEach(func(ctx context.Context) {
    61  		if ginkgo.CurrentSpecReport().Failed() {
    62  			DescribeSvc(f.Namespace.Name)
    63  		}
    64  		for _, lb := range serviceLBNames {
    65  			framework.Logf("cleaning gce resource for %s", lb)
    66  			framework.TestContext.CloudConfig.Provider.CleanupServiceResources(ctx, cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone)
    67  		}
    68  		//reset serviceLBNames
    69  		serviceLBNames = []string{}
    70  	})
    71  	f.It("should be able to create and tear down a standard-tier load balancer", f.WithSlow(), func(ctx context.Context) {
    72  		lagTimeout := e2eservice.LoadBalancerLagTimeoutDefault
    73  		createTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, cs)
    74  
    75  		svcName := "net-tiers-svc"
    76  		ns := f.Namespace.Name
    77  		jig := e2eservice.NewTestJig(cs, ns, svcName)
    78  
    79  		ginkgo.By("creating a pod to be part of the service " + svcName)
    80  		_, err := jig.Run(ctx, nil)
    81  		framework.ExpectNoError(err)
    82  
    83  		// Test 1: create a standard tiered LB for the Service.
    84  		ginkgo.By("creating a Service of type LoadBalancer using the standard network tier")
    85  		svc, err := jig.CreateTCPService(ctx, func(svc *v1.Service) {
    86  			svc.Spec.Type = v1.ServiceTypeLoadBalancer
    87  			setNetworkTier(svc, string(gcecloud.NetworkTierAnnotationStandard))
    88  		})
    89  		framework.ExpectNoError(err)
    90  		// Verify that service has been updated properly.
    91  		svcTier, err := gcecloud.GetServiceNetworkTier(svc)
    92  		framework.ExpectNoError(err)
    93  		gomega.Expect(svcTier).To(gomega.Equal(cloud.NetworkTierStandard))
    94  		// Record the LB name for test cleanup.
    95  		serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
    96  
    97  		// Wait and verify the LB.
    98  		ingressIP := waitAndVerifyLBWithTier(ctx, jig, "", createTimeout, lagTimeout)
    99  
   100  		// Test 2: re-create a LB of a different tier for the updated Service.
   101  		ginkgo.By("updating the Service to use the premium (default) tier")
   102  		svc, err = jig.UpdateService(ctx, func(svc *v1.Service) {
   103  			setNetworkTier(svc, string(gcecloud.NetworkTierAnnotationPremium))
   104  		})
   105  		framework.ExpectNoError(err)
   106  		// Verify that service has been updated properly.
   107  		svcTier, err = gcecloud.GetServiceNetworkTier(svc)
   108  		framework.ExpectNoError(err)
   109  		gomega.Expect(svcTier).To(gomega.Equal(cloud.NetworkTierDefault))
   110  
   111  		// Wait until the ingress IP changes. Each tier has its own pool of
   112  		// IPs, so changing tiers implies changing IPs.
   113  		ingressIP = waitAndVerifyLBWithTier(ctx, jig, ingressIP, createTimeout, lagTimeout)
   114  
   115  		// Test 3: create a standard-tierd LB with a user-requested IP.
   116  		ginkgo.By("reserving a static IP for the load balancer")
   117  		requestedAddrName := fmt.Sprintf("e2e-ext-lb-net-tier-%s", framework.RunID)
   118  		gceCloud, err := gce.GetGCECloud()
   119  		framework.ExpectNoError(err)
   120  		requestedIP, err := reserveRegionalAddress(gceCloud, requestedAddrName, cloud.NetworkTierStandard)
   121  		framework.ExpectNoError(err, "failed to reserve a STANDARD tiered address")
   122  		defer func() {
   123  			if requestedAddrName != "" {
   124  				// Release GCE static address - this is not kube-managed and will not be automatically released.
   125  				if err := gceCloud.DeleteRegionAddress(requestedAddrName, gceCloud.Region()); err != nil {
   126  					framework.Logf("failed to release static IP address %q: %v", requestedAddrName, err)
   127  				}
   128  			}
   129  		}()
   130  		framework.ExpectNoError(err)
   131  		framework.Logf("Allocated static IP to be used by the load balancer: %q", requestedIP)
   132  
   133  		ginkgo.By("updating the Service to use the standard tier with a requested IP")
   134  		svc, err = jig.UpdateService(ctx, func(svc *v1.Service) {
   135  			svc.Spec.LoadBalancerIP = requestedIP
   136  			setNetworkTier(svc, string(gcecloud.NetworkTierAnnotationStandard))
   137  		})
   138  		framework.ExpectNoError(err)
   139  		// Verify that service has been updated properly.
   140  		gomega.Expect(svc.Spec.LoadBalancerIP).To(gomega.Equal(requestedIP))
   141  		svcTier, err = gcecloud.GetServiceNetworkTier(svc)
   142  		framework.ExpectNoError(err)
   143  		gomega.Expect(svcTier).To(gomega.Equal(cloud.NetworkTierStandard))
   144  
   145  		// Wait until the ingress IP changes and verifies the LB.
   146  		waitAndVerifyLBWithTier(ctx, jig, ingressIP, createTimeout, lagTimeout)
   147  	})
   148  })
   149  
   150  func waitAndVerifyLBWithTier(ctx context.Context, jig *e2eservice.TestJig, existingIP string, waitTimeout, checkTimeout time.Duration) string {
   151  	// If existingIP is "" this will wait for any ingress IP to show up. Otherwise
   152  	// it will wait for the ingress IP to change to something different.
   153  	svc, err := jig.WaitForNewIngressIP(ctx, existingIP, waitTimeout)
   154  	framework.ExpectNoError(err)
   155  
   156  	svcPort := int(svc.Spec.Ports[0].Port)
   157  	lbIngress := &svc.Status.LoadBalancer.Ingress[0]
   158  	ingressIP := e2eservice.GetIngressPoint(lbIngress)
   159  
   160  	ginkgo.By("running sanity and reachability checks")
   161  	if svc.Spec.LoadBalancerIP != "" {
   162  		// Verify that the new ingress IP is the requested IP if it's set.
   163  		gomega.Expect(ingressIP).To(gomega.Equal(svc.Spec.LoadBalancerIP))
   164  	}
   165  	// If the IP has been used by previous test, sometimes we get the lingering
   166  	// 404 errors even after the LB is long gone. Tolerate and retry until the
   167  	// new LB is fully established.
   168  	e2eservice.TestReachableHTTPWithRetriableErrorCodes(ctx, ingressIP, svcPort, []int{http.StatusNotFound}, checkTimeout)
   169  
   170  	// Verify the network tier matches the desired.
   171  	svcNetTier, err := gcecloud.GetServiceNetworkTier(svc)
   172  	framework.ExpectNoError(err)
   173  	netTier, err := getLBNetworkTierByIP(ingressIP)
   174  	framework.ExpectNoError(err, "failed to get the network tier of the load balancer")
   175  	gomega.Expect(netTier).To(gomega.Equal(svcNetTier))
   176  
   177  	return ingressIP
   178  }
   179  
   180  func getLBNetworkTierByIP(ip string) (cloud.NetworkTier, error) {
   181  	var rule *compute.ForwardingRule
   182  	// Retry a few times to tolerate flakes.
   183  	err := wait.PollImmediate(5*time.Second, 15*time.Second, func() (bool, error) {
   184  		obj, err := getGCEForwardingRuleByIP(ip)
   185  		if err != nil {
   186  			return false, err
   187  		}
   188  		rule = obj
   189  		return true, nil
   190  	})
   191  	if err != nil {
   192  		return "", err
   193  	}
   194  	return cloud.NetworkTierGCEValueToType(rule.NetworkTier), nil
   195  }
   196  
   197  func getGCEForwardingRuleByIP(ip string) (*compute.ForwardingRule, error) {
   198  	cloud, err := gce.GetGCECloud()
   199  	if err != nil {
   200  		return nil, err
   201  	}
   202  	ruleList, err := cloud.ListRegionForwardingRules(cloud.Region())
   203  	if err != nil {
   204  		return nil, err
   205  	}
   206  	for _, rule := range ruleList {
   207  		if rule.IPAddress == ip {
   208  			return rule, nil
   209  		}
   210  	}
   211  	return nil, fmt.Errorf("forwarding rule with ip %q not found", ip)
   212  }
   213  
   214  func setNetworkTier(svc *v1.Service, tier string) {
   215  	key := gcecloud.NetworkTierAnnotationKey
   216  	if svc.ObjectMeta.Annotations == nil {
   217  		svc.ObjectMeta.Annotations = map[string]string{}
   218  	}
   219  	svc.ObjectMeta.Annotations[key] = tier
   220  }
   221  
   222  // TODO: add retries if this turns out to be flaky.
   223  func reserveRegionalAddress(cloud *gcecloud.Cloud, name string, netTier cloud.NetworkTier) (string, error) {
   224  	Addr := &compute.Address{
   225  		Name:        name,
   226  		NetworkTier: netTier.ToGCEValue(),
   227  	}
   228  
   229  	if err := cloud.ReserveRegionAddress(Addr, cloud.Region()); err != nil {
   230  		return "", err
   231  	}
   232  
   233  	addr, err := cloud.GetRegionAddress(name, cloud.Region())
   234  	if err != nil {
   235  		return "", err
   236  	}
   237  
   238  	return addr.Address, nil
   239  }
   240
View as plain text