ingress_gce.go

Documentation: k8s.io/kubernetes/test/e2e/network

     1  //go:build !providerless
     2  // +build !providerless
     3  
     4  /*
     5  Copyright 2015 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package network
    21  
    22  import (
    23  	"context"
    24  	"encoding/json"
    25  	"fmt"
    26  	"path/filepath"
    27  	"time"
    28  
    29  	v1 "k8s.io/api/core/v1"
    30  	rbacv1 "k8s.io/api/rbac/v1"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/runtime/schema"
    33  	"k8s.io/apimachinery/pkg/util/wait"
    34  	"k8s.io/apiserver/pkg/authentication/serviceaccount"
    35  	"k8s.io/kubernetes/test/e2e/feature"
    36  	"k8s.io/kubernetes/test/e2e/framework"
    37  	e2eauth "k8s.io/kubernetes/test/e2e/framework/auth"
    38  	e2eingress "k8s.io/kubernetes/test/e2e/framework/ingress"
    39  	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
    40  	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
    41  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    42  	"k8s.io/kubernetes/test/e2e/network/common"
    43  	admissionapi "k8s.io/pod-security-admission/api"
    44  
    45  	"github.com/onsi/ginkgo/v2"
    46  	"github.com/onsi/gomega"
    47  )
    48  
    49  const (
    50  	negUpdateTimeout = 2 * time.Minute
    51  )
    52  
    53  var _ = common.SIGDescribe("Loadbalancing: L7", func() {
    54  	defer ginkgo.GinkgoRecover()
    55  	var (
    56  		ns               string
    57  		jig              *e2eingress.TestJig
    58  		conformanceTests []e2eingress.ConformanceTests
    59  	)
    60  	f := framework.NewDefaultFramework("ingress")
    61  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
    62  
    63  	ginkgo.BeforeEach(func(ctx context.Context) {
    64  		jig = e2eingress.NewIngressTestJig(f.ClientSet)
    65  		ns = f.Namespace.Name
    66  
    67  		// this test wants powerful permissions.  Since the namespace names are unique, we can leave this
    68  		// lying around so we don't have to race any caches
    69  		err := e2eauth.BindClusterRole(ctx, jig.Client.RbacV1(), "cluster-admin", f.Namespace.Name,
    70  			rbacv1.Subject{Kind: rbacv1.ServiceAccountKind, Namespace: f.Namespace.Name, Name: "default"})
    71  		framework.ExpectNoError(err)
    72  
    73  		err = e2eauth.WaitForAuthorizationUpdate(ctx, jig.Client.AuthorizationV1(),
    74  			serviceaccount.MakeUsername(f.Namespace.Name, "default"),
    75  			"", "create", schema.GroupResource{Resource: "pods"}, true)
    76  		framework.ExpectNoError(err)
    77  	})
    78  
    79  	// Before enabling this loadbalancer test in any other test list you must
    80  	// make sure the associated project has enough quota. At the time of this
    81  	// writing a GCE project is allowed 3 backend services by default. This
    82  	// test requires at least 5.
    83  	//
    84  	// Slow by design ~10m for each "It" block dominated by loadbalancer setup time
    85  	// TODO: write similar tests for nginx, haproxy and AWS Ingress.
    86  	f.Describe("GCE", framework.WithSlow(), feature.Ingress, func() {
    87  		var gceController *gce.IngressController
    88  
    89  		// Platform specific setup
    90  		ginkgo.BeforeEach(func(ctx context.Context) {
    91  			e2eskipper.SkipUnlessProviderIs("gce", "gke")
    92  			ginkgo.By("Initializing gce controller")
    93  			gceController = &gce.IngressController{
    94  				Ns:     ns,
    95  				Client: jig.Client,
    96  				Cloud:  framework.TestContext.CloudConfig,
    97  			}
    98  			err := gceController.Init(ctx)
    99  			framework.ExpectNoError(err)
   100  		})
   101  
   102  		// Platform specific cleanup
   103  		ginkgo.AfterEach(func(ctx context.Context) {
   104  			if ginkgo.CurrentSpecReport().Failed() {
   105  				e2eingress.DescribeIng(ns)
   106  			}
   107  			if jig.Ingress == nil {
   108  				ginkgo.By("No ingress created, no cleanup necessary")
   109  				return
   110  			}
   111  			ginkgo.By("Deleting ingress")
   112  			jig.TryDeleteIngress(ctx)
   113  
   114  			ginkgo.By("Cleaning up cloud resources")
   115  			err := gceController.CleanupIngressController(ctx)
   116  			framework.ExpectNoError(err)
   117  		})
   118  
   119  		ginkgo.It("should conform to Ingress spec", func(ctx context.Context) {
   120  			conformanceTests = e2eingress.CreateIngressComformanceTests(ctx, jig, ns, map[string]string{})
   121  			for _, t := range conformanceTests {
   122  				ginkgo.By(t.EntryLog)
   123  				t.Execute()
   124  				ginkgo.By(t.ExitLog)
   125  				jig.WaitForIngress(ctx, true)
   126  			}
   127  		})
   128  
   129  	})
   130  
   131  	f.Describe("GCE", framework.WithSlow(), feature.NEG, func() {
   132  		var gceController *gce.IngressController
   133  
   134  		// Platform specific setup
   135  		ginkgo.BeforeEach(func(ctx context.Context) {
   136  			e2eskipper.SkipUnlessProviderIs("gce", "gke")
   137  			ginkgo.By("Initializing gce controller")
   138  			gceController = &gce.IngressController{
   139  				Ns:     ns,
   140  				Client: jig.Client,
   141  				Cloud:  framework.TestContext.CloudConfig,
   142  			}
   143  			err := gceController.Init(ctx)
   144  			framework.ExpectNoError(err)
   145  		})
   146  
   147  		// Platform specific cleanup
   148  		ginkgo.AfterEach(func(ctx context.Context) {
   149  			if ginkgo.CurrentSpecReport().Failed() {
   150  				e2eingress.DescribeIng(ns)
   151  			}
   152  			if jig.Ingress == nil {
   153  				ginkgo.By("No ingress created, no cleanup necessary")
   154  				return
   155  			}
   156  			ginkgo.By("Deleting ingress")
   157  			jig.TryDeleteIngress(ctx)
   158  
   159  			ginkgo.By("Cleaning up cloud resources")
   160  			err := gceController.CleanupIngressController(ctx)
   161  			framework.ExpectNoError(err)
   162  		})
   163  
   164  		ginkgo.It("should conform to Ingress spec", func(ctx context.Context) {
   165  			jig.PollInterval = 5 * time.Second
   166  			conformanceTests = e2eingress.CreateIngressComformanceTests(ctx, jig, ns, map[string]string{
   167  				e2eingress.NEGAnnotation: `{"ingress": true}`,
   168  			})
   169  			for _, t := range conformanceTests {
   170  				ginkgo.By(t.EntryLog)
   171  				t.Execute()
   172  				ginkgo.By(t.ExitLog)
   173  				jig.WaitForIngress(ctx, true)
   174  				err := gceController.WaitForNegBackendService(ctx, jig.GetServicePorts(ctx, false))
   175  				framework.ExpectNoError(err)
   176  			}
   177  		})
   178  
   179  		ginkgo.It("should be able to switch between IG and NEG modes", func(ctx context.Context) {
   180  			var err error
   181  			propagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, f.ClientSet)
   182  			ginkgo.By("Create a basic HTTP ingress using NEG")
   183  			jig.CreateIngress(ctx, filepath.Join(e2eingress.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{})
   184  			jig.WaitForIngress(ctx, true)
   185  			err = gceController.WaitForNegBackendService(ctx, jig.GetServicePorts(ctx, false))
   186  			framework.ExpectNoError(err)
   187  
   188  			ginkgo.By("Switch backend service to use IG")
   189  			svcList, err := f.ClientSet.CoreV1().Services(ns).List(ctx, metav1.ListOptions{})
   190  			framework.ExpectNoError(err)
   191  			for _, svc := range svcList.Items {
   192  				svc.Annotations[e2eingress.NEGAnnotation] = `{"ingress": false}`
   193  				_, err = f.ClientSet.CoreV1().Services(ns).Update(ctx, &svc, metav1.UpdateOptions{})
   194  				framework.ExpectNoError(err)
   195  			}
   196  			err = wait.PollWithContext(ctx, 5*time.Second, propagationTimeout, func(ctx context.Context) (bool, error) {
   197  				if err := gceController.BackendServiceUsingIG(jig.GetServicePorts(ctx, false)); err != nil {
   198  					framework.Logf("ginkgo.Failed to verify IG backend service: %v", err)
   199  					return false, nil
   200  				}
   201  				return true, nil
   202  			})
   203  			framework.ExpectNoError(err, "Expect backend service to target IG, but failed to observe")
   204  			jig.WaitForIngress(ctx, true)
   205  
   206  			ginkgo.By("Switch backend service to use NEG")
   207  			svcList, err = f.ClientSet.CoreV1().Services(ns).List(ctx, metav1.ListOptions{})
   208  			framework.ExpectNoError(err)
   209  			for _, svc := range svcList.Items {
   210  				svc.Annotations[e2eingress.NEGAnnotation] = `{"ingress": true}`
   211  				_, err = f.ClientSet.CoreV1().Services(ns).Update(ctx, &svc, metav1.UpdateOptions{})
   212  				framework.ExpectNoError(err)
   213  			}
   214  			err = wait.PollWithContext(ctx, 5*time.Second, propagationTimeout, func(ctx context.Context) (bool, error) {
   215  				if err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(ctx, false)); err != nil {
   216  					framework.Logf("ginkgo.Failed to verify NEG backend service: %v", err)
   217  					return false, nil
   218  				}
   219  				return true, nil
   220  			})
   221  			framework.ExpectNoError(err, "Expect backend service to target NEG, but failed to observe")
   222  			jig.WaitForIngress(ctx, true)
   223  		})
   224  
   225  		ginkgo.It("should be able to create a ClusterIP service", func(ctx context.Context) {
   226  			ginkgo.By("Create a basic HTTP ingress using NEG")
   227  			jig.CreateIngress(ctx, filepath.Join(e2eingress.IngressManifestPath, "neg-clusterip"), ns, map[string]string{}, map[string]string{})
   228  			jig.WaitForIngress(ctx, true)
   229  			svcPorts := jig.GetServicePorts(ctx, false)
   230  			err := gceController.WaitForNegBackendService(ctx, svcPorts)
   231  			framework.ExpectNoError(err)
   232  
   233  			// ClusterIP ServicePorts have no NodePort
   234  			for _, sp := range svcPorts {
   235  				gomega.Expect(sp.NodePort).To(gomega.Equal(int32(0)))
   236  			}
   237  		})
   238  
   239  		ginkgo.It("should sync endpoints to NEG", func(ctx context.Context) {
   240  			name := "hostname"
   241  			scaleAndValidateNEG := func(num int) {
   242  				scale, err := f.ClientSet.AppsV1().Deployments(ns).GetScale(ctx, name, metav1.GetOptions{})
   243  				framework.ExpectNoError(err)
   244  				if scale.Spec.Replicas != int32(num) {
   245  					scale.ResourceVersion = "" // indicate the scale update should be unconditional
   246  					scale.Spec.Replicas = int32(num)
   247  					_, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(ctx, name, scale, metav1.UpdateOptions{})
   248  					framework.ExpectNoError(err)
   249  				}
   250  				err = wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) {
   251  					res, err := jig.GetDistinctResponseFromIngress(ctx)
   252  					if err != nil {
   253  						return false, nil
   254  					}
   255  					framework.Logf("Expecting %d backends, got %d", num, res.Len())
   256  					return res.Len() == num, nil
   257  				})
   258  				framework.ExpectNoError(err)
   259  			}
   260  
   261  			ginkgo.By("Create a basic HTTP ingress using NEG")
   262  			jig.CreateIngress(ctx, filepath.Join(e2eingress.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{})
   263  			jig.WaitForIngress(ctx, true)
   264  			jig.WaitForIngressToStable(ctx)
   265  			err := gceController.WaitForNegBackendService(ctx, jig.GetServicePorts(ctx, false))
   266  			framework.ExpectNoError(err)
   267  			// initial replicas number is 1
   268  			scaleAndValidateNEG(1)
   269  
   270  			ginkgo.By("Scale up number of backends to 5")
   271  			scaleAndValidateNEG(5)
   272  
   273  			ginkgo.By("Scale down number of backends to 3")
   274  			scaleAndValidateNEG(3)
   275  
   276  			ginkgo.By("Scale up number of backends to 6")
   277  			scaleAndValidateNEG(6)
   278  
   279  			ginkgo.By("Scale down number of backends to 2")
   280  			scaleAndValidateNEG(3)
   281  		})
   282  
   283  		ginkgo.It("rolling update backend pods should not cause service disruption", func(ctx context.Context) {
   284  			name := "hostname"
   285  			replicas := 8
   286  			ginkgo.By("Create a basic HTTP ingress using NEG")
   287  			jig.CreateIngress(ctx, filepath.Join(e2eingress.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{})
   288  			jig.WaitForIngress(ctx, true)
   289  			jig.WaitForIngressToStable(ctx)
   290  			err := gceController.WaitForNegBackendService(ctx, jig.GetServicePorts(ctx, false))
   291  			framework.ExpectNoError(err)
   292  
   293  			ginkgo.By(fmt.Sprintf("Scale backend replicas to %d", replicas))
   294  			scale, err := f.ClientSet.AppsV1().Deployments(ns).GetScale(ctx, name, metav1.GetOptions{})
   295  			framework.ExpectNoError(err)
   296  			scale.ResourceVersion = "" // indicate the scale update should be unconditional
   297  			scale.Spec.Replicas = int32(replicas)
   298  			_, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(ctx, name, scale, metav1.UpdateOptions{})
   299  			framework.ExpectNoError(err)
   300  
   301  			propagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, f.ClientSet)
   302  			err = wait.Poll(10*time.Second, propagationTimeout, func() (bool, error) {
   303  				res, err := jig.GetDistinctResponseFromIngress(ctx)
   304  				if err != nil {
   305  					return false, nil
   306  				}
   307  				return res.Len() == replicas, nil
   308  			})
   309  			framework.ExpectNoError(err)
   310  
   311  			ginkgo.By("Trigger rolling update and observe service disruption")
   312  			deploy, err := f.ClientSet.AppsV1().Deployments(ns).Get(ctx, name, metav1.GetOptions{})
   313  			framework.ExpectNoError(err)
   314  			// trigger by changing graceful termination period to 60 seconds
   315  			gracePeriod := int64(60)
   316  			deploy.Spec.Template.Spec.TerminationGracePeriodSeconds = &gracePeriod
   317  			_, err = f.ClientSet.AppsV1().Deployments(ns).Update(ctx, deploy, metav1.UpdateOptions{})
   318  			framework.ExpectNoError(err)
   319  			err = wait.Poll(10*time.Second, propagationTimeout, func() (bool, error) {
   320  				res, err := jig.GetDistinctResponseFromIngress(ctx)
   321  				if err != nil {
   322  					return false, err
   323  				}
   324  				deploy, err := f.ClientSet.AppsV1().Deployments(ns).Get(ctx, name, metav1.GetOptions{})
   325  				if err != nil {
   326  					return false, err
   327  				}
   328  				if int(deploy.Status.UpdatedReplicas) == replicas {
   329  					if res.Len() == replicas {
   330  						return true, nil
   331  					}
   332  					framework.Logf("Expecting %d different responses, but got %d.", replicas, res.Len())
   333  					return false, nil
   334  
   335  				}
   336  				framework.Logf("Waiting for rolling update to finished. Keep sending traffic.")
   337  				return false, nil
   338  			})
   339  			framework.ExpectNoError(err)
   340  		})
   341  
   342  		ginkgo.It("should sync endpoints for both Ingress-referenced NEG and standalone NEG", func(ctx context.Context) {
   343  			name := "hostname"
   344  			expectedKeys := []int32{80, 443}
   345  
   346  			scaleAndValidateExposedNEG := func(num int) {
   347  				scale, err := f.ClientSet.AppsV1().Deployments(ns).GetScale(ctx, name, metav1.GetOptions{})
   348  				framework.ExpectNoError(err)
   349  				if scale.Spec.Replicas != int32(num) {
   350  					scale.ResourceVersion = "" // indicate the scale update should be unconditional
   351  					scale.Spec.Replicas = int32(num)
   352  					_, err = f.ClientSet.AppsV1().Deployments(ns).UpdateScale(ctx, name, scale, metav1.UpdateOptions{})
   353  					framework.ExpectNoError(err)
   354  				}
   355  				err = wait.Poll(10*time.Second, negUpdateTimeout, func() (bool, error) {
   356  					svc, err := f.ClientSet.CoreV1().Services(ns).Get(ctx, name, metav1.GetOptions{})
   357  					framework.ExpectNoError(err)
   358  
   359  					var status e2eingress.NegStatus
   360  					v, ok := svc.Annotations[e2eingress.NEGStatusAnnotation]
   361  					if !ok {
   362  						// Wait for NEG sync loop to find NEGs
   363  						framework.Logf("Waiting for %v, got: %+v", e2eingress.NEGStatusAnnotation, svc.Annotations)
   364  						return false, nil
   365  					}
   366  					err = json.Unmarshal([]byte(v), &status)
   367  					if err != nil {
   368  						framework.Logf("Error in parsing Expose NEG annotation: %v", err)
   369  						return false, nil
   370  					}
   371  					framework.Logf("Got %v: %v", e2eingress.NEGStatusAnnotation, v)
   372  
   373  					// Expect 2 NEGs to be created based on the test setup (neg-exposed)
   374  					if len(status.NetworkEndpointGroups) != 2 {
   375  						framework.Logf("Expected 2 NEGs, got %d", len(status.NetworkEndpointGroups))
   376  						return false, nil
   377  					}
   378  
   379  					for _, port := range expectedKeys {
   380  						if _, ok := status.NetworkEndpointGroups[port]; !ok {
   381  							framework.Logf("Expected ServicePort key %v, but does not exist", port)
   382  						}
   383  					}
   384  
   385  					if len(status.NetworkEndpointGroups) != len(expectedKeys) {
   386  						framework.Logf("Expected length of %+v to equal length of %+v, but does not", status.NetworkEndpointGroups, expectedKeys)
   387  					}
   388  
   389  					gceCloud, err := gce.GetGCECloud()
   390  					framework.ExpectNoError(err)
   391  					for _, neg := range status.NetworkEndpointGroups {
   392  						networkEndpoints, err := gceCloud.ListNetworkEndpoints(neg, gceController.Cloud.Zone, false)
   393  						framework.ExpectNoError(err)
   394  						if len(networkEndpoints) != num {
   395  							framework.Logf("Expect number of endpoints to be %d, but got %d", num, len(networkEndpoints))
   396  							return false, nil
   397  						}
   398  					}
   399  
   400  					return true, nil
   401  				})
   402  				framework.ExpectNoError(err)
   403  			}
   404  
   405  			ginkgo.By("Create a basic HTTP ingress using NEG")
   406  			jig.CreateIngress(ctx, filepath.Join(e2eingress.IngressManifestPath, "neg-exposed"), ns, map[string]string{}, map[string]string{})
   407  			jig.WaitForIngress(ctx, true)
   408  			err := gceController.WaitForNegBackendService(ctx, jig.GetServicePorts(ctx, false))
   409  			framework.ExpectNoError(err)
   410  			// initial replicas number is 1
   411  			scaleAndValidateExposedNEG(1)
   412  
   413  			ginkgo.By("Scale up number of backends to 5")
   414  			scaleAndValidateExposedNEG(5)
   415  
   416  			ginkgo.By("Scale down number of backends to 3")
   417  			scaleAndValidateExposedNEG(3)
   418  
   419  			ginkgo.By("Scale up number of backends to 6")
   420  			scaleAndValidateExposedNEG(6)
   421  
   422  			ginkgo.By("Scale down number of backends to 2")
   423  			scaleAndValidateExposedNEG(3)
   424  		})
   425  
   426  		ginkgo.It("should create NEGs for all ports with the Ingress annotation, and NEGs for the standalone annotation otherwise", func(ctx context.Context) {
   427  			ginkgo.By("Create a basic HTTP ingress using standalone NEG")
   428  			jig.CreateIngress(ctx, filepath.Join(e2eingress.IngressManifestPath, "neg-exposed"), ns, map[string]string{}, map[string]string{})
   429  			jig.WaitForIngress(ctx, true)
   430  
   431  			name := "hostname"
   432  			detectNegAnnotation(ctx, f, jig, gceController, ns, name, 2)
   433  
   434  			// Add Ingress annotation - NEGs should stay the same.
   435  			ginkgo.By("Adding NEG Ingress annotation")
   436  			svcList, err := f.ClientSet.CoreV1().Services(ns).List(ctx, metav1.ListOptions{})
   437  			framework.ExpectNoError(err)
   438  			for _, svc := range svcList.Items {
   439  				svc.Annotations[e2eingress.NEGAnnotation] = `{"ingress":true,"exposed_ports":{"80":{},"443":{}}}`
   440  				_, err = f.ClientSet.CoreV1().Services(ns).Update(ctx, &svc, metav1.UpdateOptions{})
   441  				framework.ExpectNoError(err)
   442  			}
   443  			detectNegAnnotation(ctx, f, jig, gceController, ns, name, 2)
   444  
   445  			// Modify exposed NEG annotation, but keep ingress annotation
   446  			ginkgo.By("Modifying exposed NEG annotation, but keep Ingress annotation")
   447  			svcList, err = f.ClientSet.CoreV1().Services(ns).List(ctx, metav1.ListOptions{})
   448  			framework.ExpectNoError(err)
   449  			for _, svc := range svcList.Items {
   450  				svc.Annotations[e2eingress.NEGAnnotation] = `{"ingress":true,"exposed_ports":{"443":{}}}`
   451  				_, err = f.ClientSet.CoreV1().Services(ns).Update(ctx, &svc, metav1.UpdateOptions{})
   452  				framework.ExpectNoError(err)
   453  			}
   454  			detectNegAnnotation(ctx, f, jig, gceController, ns, name, 2)
   455  
   456  			// Remove Ingress annotation. Expect 1 NEG
   457  			ginkgo.By("Disabling Ingress annotation, but keeping one standalone NEG")
   458  			svcList, err = f.ClientSet.CoreV1().Services(ns).List(ctx, metav1.ListOptions{})
   459  			framework.ExpectNoError(err)
   460  			for _, svc := range svcList.Items {
   461  				svc.Annotations[e2eingress.NEGAnnotation] = `{"ingress":false,"exposed_ports":{"443":{}}}`
   462  				_, err = f.ClientSet.CoreV1().Services(ns).Update(ctx, &svc, metav1.UpdateOptions{})
   463  				framework.ExpectNoError(err)
   464  			}
   465  			detectNegAnnotation(ctx, f, jig, gceController, ns, name, 1)
   466  
   467  			// Remove NEG annotation entirely. Expect 0 NEGs.
   468  			ginkgo.By("Removing NEG annotation")
   469  			svcList, err = f.ClientSet.CoreV1().Services(ns).List(ctx, metav1.ListOptions{})
   470  			framework.ExpectNoError(err)
   471  			for _, svc := range svcList.Items {
   472  				delete(svc.Annotations, e2eingress.NEGAnnotation)
   473  				// Service cannot be ClusterIP if it's using Instance Groups.
   474  				svc.Spec.Type = v1.ServiceTypeNodePort
   475  				_, err = f.ClientSet.CoreV1().Services(ns).Update(ctx, &svc, metav1.UpdateOptions{})
   476  				framework.ExpectNoError(err)
   477  			}
   478  			detectNegAnnotation(ctx, f, jig, gceController, ns, name, 0)
   479  		})
   480  	})
   481  })
   482  
   483  func detectNegAnnotation(ctx context.Context, f *framework.Framework, jig *e2eingress.TestJig, gceController *gce.IngressController, ns, name string, negs int) {
   484  	if err := wait.Poll(5*time.Second, negUpdateTimeout, func() (bool, error) {
   485  		svc, err := f.ClientSet.CoreV1().Services(ns).Get(ctx, name, metav1.GetOptions{})
   486  		if err != nil {
   487  			return false, nil
   488  		}
   489  
   490  		// if we expect no NEGs, then we should be using IGs
   491  		if negs == 0 {
   492  			err := gceController.BackendServiceUsingIG(jig.GetServicePorts(ctx, false))
   493  			if err != nil {
   494  				framework.Logf("ginkgo.Failed to validate IG backend service: %v", err)
   495  				return false, nil
   496  			}
   497  			return true, nil
   498  		}
   499  
   500  		var status e2eingress.NegStatus
   501  		v, ok := svc.Annotations[e2eingress.NEGStatusAnnotation]
   502  		if !ok {
   503  			framework.Logf("Waiting for %v, got: %+v", e2eingress.NEGStatusAnnotation, svc.Annotations)
   504  			return false, nil
   505  		}
   506  
   507  		err = json.Unmarshal([]byte(v), &status)
   508  		if err != nil {
   509  			framework.Logf("Error in parsing Expose NEG annotation: %v", err)
   510  			return false, nil
   511  		}
   512  		framework.Logf("Got %v: %v", e2eingress.NEGStatusAnnotation, v)
   513  
   514  		if len(status.NetworkEndpointGroups) != negs {
   515  			framework.Logf("Expected %d NEGs, got %d", negs, len(status.NetworkEndpointGroups))
   516  			return false, nil
   517  		}
   518  
   519  		gceCloud, err := gce.GetGCECloud()
   520  		framework.ExpectNoError(err)
   521  		for _, neg := range status.NetworkEndpointGroups {
   522  			networkEndpoints, err := gceCloud.ListNetworkEndpoints(neg, gceController.Cloud.Zone, false)
   523  			framework.ExpectNoError(err)
   524  			if len(networkEndpoints) != 1 {
   525  				framework.Logf("Expect NEG %s to exist, but got %d", neg, len(networkEndpoints))
   526  				return false, nil
   527  			}
   528  		}
   529  
   530  		err = gceController.BackendServiceUsingNEG(jig.GetServicePorts(ctx, false))
   531  		if err != nil {
   532  			framework.Logf("ginkgo.Failed to validate NEG backend service: %v", err)
   533  			return false, nil
   534  		}
   535  		return true, nil
   536  	}); err != nil {
   537  		framework.ExpectNoError(err)
   538  	}
   539  }
   540
View as plain text