ingress_utils.go

Documentation: k8s.io/kubernetes/test/e2e/framework/ingress

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package ingress
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto/rand"
    23  	"crypto/rsa"
    24  	"crypto/tls"
    25  	"crypto/x509"
    26  	"crypto/x509/pkix"
    27  	"encoding/pem"
    28  	"fmt"
    29  	"io"
    30  	"math/big"
    31  	"net"
    32  	"net/http"
    33  	"os"
    34  	"path/filepath"
    35  	"regexp"
    36  	"strconv"
    37  	"strings"
    38  	"time"
    39  
    40  	compute "google.golang.org/api/compute/v1"
    41  	netutils "k8s.io/utils/net"
    42  
    43  	appsv1 "k8s.io/api/apps/v1"
    44  	v1 "k8s.io/api/core/v1"
    45  	networkingv1 "k8s.io/api/networking/v1"
    46  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    47  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    48  	"k8s.io/apimachinery/pkg/fields"
    49  	"k8s.io/apimachinery/pkg/labels"
    50  	"k8s.io/apimachinery/pkg/runtime"
    51  	"k8s.io/apimachinery/pkg/runtime/schema"
    52  	"k8s.io/apimachinery/pkg/util/intstr"
    53  	utilnet "k8s.io/apimachinery/pkg/util/net"
    54  	"k8s.io/apimachinery/pkg/util/sets"
    55  	"k8s.io/apimachinery/pkg/util/wait"
    56  	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
    57  	clientset "k8s.io/client-go/kubernetes"
    58  	"k8s.io/client-go/kubernetes/scheme"
    59  	"k8s.io/kubernetes/test/e2e/framework"
    60  	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
    61  	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
    62  	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
    63  	e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
    64  	testutils "k8s.io/kubernetes/test/utils"
    65  	imageutils "k8s.io/kubernetes/test/utils/image"
    66  
    67  	"github.com/onsi/ginkgo/v2"
    68  )
    69  
    70  const (
    71  	rsaBits  = 2048
    72  	validFor = 365 * 24 * time.Hour
    73  
    74  	// IngressClassKey is ingress class annotation defined in ingress repository.
    75  	// TODO: All these annotations should be reused from
    76  	// ingress-gce/pkg/annotations instead of duplicating them here.
    77  	IngressClassKey = "kubernetes.io/ingress.class"
    78  
    79  	// MulticlusterIngressClassValue is ingress class annotation value for multi cluster ingress.
    80  	MulticlusterIngressClassValue = "gce-multi-cluster"
    81  
    82  	// IngressStaticIPKey is static IP annotation defined in ingress repository.
    83  	IngressStaticIPKey = "kubernetes.io/ingress.global-static-ip-name"
    84  
    85  	// IngressAllowHTTPKey is Allow HTTP annotation defined in ingress repository.
    86  	IngressAllowHTTPKey = "kubernetes.io/ingress.allow-http"
    87  
    88  	// IngressPreSharedCertKey is Pre-shared-cert annotation defined in ingress repository.
    89  	IngressPreSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert"
    90  
    91  	// ServiceApplicationProtocolKey annotation defined in ingress repository.
    92  	ServiceApplicationProtocolKey = "service.alpha.kubernetes.io/app-protocols"
    93  
    94  	// Name of the default http backend service
    95  	defaultBackendName = "default-http-backend"
    96  
    97  	// IngressManifestPath is the parent path to yaml test manifests.
    98  	IngressManifestPath = "test/e2e/testing-manifests/ingress"
    99  
   100  	// GCEIngressManifestPath is the parent path to GCE-specific yaml test manifests.
   101  	GCEIngressManifestPath = IngressManifestPath + "/gce"
   102  
   103  	// IngressReqTimeout is the timeout on a single http request.
   104  	IngressReqTimeout = 10 * time.Second
   105  
   106  	// NEGAnnotation is NEG annotation.
   107  	NEGAnnotation = "cloud.google.com/neg"
   108  
   109  	// NEGStatusAnnotation is NEG status annotation.
   110  	NEGStatusAnnotation = "cloud.google.com/neg-status"
   111  
   112  	// StatusPrefix is prefix for annotation keys used by the ingress controller to specify the
   113  	// names of GCP resources such as forwarding rules, url maps, target proxies, etc
   114  	// that it created for the corresponding ingress.
   115  	StatusPrefix = "ingress.kubernetes.io"
   116  
   117  	// poll is how often to Poll pods, nodes and claims.
   118  	poll = 2 * time.Second
   119  )
   120  
   121  // TestLogger is an interface for log.
   122  type TestLogger interface {
   123  	Infof(format string, args ...interface{})
   124  	Errorf(format string, args ...interface{})
   125  }
   126  
   127  // E2ELogger is test logger.
   128  type E2ELogger struct{}
   129  
   130  // Infof outputs log.
   131  func (l *E2ELogger) Infof(format string, args ...interface{}) {
   132  	framework.Logf(format, args...)
   133  }
   134  
   135  // Errorf outputs log.
   136  func (l *E2ELogger) Errorf(format string, args ...interface{}) {
   137  	framework.Logf(format, args...)
   138  }
   139  
   140  // ConformanceTests contains a closure with an entry and exit log line.
   141  type ConformanceTests struct {
   142  	EntryLog string
   143  	Execute  func()
   144  	ExitLog  string
   145  }
   146  
   147  // NegStatus contains name and zone of the Network Endpoint Group
   148  // resources associated with this service.
   149  // Needs to be consistent with the NEG internal structs in ingress-gce.
   150  type NegStatus struct {
   151  	// NetworkEndpointGroups returns the mapping between service port and NEG
   152  	// resource. key is service port, value is the name of the NEG resource.
   153  	NetworkEndpointGroups map[int32]string `json:"network_endpoint_groups,omitempty"`
   154  	Zones                 []string         `json:"zones,omitempty"`
   155  }
   156  
   157  // SimpleGET executes a get on the given url, returns error if non-200 returned.
   158  func SimpleGET(ctx context.Context, c *http.Client, url, host string) (string, error) {
   159  	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
   160  	if err != nil {
   161  		return "", err
   162  	}
   163  	req.Host = host
   164  	res, err := c.Do(req)
   165  	if err != nil {
   166  		return "", err
   167  	}
   168  	defer res.Body.Close()
   169  	rawBody, err := io.ReadAll(res.Body)
   170  	if err != nil {
   171  		return "", err
   172  	}
   173  	body := string(rawBody)
   174  	if res.StatusCode != http.StatusOK {
   175  		err = fmt.Errorf(
   176  			"GET returned http error %v", res.StatusCode)
   177  	}
   178  	return body, err
   179  }
   180  
   181  // PollURL polls till the url responds with a healthy http code. If
   182  // expectUnreachable is true, it breaks on first non-healthy http code instead.
   183  func PollURL(ctx context.Context, route, host string, timeout time.Duration, interval time.Duration, httpClient *http.Client, expectUnreachable bool) error {
   184  	var lastBody string
   185  	pollErr := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
   186  		var err error
   187  		lastBody, err = SimpleGET(ctx, httpClient, route, host)
   188  		if err != nil {
   189  			framework.Logf("host %v path %v: %v unreachable", host, route, err)
   190  			return expectUnreachable, nil
   191  		}
   192  		framework.Logf("host %v path %v: reached", host, route)
   193  		return !expectUnreachable, nil
   194  	})
   195  	if pollErr != nil {
   196  		return fmt.Errorf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v",
   197  			timeout, route, host, lastBody, pollErr)
   198  	}
   199  	return nil
   200  }
   201  
   202  // CreateIngressComformanceTests generates an slice of sequential test cases:
   203  // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname,
   204  // ingress https with a modified URLMap
   205  func CreateIngressComformanceTests(ctx context.Context, jig *TestJig, ns string, annotations map[string]string) []ConformanceTests {
   206  	manifestPath := filepath.Join(IngressManifestPath, "http")
   207  	// These constants match the manifests used in IngressManifestPath
   208  	tlsHost := "foo.bar.com"
   209  	tlsSecretName := "foo"
   210  	updatedTLSHost := "foobar.com"
   211  	updateURLMapHost := "bar.baz.com"
   212  	updateURLMapPath := "/testurl"
   213  	prefixPathType := networkingv1.PathTypeImplementationSpecific
   214  	// Platform agnostic list of tests that must be satisfied by all controllers
   215  	tests := []ConformanceTests{
   216  		{
   217  			fmt.Sprintf("should create a basic HTTP ingress"),
   218  			func() { jig.CreateIngress(ctx, manifestPath, ns, annotations, annotations) },
   219  			fmt.Sprintf("waiting for urls on basic HTTP ingress"),
   220  		},
   221  		{
   222  			fmt.Sprintf("should terminate TLS for host %v", tlsHost),
   223  			func() { jig.SetHTTPS(ctx, tlsSecretName, tlsHost) },
   224  			fmt.Sprintf("waiting for HTTPS updates to reflect in ingress"),
   225  		},
   226  		{
   227  			fmt.Sprintf("should update url map for host %v to expose a single url: %v", updateURLMapHost, updateURLMapPath),
   228  			func() {
   229  				var pathToFail string
   230  				jig.Update(ctx, func(ing *networkingv1.Ingress) {
   231  					newRules := []networkingv1.IngressRule{}
   232  					for _, rule := range ing.Spec.Rules {
   233  						if rule.Host != updateURLMapHost {
   234  							newRules = append(newRules, rule)
   235  							continue
   236  						}
   237  						existingPath := rule.IngressRuleValue.HTTP.Paths[0]
   238  						pathToFail = existingPath.Path
   239  						newRules = append(newRules, networkingv1.IngressRule{
   240  							Host: updateURLMapHost,
   241  							IngressRuleValue: networkingv1.IngressRuleValue{
   242  								HTTP: &networkingv1.HTTPIngressRuleValue{
   243  									Paths: []networkingv1.HTTPIngressPath{
   244  										{
   245  											Path:     updateURLMapPath,
   246  											PathType: &prefixPathType,
   247  											Backend:  existingPath.Backend,
   248  										},
   249  									},
   250  								},
   251  							},
   252  						})
   253  					}
   254  					ing.Spec.Rules = newRules
   255  				})
   256  				ginkgo.By("Checking that " + pathToFail + " is not exposed by polling for failure")
   257  				route := fmt.Sprintf("http://%v%v", jig.Address, pathToFail)
   258  				framework.ExpectNoError(PollURL(ctx, route, updateURLMapHost, e2eservice.LoadBalancerCleanupTimeout, jig.PollInterval, &http.Client{Timeout: IngressReqTimeout}, true))
   259  			},
   260  			fmt.Sprintf("Waiting for path updates to reflect in L7"),
   261  		},
   262  	}
   263  	// Skip the Update TLS cert test for kubemci: https://github.com/GoogleCloudPlatform/k8s-multicluster-ingress/issues/141.
   264  	if jig.Class != MulticlusterIngressClassValue {
   265  		tests = append(tests, ConformanceTests{
   266  			fmt.Sprintf("should update SSL certificate with modified hostname %v", updatedTLSHost),
   267  			func() {
   268  				jig.Update(ctx, func(ing *networkingv1.Ingress) {
   269  					newRules := []networkingv1.IngressRule{}
   270  					for _, rule := range ing.Spec.Rules {
   271  						if rule.Host != tlsHost {
   272  							newRules = append(newRules, rule)
   273  							continue
   274  						}
   275  						newRules = append(newRules, networkingv1.IngressRule{
   276  							Host:             updatedTLSHost,
   277  							IngressRuleValue: rule.IngressRuleValue,
   278  						})
   279  					}
   280  					ing.Spec.Rules = newRules
   281  				})
   282  				jig.SetHTTPS(ctx, tlsSecretName, updatedTLSHost)
   283  			},
   284  			fmt.Sprintf("Waiting for updated certificates to accept requests for host %v", updatedTLSHost),
   285  		})
   286  	}
   287  	return tests
   288  }
   289  
   290  // GenerateRSACerts generates a basic self signed certificate using a key length
   291  // of rsaBits, valid for validFor time.
   292  func GenerateRSACerts(host string, isCA bool) ([]byte, []byte, error) {
   293  	if len(host) == 0 {
   294  		return nil, nil, fmt.Errorf("Require a non-empty host for client hello")
   295  	}
   296  	priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
   297  	if err != nil {
   298  		return nil, nil, fmt.Errorf("Failed to generate key: %w", err)
   299  	}
   300  	notBefore := time.Now()
   301  	notAfter := notBefore.Add(validFor)
   302  
   303  	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
   304  	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
   305  
   306  	if err != nil {
   307  		return nil, nil, fmt.Errorf("failed to generate serial number: %w", err)
   308  	}
   309  	template := x509.Certificate{
   310  		SerialNumber: serialNumber,
   311  		Subject: pkix.Name{
   312  			CommonName:   "default",
   313  			Organization: []string{"Acme Co"},
   314  		},
   315  		NotBefore: notBefore,
   316  		NotAfter:  notAfter,
   317  
   318  		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
   319  		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
   320  		BasicConstraintsValid: true,
   321  	}
   322  
   323  	hosts := strings.Split(host, ",")
   324  	for _, h := range hosts {
   325  		if ip := netutils.ParseIPSloppy(h); ip != nil {
   326  			template.IPAddresses = append(template.IPAddresses, ip)
   327  		} else {
   328  			template.DNSNames = append(template.DNSNames, h)
   329  		}
   330  	}
   331  
   332  	if isCA {
   333  		template.IsCA = true
   334  		template.KeyUsage |= x509.KeyUsageCertSign
   335  	}
   336  
   337  	var keyOut, certOut bytes.Buffer
   338  	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
   339  	if err != nil {
   340  		return nil, nil, fmt.Errorf("Failed to create certificate: %w", err)
   341  	}
   342  	if err := pem.Encode(&certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
   343  		return nil, nil, fmt.Errorf("Failed creating cert: %w", err)
   344  	}
   345  	if err := pem.Encode(&keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
   346  		return nil, nil, fmt.Errorf("Failed creating key: %w", err)
   347  	}
   348  	return certOut.Bytes(), keyOut.Bytes(), nil
   349  }
   350  
   351  // buildTransportWithCA creates a transport for use in executing HTTPS requests with
   352  // the given certs. Note that the given rootCA must be configured with isCA=true.
   353  func buildTransportWithCA(serverName string, rootCA []byte) (*http.Transport, error) {
   354  	pool := x509.NewCertPool()
   355  	ok := pool.AppendCertsFromPEM(rootCA)
   356  	if !ok {
   357  		return nil, fmt.Errorf("Unable to load serverCA")
   358  	}
   359  	return utilnet.SetTransportDefaults(&http.Transport{
   360  		TLSClientConfig: &tls.Config{
   361  			InsecureSkipVerify: false,
   362  			ServerName:         serverName,
   363  			RootCAs:            pool,
   364  		},
   365  	}), nil
   366  }
   367  
   368  // BuildInsecureClient returns an insecure http client. Can be used for "curl -k".
   369  func BuildInsecureClient(timeout time.Duration) *http.Client {
   370  	t := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
   371  	return &http.Client{Timeout: timeout, Transport: utilnet.SetTransportDefaults(t)}
   372  }
   373  
   374  // createTLSSecret creates a secret containing TLS certificates.
   375  // If a secret with the same name already pathExists in the namespace of the
   376  // Ingress, it's updated.
   377  func createTLSSecret(ctx context.Context, kubeClient clientset.Interface, namespace, secretName string, hosts ...string) (host string, rootCA, privKey []byte, err error) {
   378  	host = strings.Join(hosts, ",")
   379  	framework.Logf("Generating RSA cert for host %v", host)
   380  	cert, key, err := GenerateRSACerts(host, true)
   381  	if err != nil {
   382  		return
   383  	}
   384  	secret := &v1.Secret{
   385  		ObjectMeta: metav1.ObjectMeta{
   386  			Name: secretName,
   387  		},
   388  		Data: map[string][]byte{
   389  			v1.TLSCertKey:       cert,
   390  			v1.TLSPrivateKeyKey: key,
   391  		},
   392  	}
   393  	var s *v1.Secret
   394  	if s, err = kubeClient.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}); err == nil {
   395  		framework.Logf("Updating secret %v in ns %v with hosts %v", secret.Name, namespace, host)
   396  		s.Data = secret.Data
   397  		_, err = kubeClient.CoreV1().Secrets(namespace).Update(ctx, s, metav1.UpdateOptions{})
   398  	} else {
   399  		framework.Logf("Creating secret %v in ns %v with hosts %v", secret.Name, namespace, host)
   400  		_, err = kubeClient.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
   401  	}
   402  	return host, cert, key, err
   403  }
   404  
   405  // TestJig holds the relevant state and parameters of the ingress test.
   406  type TestJig struct {
   407  	Client clientset.Interface
   408  	Logger TestLogger
   409  
   410  	RootCAs map[string][]byte
   411  	Address string
   412  	Ingress *networkingv1.Ingress
   413  	// class was the value of the annotation keyed under `kubernetes.io/ingress.class`.
   414  	// A new ingressClassName field has been added that is used to reference the IngressClass.
   415  	// It's added to all ingresses created by this jig.
   416  	Class string
   417  
   418  	// The interval used to poll urls
   419  	PollInterval time.Duration
   420  }
   421  
   422  // NewIngressTestJig instantiates struct with client
   423  func NewIngressTestJig(c clientset.Interface) *TestJig {
   424  	return &TestJig{
   425  		Client:       c,
   426  		RootCAs:      map[string][]byte{},
   427  		PollInterval: e2eservice.LoadBalancerPollInterval,
   428  		Logger:       &E2ELogger{},
   429  	}
   430  }
   431  
   432  // CreateIngress creates the Ingress and associated service/rc.
   433  // Required: ing.yaml, rc.yaml, svc.yaml must exist in manifestPath
   434  // Optional: secret.yaml, ingAnnotations
   435  // If ingAnnotations is specified it will overwrite any annotations in ing.yaml
   436  // If svcAnnotations is specified it will overwrite any annotations in svc.yaml
   437  func (j *TestJig) CreateIngress(ctx context.Context, manifestPath, ns string, ingAnnotations map[string]string, svcAnnotations map[string]string) {
   438  	var err error
   439  	read := func(file string) string {
   440  		data, err := e2etestfiles.Read(filepath.Join(manifestPath, file))
   441  		if err != nil {
   442  			framework.Fail(err.Error())
   443  		}
   444  		return string(data)
   445  	}
   446  	exists := func(file string) bool {
   447  		found, err := e2etestfiles.Exists(filepath.Join(manifestPath, file))
   448  		if err != nil {
   449  			framework.Fail(fmt.Sprintf("fatal error looking for test file %s: %s", file, err))
   450  		}
   451  		return found
   452  	}
   453  
   454  	j.Logger.Infof("creating replication controller")
   455  	e2ekubectl.RunKubectlOrDieInput(ns, read("rc.yaml"), "create", "-f", "-")
   456  
   457  	j.Logger.Infof("creating service")
   458  	e2ekubectl.RunKubectlOrDieInput(ns, read("svc.yaml"), "create", "-f", "-")
   459  	if len(svcAnnotations) > 0 {
   460  		svcList, err := j.Client.CoreV1().Services(ns).List(ctx, metav1.ListOptions{})
   461  		framework.ExpectNoError(err)
   462  		for _, svc := range svcList.Items {
   463  			svc.Annotations = svcAnnotations
   464  			_, err = j.Client.CoreV1().Services(ns).Update(ctx, &svc, metav1.UpdateOptions{})
   465  			framework.ExpectNoError(err)
   466  		}
   467  	}
   468  
   469  	if exists("secret.yaml") {
   470  		j.Logger.Infof("creating secret")
   471  		e2ekubectl.RunKubectlOrDieInput(ns, read("secret.yaml"), "create", "-f", "-")
   472  	}
   473  	j.Logger.Infof("Parsing ingress from %v", filepath.Join(manifestPath, "ing.yaml"))
   474  
   475  	j.Ingress, err = ingressFromManifest(filepath.Join(manifestPath, "ing.yaml"))
   476  	framework.ExpectNoError(err)
   477  	j.Ingress.Namespace = ns
   478  	if j.Class != "" {
   479  		j.Ingress.Spec.IngressClassName = &j.Class
   480  	}
   481  	j.Logger.Infof("creating %v ingress", j.Ingress.Name)
   482  	j.Ingress, err = j.runCreate(ctx, j.Ingress)
   483  	framework.ExpectNoError(err)
   484  }
   485  
   486  // marshalToYaml marshals an object into YAML for a given GroupVersion.
   487  // The object must be known in SupportedMediaTypes() for the Codecs under "client-go/kubernetes/scheme".
   488  func marshalToYaml(obj runtime.Object, gv schema.GroupVersion) ([]byte, error) {
   489  	mediaType := "application/yaml"
   490  	info, ok := runtime.SerializerInfoForMediaType(scheme.Codecs.SupportedMediaTypes(), mediaType)
   491  	if !ok {
   492  		return []byte{}, fmt.Errorf("unsupported media type %q", mediaType)
   493  	}
   494  	encoder := scheme.Codecs.EncoderForVersion(info.Serializer, gv)
   495  	return runtime.Encode(encoder, obj)
   496  }
   497  
   498  // ingressFromManifest reads a .json/yaml file and returns the ingress in it.
   499  func ingressFromManifest(fileName string) (*networkingv1.Ingress, error) {
   500  	var ing networkingv1.Ingress
   501  	data, err := e2etestfiles.Read(fileName)
   502  	if err != nil {
   503  		return nil, err
   504  	}
   505  
   506  	json, err := utilyaml.ToJSON(data)
   507  	if err != nil {
   508  		return nil, err
   509  	}
   510  	if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), json, &ing); err != nil {
   511  		return nil, err
   512  	}
   513  	return &ing, nil
   514  }
   515  
   516  // ingressToManifest generates a yaml file in the given path with the given ingress.
   517  // Assumes that a directory exists at the given path.
   518  func ingressToManifest(ing *networkingv1.Ingress, path string) error {
   519  	serialized, err := marshalToYaml(ing, networkingv1.SchemeGroupVersion)
   520  	if err != nil {
   521  		return fmt.Errorf("failed to marshal ingress %v to YAML: %w", ing, err)
   522  	}
   523  
   524  	if err := os.WriteFile(path, serialized, 0600); err != nil {
   525  		return fmt.Errorf("error in writing ingress to file: %w", err)
   526  	}
   527  	return nil
   528  }
   529  
   530  // runCreate runs the required command to create the given ingress.
   531  func (j *TestJig) runCreate(ctx context.Context, ing *networkingv1.Ingress) (*networkingv1.Ingress, error) {
   532  	if j.Class != MulticlusterIngressClassValue {
   533  		return j.Client.NetworkingV1().Ingresses(ing.Namespace).Create(ctx, ing, metav1.CreateOptions{})
   534  	}
   535  	// Use kubemci to create a multicluster ingress.
   536  	filePath := framework.TestContext.OutputDir + "/mci.yaml"
   537  	if err := ingressToManifest(ing, filePath); err != nil {
   538  		return nil, err
   539  	}
   540  	_, err := e2ekubectl.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
   541  	return ing, err
   542  }
   543  
   544  // runUpdate runs the required command to update the given ingress.
   545  func (j *TestJig) runUpdate(ctx context.Context, ing *networkingv1.Ingress) (*networkingv1.Ingress, error) {
   546  	if j.Class != MulticlusterIngressClassValue {
   547  		return j.Client.NetworkingV1().Ingresses(ing.Namespace).Update(ctx, ing, metav1.UpdateOptions{})
   548  	}
   549  	// Use kubemci to update a multicluster ingress.
   550  	// kubemci does not have an update command. We use "create --force" to update an existing ingress.
   551  	filePath := framework.TestContext.OutputDir + "/mci.yaml"
   552  	if err := ingressToManifest(ing, filePath); err != nil {
   553  		return nil, err
   554  	}
   555  	_, err := e2ekubectl.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force")
   556  	return ing, err
   557  }
   558  
   559  // DescribeIng describes information of ingress by running kubectl describe ing.
   560  func DescribeIng(ns string) {
   561  	framework.Logf("\nOutput of kubectl describe ing:\n")
   562  	desc, _ := e2ekubectl.RunKubectl(
   563  		ns, "describe", "ing")
   564  	framework.Logf(desc)
   565  }
   566  
   567  // Update retrieves the ingress, performs the passed function, and then updates it.
   568  func (j *TestJig) Update(ctx context.Context, update func(ing *networkingv1.Ingress)) {
   569  	var err error
   570  	ns, name := j.Ingress.Namespace, j.Ingress.Name
   571  	for i := 0; i < 3; i++ {
   572  		j.Ingress, err = j.Client.NetworkingV1().Ingresses(ns).Get(ctx, name, metav1.GetOptions{})
   573  		if err != nil {
   574  			framework.Failf("failed to get ingress %s/%s: %v", ns, name, err)
   575  		}
   576  		update(j.Ingress)
   577  		j.Ingress, err = j.runUpdate(ctx, j.Ingress)
   578  		if err == nil {
   579  			DescribeIng(j.Ingress.Namespace)
   580  			return
   581  		}
   582  		if !apierrors.IsConflict(err) && !apierrors.IsServerTimeout(err) {
   583  			framework.Failf("failed to update ingress %s/%s: %v", ns, name, err)
   584  		}
   585  	}
   586  	framework.Failf("too many retries updating ingress %s/%s", ns, name)
   587  }
   588  
   589  // AddHTTPS updates the ingress to add this secret for these hosts.
   590  func (j *TestJig) AddHTTPS(ctx context.Context, secretName string, hosts ...string) {
   591  	// TODO: Just create the secret in GetRootCAs once we're watching secrets in
   592  	// the ingress controller.
   593  	_, cert, _, err := createTLSSecret(ctx, j.Client, j.Ingress.Namespace, secretName, hosts...)
   594  	framework.ExpectNoError(err)
   595  	j.Logger.Infof("Updating ingress %v to also use secret %v for TLS termination", j.Ingress.Name, secretName)
   596  	j.Update(ctx, func(ing *networkingv1.Ingress) {
   597  		ing.Spec.TLS = append(ing.Spec.TLS, networkingv1.IngressTLS{Hosts: hosts, SecretName: secretName})
   598  	})
   599  	j.RootCAs[secretName] = cert
   600  }
   601  
   602  // SetHTTPS updates the ingress to use only this secret for these hosts.
   603  func (j *TestJig) SetHTTPS(ctx context.Context, secretName string, hosts ...string) {
   604  	_, cert, _, err := createTLSSecret(ctx, j.Client, j.Ingress.Namespace, secretName, hosts...)
   605  	framework.ExpectNoError(err)
   606  	j.Logger.Infof("Updating ingress %v to only use secret %v for TLS termination", j.Ingress.Name, secretName)
   607  	j.Update(ctx, func(ing *networkingv1.Ingress) {
   608  		ing.Spec.TLS = []networkingv1.IngressTLS{{Hosts: hosts, SecretName: secretName}}
   609  	})
   610  	j.RootCAs = map[string][]byte{secretName: cert}
   611  }
   612  
   613  // RemoveHTTPS updates the ingress to not use this secret for TLS.
   614  // Note: Does not delete the secret.
   615  func (j *TestJig) RemoveHTTPS(ctx context.Context, secretName string) {
   616  	newTLS := []networkingv1.IngressTLS{}
   617  	for _, ingressTLS := range j.Ingress.Spec.TLS {
   618  		if secretName != ingressTLS.SecretName {
   619  			newTLS = append(newTLS, ingressTLS)
   620  		}
   621  	}
   622  	j.Logger.Infof("Updating ingress %v to not use secret %v for TLS termination", j.Ingress.Name, secretName)
   623  	j.Update(ctx, func(ing *networkingv1.Ingress) {
   624  		ing.Spec.TLS = newTLS
   625  	})
   626  	delete(j.RootCAs, secretName)
   627  }
   628  
   629  // PrepareTLSSecret creates a TLS secret and caches the cert.
   630  func (j *TestJig) PrepareTLSSecret(ctx context.Context, namespace, secretName string, hosts ...string) error {
   631  	_, cert, _, err := createTLSSecret(ctx, j.Client, namespace, secretName, hosts...)
   632  	if err != nil {
   633  		return err
   634  	}
   635  	j.RootCAs[secretName] = cert
   636  	return nil
   637  }
   638  
   639  // GetRootCA returns a rootCA from the ingress test jig.
   640  func (j *TestJig) GetRootCA(secretName string) (rootCA []byte) {
   641  	var ok bool
   642  	rootCA, ok = j.RootCAs[secretName]
   643  	if !ok {
   644  		framework.Failf("Failed to retrieve rootCAs, no recorded secret by name %v", secretName)
   645  	}
   646  	return
   647  }
   648  
   649  // TryDeleteIngress attempts to delete the ingress resource and logs errors if they occur.
   650  func (j *TestJig) TryDeleteIngress(ctx context.Context) {
   651  	j.tryDeleteGivenIngress(ctx, j.Ingress)
   652  }
   653  
   654  func (j *TestJig) tryDeleteGivenIngress(ctx context.Context, ing *networkingv1.Ingress) {
   655  	if err := j.runDelete(ctx, ing); err != nil {
   656  		j.Logger.Infof("Error while deleting the ingress %v/%v with class %s: %v", ing.Namespace, ing.Name, j.Class, err)
   657  	}
   658  }
   659  
   660  // runDelete runs the required command to delete the given ingress.
   661  func (j *TestJig) runDelete(ctx context.Context, ing *networkingv1.Ingress) error {
   662  	if j.Class != MulticlusterIngressClassValue {
   663  		return j.Client.NetworkingV1().Ingresses(ing.Namespace).Delete(ctx, ing.Name, metav1.DeleteOptions{})
   664  	}
   665  	// Use kubemci to delete a multicluster ingress.
   666  	filePath := framework.TestContext.OutputDir + "/mci.yaml"
   667  	if err := ingressToManifest(ing, filePath); err != nil {
   668  		return err
   669  	}
   670  	_, err := e2ekubectl.RunKubemciWithKubeconfig("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
   671  	return err
   672  }
   673  
   674  // getIngressAddressFromKubemci returns the IP address of the given multicluster ingress using kubemci.
   675  // TODO(nikhiljindal): Update this to be able to return hostname as well.
   676  func getIngressAddressFromKubemci(name string) ([]string, error) {
   677  	var addresses []string
   678  	out, err := e2ekubectl.RunKubemciCmd("get-status", name)
   679  	if err != nil {
   680  		return addresses, err
   681  	}
   682  	ip := findIPv4(out)
   683  	if ip != "" {
   684  		addresses = append(addresses, ip)
   685  	}
   686  	return addresses, err
   687  }
   688  
   689  // findIPv4 returns the first IPv4 address found in the given string.
   690  func findIPv4(input string) string {
   691  	numBlock := "(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])"
   692  	regexPattern := numBlock + "\\." + numBlock + "\\." + numBlock + "\\." + numBlock
   693  
   694  	regEx := regexp.MustCompile(regexPattern)
   695  	return regEx.FindString(input)
   696  }
   697  
   698  // getIngressAddress returns the ips/hostnames associated with the Ingress.
   699  func getIngressAddress(ctx context.Context, client clientset.Interface, ns, name, class string) ([]string, error) {
   700  	if class == MulticlusterIngressClassValue {
   701  		return getIngressAddressFromKubemci(name)
   702  	}
   703  	ing, err := client.NetworkingV1().Ingresses(ns).Get(ctx, name, metav1.GetOptions{})
   704  	if err != nil {
   705  		return nil, err
   706  	}
   707  	var addresses []string
   708  	for _, a := range ing.Status.LoadBalancer.Ingress {
   709  		if a.IP != "" {
   710  			addresses = append(addresses, a.IP)
   711  		}
   712  		if a.Hostname != "" {
   713  			addresses = append(addresses, a.Hostname)
   714  		}
   715  	}
   716  	return addresses, nil
   717  }
   718  
   719  // WaitForIngressAddress waits for the Ingress to acquire an address.
   720  func (j *TestJig) WaitForIngressAddress(ctx context.Context, c clientset.Interface, ns, ingName string, timeout time.Duration) (string, error) {
   721  	var address string
   722  	err := wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
   723  		ipOrNameList, err := getIngressAddress(ctx, c, ns, ingName, j.Class)
   724  		if err != nil || len(ipOrNameList) == 0 {
   725  			j.Logger.Errorf("Waiting for Ingress %s/%s to acquire IP, error: %v, ipOrNameList: %v", ns, ingName, err, ipOrNameList)
   726  			return false, err
   727  		}
   728  		address = ipOrNameList[0]
   729  		j.Logger.Infof("Found address %s for ingress %s/%s", address, ns, ingName)
   730  		return true, nil
   731  	})
   732  	return address, err
   733  }
   734  
   735  func (j *TestJig) pollIngressWithCert(ctx context.Context, ing *networkingv1.Ingress, address string, knownHosts []string, cert []byte, waitForNodePort bool, timeout time.Duration) error {
   736  	// Check that all rules respond to a simple GET.
   737  	knownHostsSet := sets.NewString(knownHosts...)
   738  	for _, rules := range ing.Spec.Rules {
   739  		timeoutClient := &http.Client{Timeout: IngressReqTimeout}
   740  		proto := "http"
   741  		if knownHostsSet.Has(rules.Host) {
   742  			var err error
   743  			// Create transport with cert to verify if the server uses the correct one.
   744  			timeoutClient.Transport, err = buildTransportWithCA(rules.Host, cert)
   745  			if err != nil {
   746  				return err
   747  			}
   748  			proto = "https"
   749  		}
   750  		for _, p := range rules.IngressRuleValue.HTTP.Paths {
   751  			if waitForNodePort {
   752  				nodePort := int(p.Backend.Service.Port.Number)
   753  				if err := j.pollServiceNodePort(ctx, ing.Namespace, p.Backend.Service.Name, nodePort); err != nil {
   754  					j.Logger.Infof("Error in waiting for nodeport %d on service %v/%v: %s", nodePort, ing.Namespace, p.Backend.Service.Name, err)
   755  					return err
   756  				}
   757  			}
   758  			route := fmt.Sprintf("%v://%v%v", proto, address, p.Path)
   759  			j.Logger.Infof("Testing route %v host %v with simple GET", route, rules.Host)
   760  			if err := PollURL(ctx, route, rules.Host, timeout, j.PollInterval, timeoutClient, false); err != nil {
   761  				return err
   762  			}
   763  		}
   764  	}
   765  	j.Logger.Infof("Finished polling on all rules on ingress %q", ing.Name)
   766  	return nil
   767  }
   768  
   769  // WaitForIngress waits for the Ingress to get an address.
   770  // WaitForIngress returns when it gets the first 200 response
   771  func (j *TestJig) WaitForIngress(ctx context.Context, waitForNodePort bool) {
   772  	if err := j.WaitForGivenIngressWithTimeout(ctx, j.Ingress, waitForNodePort, e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, j.Client)); err != nil {
   773  		framework.Failf("error in waiting for ingress to get an address: %s", err)
   774  	}
   775  }
   776  
   777  // WaitForIngressToStable waits for the LB return 100 consecutive 200 responses.
   778  func (j *TestJig) WaitForIngressToStable(ctx context.Context) {
   779  	if err := wait.PollWithContext(ctx, 10*time.Second, e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, j.Client), func(ctx context.Context) (bool, error) {
   780  		_, err := j.GetDistinctResponseFromIngress(ctx)
   781  		if err != nil {
   782  			return false, nil
   783  		}
   784  		return true, nil
   785  	}); err != nil {
   786  		framework.Failf("error in waiting for ingress to stabilize: %v", err)
   787  	}
   788  }
   789  
   790  // WaitForGivenIngressWithTimeout waits till the ingress acquires an IP,
   791  // then waits for its hosts/urls to respond to a protocol check (either
   792  // http or https). If waitForNodePort is true, the NodePort of the Service
   793  // is verified before verifying the Ingress. NodePort is currently a
   794  // requirement for cloudprovider Ingress.
   795  func (j *TestJig) WaitForGivenIngressWithTimeout(ctx context.Context, ing *networkingv1.Ingress, waitForNodePort bool, timeout time.Duration) error {
   796  	// Wait for the loadbalancer IP.
   797  	address, err := j.WaitForIngressAddress(ctx, j.Client, ing.Namespace, ing.Name, timeout)
   798  	if err != nil {
   799  		return fmt.Errorf("Ingress failed to acquire an IP address within %v", timeout)
   800  	}
   801  
   802  	var knownHosts []string
   803  	var cert []byte
   804  	if len(ing.Spec.TLS) > 0 {
   805  		knownHosts = ing.Spec.TLS[0].Hosts
   806  		cert = j.GetRootCA(ing.Spec.TLS[0].SecretName)
   807  	}
   808  	return j.pollIngressWithCert(ctx, ing, address, knownHosts, cert, waitForNodePort, timeout)
   809  }
   810  
   811  // WaitForIngressWithCert waits till the ingress acquires an IP, then waits for its
   812  // hosts/urls to respond to a protocol check (either http or https). If
   813  // waitForNodePort is true, the NodePort of the Service is verified before
   814  // verifying the Ingress. NodePort is currently a requirement for cloudprovider
   815  // Ingress. Hostnames and certificate need to be explicitly passed in.
   816  func (j *TestJig) WaitForIngressWithCert(ctx context.Context, waitForNodePort bool, knownHosts []string, cert []byte) error {
   817  	// Wait for the loadbalancer IP.
   818  	propagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, j.Client)
   819  	address, err := j.WaitForIngressAddress(ctx, j.Client, j.Ingress.Namespace, j.Ingress.Name, propagationTimeout)
   820  	if err != nil {
   821  		return fmt.Errorf("Ingress failed to acquire an IP address within %v", propagationTimeout)
   822  	}
   823  
   824  	return j.pollIngressWithCert(ctx, j.Ingress, address, knownHosts, cert, waitForNodePort, propagationTimeout)
   825  }
   826  
   827  // VerifyURL polls for the given iterations, in intervals, and fails if the
   828  // given url returns a non-healthy http code even once.
   829  func (j *TestJig) VerifyURL(ctx context.Context, route, host string, iterations int, interval time.Duration, httpClient *http.Client) error {
   830  	for i := 0; i < iterations; i++ {
   831  		b, err := SimpleGET(ctx, httpClient, route, host)
   832  		if err != nil {
   833  			framework.Logf(b)
   834  			return err
   835  		}
   836  		j.Logger.Infof("Verified %v with host %v %d times, sleeping for %v", route, host, i, interval)
   837  		time.Sleep(interval)
   838  	}
   839  	return nil
   840  }
   841  
   842  func (j *TestJig) pollServiceNodePort(ctx context.Context, ns, name string, port int) error {
   843  	// TODO: Curl all nodes?
   844  	u, err := getPortURL(ctx, j.Client, ns, name, port)
   845  	if err != nil {
   846  		return err
   847  	}
   848  	return PollURL(ctx, u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false)
   849  }
   850  
   851  // getSvcNodePort returns the node port for the given service:port.
   852  func getSvcNodePort(ctx context.Context, client clientset.Interface, ns, name string, svcPort int) (int, error) {
   853  	svc, err := client.CoreV1().Services(ns).Get(ctx, name, metav1.GetOptions{})
   854  	if err != nil {
   855  		return 0, err
   856  	}
   857  	for _, p := range svc.Spec.Ports {
   858  		if p.Port == int32(svcPort) {
   859  			if p.NodePort != 0 {
   860  				return int(p.NodePort), nil
   861  			}
   862  		}
   863  	}
   864  	return 0, fmt.Errorf(
   865  		"no node port found for service %v, port %v", name, svcPort)
   866  }
   867  
   868  // getPortURL returns the url to a nodeport Service.
   869  func getPortURL(ctx context.Context, client clientset.Interface, ns, name string, svcPort int) (string, error) {
   870  	nodePort, err := getSvcNodePort(ctx, client, ns, name, svcPort)
   871  	if err != nil {
   872  		return "", err
   873  	}
   874  	// This list of nodes must not include the any control plane nodes, which are marked
   875  	// unschedulable, since control plane nodes don't run kube-proxy. Without
   876  	// kube-proxy NodePorts won't work.
   877  	var nodes *v1.NodeList
   878  	if wait.PollUntilContextTimeout(ctx, poll, framework.SingleCallTimeout, true, func(ctx context.Context) (bool, error) {
   879  		nodes, err = client.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{
   880  			"spec.unschedulable": "false",
   881  		}.AsSelector().String()})
   882  		if err != nil {
   883  			return false, err
   884  		}
   885  		return true, nil
   886  	}) != nil {
   887  		return "", err
   888  	}
   889  	if len(nodes.Items) == 0 {
   890  		return "", fmt.Errorf("Unable to list nodes in cluster")
   891  	}
   892  	for _, node := range nodes.Items {
   893  		for _, address := range node.Status.Addresses {
   894  			if address.Type == v1.NodeExternalIP {
   895  				if address.Address != "" {
   896  					host := net.JoinHostPort(address.Address, fmt.Sprint(nodePort))
   897  					return fmt.Sprintf("http://%s", host), nil
   898  				}
   899  			}
   900  		}
   901  	}
   902  	return "", fmt.Errorf("failed to find external address for service %v", name)
   903  }
   904  
   905  // GetIngressNodePorts returns related backend services' nodePorts.
   906  // Current GCE ingress controller allows traffic to the default HTTP backend
   907  // by default, so retrieve its nodePort if includeDefaultBackend is true.
   908  func (j *TestJig) GetIngressNodePorts(ctx context.Context, includeDefaultBackend bool) []string {
   909  	nodePorts := []string{}
   910  	svcPorts := j.GetServicePorts(ctx, includeDefaultBackend)
   911  	for _, svcPort := range svcPorts {
   912  		nodePorts = append(nodePorts, strconv.Itoa(int(svcPort.NodePort)))
   913  	}
   914  	return nodePorts
   915  }
   916  
   917  // GetServicePorts returns related backend services' svcPorts.
   918  // Current GCE ingress controller allows traffic to the default HTTP backend
   919  // by default, so retrieve its nodePort if includeDefaultBackend is true.
   920  func (j *TestJig) GetServicePorts(ctx context.Context, includeDefaultBackend bool) map[string]v1.ServicePort {
   921  	svcPorts := make(map[string]v1.ServicePort)
   922  	if includeDefaultBackend {
   923  		defaultSvc, err := j.Client.CoreV1().Services(metav1.NamespaceSystem).Get(ctx, defaultBackendName, metav1.GetOptions{})
   924  		framework.ExpectNoError(err)
   925  		svcPorts[defaultBackendName] = defaultSvc.Spec.Ports[0]
   926  	}
   927  
   928  	backendSvcs := []string{}
   929  	if j.Ingress.Spec.DefaultBackend != nil {
   930  		backendSvcs = append(backendSvcs, j.Ingress.Spec.DefaultBackend.Service.Name)
   931  	}
   932  	for _, rule := range j.Ingress.Spec.Rules {
   933  		for _, ingPath := range rule.HTTP.Paths {
   934  			backendSvcs = append(backendSvcs, ingPath.Backend.Service.Name)
   935  		}
   936  	}
   937  	for _, svcName := range backendSvcs {
   938  		svc, err := j.Client.CoreV1().Services(j.Ingress.Namespace).Get(ctx, svcName, metav1.GetOptions{})
   939  		framework.ExpectNoError(err)
   940  		svcPorts[svcName] = svc.Spec.Ports[0]
   941  	}
   942  	return svcPorts
   943  }
   944  
   945  // ConstructFirewallForIngress returns the expected GCE firewall rule for the ingress resource
   946  func (j *TestJig) ConstructFirewallForIngress(ctx context.Context, firewallRuleName string, nodeTags []string) *compute.Firewall {
   947  	nodePorts := j.GetIngressNodePorts(ctx, true)
   948  
   949  	fw := compute.Firewall{}
   950  	fw.Name = firewallRuleName
   951  	fw.SourceRanges = framework.TestContext.CloudConfig.Provider.LoadBalancerSrcRanges()
   952  	fw.TargetTags = nodeTags
   953  	fw.Allowed = []*compute.FirewallAllowed{
   954  		{
   955  			IPProtocol: "tcp",
   956  			Ports:      nodePorts,
   957  		},
   958  	}
   959  	return &fw
   960  }
   961  
   962  // GetDistinctResponseFromIngress tries GET call to the ingress VIP and return all distinct responses.
   963  func (j *TestJig) GetDistinctResponseFromIngress(ctx context.Context) (sets.String, error) {
   964  	// Wait for the loadbalancer IP.
   965  	propagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, j.Client)
   966  	address, err := j.WaitForIngressAddress(ctx, j.Client, j.Ingress.Namespace, j.Ingress.Name, propagationTimeout)
   967  	if err != nil {
   968  		framework.Failf("Ingress failed to acquire an IP address within %v", propagationTimeout)
   969  	}
   970  	responses := sets.NewString()
   971  	timeoutClient := &http.Client{Timeout: IngressReqTimeout}
   972  
   973  	for i := 0; i < 100; i++ {
   974  		url := fmt.Sprintf("http://%v", address)
   975  		res, err := SimpleGET(ctx, timeoutClient, url, "")
   976  		if err != nil {
   977  			j.Logger.Errorf("Failed to GET %q. Got responses: %q: %v", url, res, err)
   978  			return responses, err
   979  		}
   980  		responses.Insert(res)
   981  	}
   982  	return responses, nil
   983  }
   984  
   985  // NginxIngressController manages implementation details of Ingress on Nginx.
   986  type NginxIngressController struct {
   987  	Ns     string
   988  	rc     *v1.ReplicationController
   989  	pod    *v1.Pod
   990  	Client clientset.Interface
   991  	lbSvc  *v1.Service
   992  }
   993  
   994  // Init initializes the NginxIngressController
   995  func (cont *NginxIngressController) Init(ctx context.Context) {
   996  	// Set up a LoadBalancer service in front of nginx ingress controller and pass it via
   997  	// --publish-service flag (see <IngressManifestPath>/nginx/rc.yaml) to make it work in private
   998  	// clusters, i.e. clusters where nodes don't have public IPs.
   999  	framework.Logf("Creating load balancer service for nginx ingress controller")
  1000  	serviceJig := e2eservice.NewTestJig(cont.Client, cont.Ns, "nginx-ingress-lb")
  1001  	_, err := serviceJig.CreateTCPService(ctx, func(svc *v1.Service) {
  1002  		svc.Spec.Type = v1.ServiceTypeLoadBalancer
  1003  		svc.Spec.Selector = map[string]string{"k8s-app": "nginx-ingress-lb"}
  1004  		svc.Spec.Ports = []v1.ServicePort{
  1005  			{Name: "http", Port: 80},
  1006  			{Name: "https", Port: 443},
  1007  			{Name: "stats", Port: 18080}}
  1008  	})
  1009  	framework.ExpectNoError(err)
  1010  	cont.lbSvc, err = serviceJig.WaitForLoadBalancer(ctx, e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, cont.Client))
  1011  	framework.ExpectNoError(err)
  1012  
  1013  	read := func(file string) string {
  1014  		data, err := e2etestfiles.Read(filepath.Join(IngressManifestPath, "nginx", file))
  1015  		if err != nil {
  1016  			framework.Fail(err.Error())
  1017  		}
  1018  		return string(data)
  1019  	}
  1020  
  1021  	framework.Logf("initializing nginx ingress controller")
  1022  	e2ekubectl.RunKubectlOrDieInput(cont.Ns, read("rc.yaml"), "create", "-f", "-")
  1023  
  1024  	rc, err := cont.Client.CoreV1().ReplicationControllers(cont.Ns).Get(ctx, "nginx-ingress-controller", metav1.GetOptions{})
  1025  	framework.ExpectNoError(err)
  1026  	cont.rc = rc
  1027  
  1028  	framework.Logf("waiting for pods with label %v", rc.Spec.Selector)
  1029  	sel := labels.SelectorFromSet(labels.Set(rc.Spec.Selector))
  1030  	framework.ExpectNoError(testutils.WaitForPodsWithLabelRunning(cont.Client, cont.Ns, sel))
  1031  	pods, err := cont.Client.CoreV1().Pods(cont.Ns).List(ctx, metav1.ListOptions{LabelSelector: sel.String()})
  1032  	framework.ExpectNoError(err)
  1033  	if len(pods.Items) == 0 {
  1034  		framework.Failf("Failed to find nginx ingress controller pods with selector %v", sel)
  1035  	}
  1036  	cont.pod = &pods.Items[0]
  1037  	framework.Logf("ingress controller running in pod %v", cont.pod.Name)
  1038  }
  1039  
  1040  // TearDown cleans up the NginxIngressController.
  1041  func (cont *NginxIngressController) TearDown(ctx context.Context) {
  1042  	if cont.lbSvc == nil {
  1043  		framework.Logf("No LoadBalancer service created, no cleanup necessary")
  1044  		return
  1045  	}
  1046  	e2eservice.WaitForServiceDeletedWithFinalizer(ctx, cont.Client, cont.Ns, cont.lbSvc.Name)
  1047  }
  1048  
  1049  func generateBacksideHTTPSIngressSpec(ns string) *networkingv1.Ingress {
  1050  	return &networkingv1.Ingress{
  1051  		ObjectMeta: metav1.ObjectMeta{
  1052  			Name:      "echoheaders-https",
  1053  			Namespace: ns,
  1054  		},
  1055  		Spec: networkingv1.IngressSpec{
  1056  			// Note kubemci requires a default backend.
  1057  			DefaultBackend: &networkingv1.IngressBackend{
  1058  				Service: &networkingv1.IngressServiceBackend{
  1059  					Name: "echoheaders-https",
  1060  					Port: networkingv1.ServiceBackendPort{
  1061  						Number: 443,
  1062  					},
  1063  				},
  1064  			},
  1065  		},
  1066  	}
  1067  }
  1068  
  1069  func generateBacksideHTTPSServiceSpec() *v1.Service {
  1070  	return &v1.Service{
  1071  		ObjectMeta: metav1.ObjectMeta{
  1072  			Name: "echoheaders-https",
  1073  			Annotations: map[string]string{
  1074  				ServiceApplicationProtocolKey: `{"my-https-port":"HTTPS"}`,
  1075  			},
  1076  		},
  1077  		Spec: v1.ServiceSpec{
  1078  			Ports: []v1.ServicePort{{
  1079  				Name:       "my-https-port",
  1080  				Protocol:   v1.ProtocolTCP,
  1081  				Port:       443,
  1082  				TargetPort: intstr.FromString("echo-443"),
  1083  			}},
  1084  			Selector: map[string]string{
  1085  				"app": "echoheaders-https",
  1086  			},
  1087  			Type: v1.ServiceTypeNodePort,
  1088  		},
  1089  	}
  1090  }
  1091  
  1092  func generateBacksideHTTPSDeploymentSpec() *appsv1.Deployment {
  1093  	labels := map[string]string{"app": "echoheaders-https"}
  1094  	d := e2edeployment.NewDeployment("echoheaders-https", 0, labels, "echoheaders-https", imageutils.GetE2EImage(imageutils.Agnhost), appsv1.RollingUpdateDeploymentStrategyType)
  1095  	d.Spec.Replicas = nil
  1096  	d.Spec.Template.Spec.Containers[0].Command = []string{
  1097  		"/agnhost",
  1098  		"netexec",
  1099  		"--http-port=8443",
  1100  		"--tls-cert-file=/localhost.crt",
  1101  		"--tls-private-key-file=/localhost.key",
  1102  	}
  1103  	d.Spec.Template.Spec.Containers[0].Ports = []v1.ContainerPort{{
  1104  		ContainerPort: 8443,
  1105  		Name:          "echo-443",
  1106  	}}
  1107  	return d
  1108  }
  1109  
  1110  // SetUpBacksideHTTPSIngress sets up deployment, service and ingress with backside HTTPS configured.
  1111  func (j *TestJig) SetUpBacksideHTTPSIngress(ctx context.Context, cs clientset.Interface, namespace string, staticIPName string) (*appsv1.Deployment, *v1.Service, *networkingv1.Ingress, error) {
  1112  	deployCreated, err := cs.AppsV1().Deployments(namespace).Create(ctx, generateBacksideHTTPSDeploymentSpec(), metav1.CreateOptions{})
  1113  	if err != nil {
  1114  		return nil, nil, nil, err
  1115  	}
  1116  	svcCreated, err := cs.CoreV1().Services(namespace).Create(ctx, generateBacksideHTTPSServiceSpec(), metav1.CreateOptions{})
  1117  	if err != nil {
  1118  		return nil, nil, nil, err
  1119  	}
  1120  	ingToCreate := generateBacksideHTTPSIngressSpec(namespace)
  1121  	if staticIPName != "" {
  1122  		if ingToCreate.Annotations == nil {
  1123  			ingToCreate.Annotations = map[string]string{}
  1124  		}
  1125  		ingToCreate.Annotations[IngressStaticIPKey] = staticIPName
  1126  	}
  1127  	ingCreated, err := j.runCreate(ctx, ingToCreate)
  1128  	if err != nil {
  1129  		return nil, nil, nil, err
  1130  	}
  1131  	return deployCreated, svcCreated, ingCreated, nil
  1132  }
  1133  
  1134  // DeleteTestResource deletes given deployment, service and ingress.
  1135  func (j *TestJig) DeleteTestResource(ctx context.Context, cs clientset.Interface, deploy *appsv1.Deployment, svc *v1.Service, ing *networkingv1.Ingress) []error {
  1136  	var errs []error
  1137  	if ing != nil {
  1138  		if err := j.runDelete(ctx, ing); err != nil {
  1139  			errs = append(errs, fmt.Errorf("error while deleting ingress %s/%s: %w", ing.Namespace, ing.Name, err))
  1140  		}
  1141  	}
  1142  	if svc != nil {
  1143  		if err := cs.CoreV1().Services(svc.Namespace).Delete(ctx, svc.Name, metav1.DeleteOptions{}); err != nil {
  1144  			errs = append(errs, fmt.Errorf("error while deleting service %s/%s: %w", svc.Namespace, svc.Name, err))
  1145  		}
  1146  	}
  1147  	if deploy != nil {
  1148  		if err := cs.AppsV1().Deployments(deploy.Namespace).Delete(ctx, deploy.Name, metav1.DeleteOptions{}); err != nil {
  1149  			errs = append(errs, fmt.Errorf("error while deleting deployment %s/%s: %w", deploy.Namespace, deploy.Name, err))
  1150  		}
  1151  	}
  1152  	return errs
  1153  }
  1154
View as plain text