helpers.go

Documentation: sigs.k8s.io/gateway-api/conformance/utils/kubernetes

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kubernetes
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"net"
    24  	"reflect"
    25  	"strconv"
    26  	"strings"
    27  	"sync"
    28  	"testing"
    29  	"time"
    30  
    31  	"github.com/stretchr/testify/require"
    32  
    33  	v1 "k8s.io/api/core/v1"
    34  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    35  	"k8s.io/apimachinery/pkg/types"
    36  	"k8s.io/apimachinery/pkg/util/wait"
    37  	"sigs.k8s.io/controller-runtime/pkg/client"
    38  
    39  	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
    40  	"sigs.k8s.io/gateway-api/apis/v1alpha2"
    41  	"sigs.k8s.io/gateway-api/conformance/utils/config"
    42  )
    43  
    44  // GatewayExcludedFromReadinessChecks is an annotation that can be placed on a
    45  // Gateway provided via the tests to indicate that it is NOT expected to be
    46  // Accepted or Provisioned in its default state. This is generally helpful for
    47  // tests which validate fixing broken Gateways, e.t.c.
    48  const GatewayExcludedFromReadinessChecks = "gateway-api/skip-this-for-readiness"
    49  
    50  // GatewayRef is a tiny type for specifying an HTTP Route ParentRef without
    51  // relying on a specific api version.
    52  type GatewayRef struct {
    53  	types.NamespacedName
    54  	listenerNames []*gatewayv1.SectionName
    55  }
    56  
    57  // NewGatewayRef creates a GatewayRef resource.  ListenerNames are optional.
    58  func NewGatewayRef(nn types.NamespacedName, listenerNames ...string) GatewayRef {
    59  	var listeners []*gatewayv1.SectionName
    60  
    61  	if len(listenerNames) == 0 {
    62  		listenerNames = append(listenerNames, "")
    63  	}
    64  
    65  	for _, listener := range listenerNames {
    66  		sectionName := gatewayv1.SectionName(listener)
    67  		listeners = append(listeners, &sectionName)
    68  	}
    69  	return GatewayRef{
    70  		NamespacedName: nn,
    71  		listenerNames:  listeners,
    72  	}
    73  }
    74  
    75  // GWCMustBeAcceptedConditionTrue waits until the specified GatewayClass has an Accepted condition set with a status value equal to True.
    76  func GWCMustHaveAcceptedConditionTrue(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, gwcName string) string {
    77  	return gwcMustBeAccepted(t, c, timeoutConfig, gwcName, string(metav1.ConditionTrue))
    78  }
    79  
    80  // GWCMustBeAcceptedConditionAny waits until the specified GatewayClass has an Accepted condition set with a status set to any value.
    81  func GWCMustHaveAcceptedConditionAny(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, gwcName string) string {
    82  	return gwcMustBeAccepted(t, c, timeoutConfig, gwcName, "")
    83  }
    84  
    85  // gwcMustBeAccepted waits until the specified GatewayClass has an Accepted
    86  // condition set. Passing an empty status string means that any value
    87  // will be accepted. It also returns the ControllerName for the GatewayClass.
    88  // This will cause the test to halt if the specified timeout is exceeded.
    89  func gwcMustBeAccepted(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, gwcName, expectedStatus string) string {
    90  	t.Helper()
    91  
    92  	var controllerName string
    93  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.GWCMustBeAccepted, true, func(ctx context.Context) (bool, error) {
    94  		gwc := &gatewayv1.GatewayClass{}
    95  		err := c.Get(ctx, types.NamespacedName{Name: gwcName}, gwc)
    96  		if err != nil {
    97  			return false, fmt.Errorf("error fetching GatewayClass: %w", err)
    98  		}
    99  
   100  		controllerName = string(gwc.Spec.ControllerName)
   101  
   102  		if err := ConditionsHaveLatestObservedGeneration(gwc, gwc.Status.Conditions); err != nil {
   103  			t.Log("GatewayClass", err)
   104  			return false, nil
   105  		}
   106  
   107  		// Passing an empty string as the Reason means that any Reason will do.
   108  		return findConditionInList(t, gwc.Status.Conditions, "Accepted", expectedStatus, ""), nil
   109  	})
   110  	require.NoErrorf(t, waitErr, "error waiting for %s GatewayClass to have Accepted condition to be set: %v", gwcName, waitErr)
   111  
   112  	return controllerName
   113  }
   114  
   115  // GatewayMustHaveLatestConditions waits until the specified Gateway has
   116  // all conditions updated with the latest observed generation.
   117  func GatewayMustHaveLatestConditions(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, gwNN types.NamespacedName) {
   118  	t.Helper()
   119  
   120  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.LatestObservedGenerationSet, true, func(ctx context.Context) (bool, error) {
   121  		gw := &gatewayv1.Gateway{}
   122  		err := c.Get(ctx, gwNN, gw)
   123  		if err != nil {
   124  			return false, fmt.Errorf("error fetching Gateway: %w", err)
   125  		}
   126  
   127  		if err := ConditionsHaveLatestObservedGeneration(gw, gw.Status.Conditions); err != nil {
   128  			t.Logf("Gateway %s latest conditions not set yet: %v", gwNN.String(), err)
   129  			return false, nil
   130  		}
   131  
   132  		return true, nil
   133  	})
   134  
   135  	require.NoErrorf(t, waitErr, "error waiting for Gateway %s to have Latest ObservedGeneration to be set: %v", gwNN.String(), waitErr)
   136  }
   137  
   138  // GatewayClassMustHaveLatestConditions will fail the test if there are
   139  // conditions that were not updated
   140  func GatewayClassMustHaveLatestConditions(t *testing.T, gwc *gatewayv1.GatewayClass) {
   141  	t.Helper()
   142  
   143  	if err := ConditionsHaveLatestObservedGeneration(gwc, gwc.Status.Conditions); err != nil {
   144  		t.Fatalf("GatewayClass %v", err)
   145  	}
   146  }
   147  
   148  // HTTPRouteMustHaveLatestConditions will fail the test if there are
   149  // conditions that were not updated
   150  func HTTPRouteMustHaveLatestConditions(t *testing.T, r *gatewayv1.HTTPRoute) {
   151  	t.Helper()
   152  
   153  	for _, parent := range r.Status.Parents {
   154  		if err := ConditionsHaveLatestObservedGeneration(r, parent.Conditions); err != nil {
   155  			t.Fatalf("HTTPRoute(controller=%v, parentRef=%#v) %v", parent.ControllerName, parent, err)
   156  		}
   157  	}
   158  }
   159  
   160  func ConditionsHaveLatestObservedGeneration(obj metav1.Object, conditions []metav1.Condition) error {
   161  	staleConditions := FilterStaleConditions(obj, conditions)
   162  
   163  	if len(staleConditions) == 0 {
   164  		return nil
   165  	}
   166  
   167  	wantGeneration := obj.GetGeneration()
   168  	var b strings.Builder
   169  	fmt.Fprintf(&b, "expected observedGeneration to be updated to %d for all conditions", wantGeneration)
   170  	fmt.Fprintf(&b, ", only %d/%d were updated.", len(conditions)-len(staleConditions), len(conditions))
   171  	fmt.Fprintf(&b, " stale conditions are: ")
   172  
   173  	for i, c := range staleConditions {
   174  		fmt.Fprintf(&b, "%s (generation %d)", c.Type, c.ObservedGeneration)
   175  		if i != len(staleConditions)-1 {
   176  			fmt.Fprintf(&b, ", ")
   177  		}
   178  	}
   179  
   180  	return errors.New(b.String())
   181  }
   182  
   183  // FilterStaleConditions returns the list of status condition whose observedGeneration does not
   184  // match the object's metadata.Generation
   185  func FilterStaleConditions(obj metav1.Object, conditions []metav1.Condition) []metav1.Condition {
   186  	stale := make([]metav1.Condition, 0, len(conditions))
   187  	for _, condition := range conditions {
   188  		if obj.GetGeneration() != condition.ObservedGeneration {
   189  			stale = append(stale, condition)
   190  		}
   191  	}
   192  	return stale
   193  }
   194  
   195  // NamespacesMustBeReady waits until all Pods are marked Ready and all Gateways
   196  // are marked Accepted and Programmed in the specified namespace(s). This will
   197  // cause the test to halt if the specified timeout is exceeded.
   198  func NamespacesMustBeReady(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, namespaces []string) {
   199  	t.Helper()
   200  
   201  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.NamespacesMustBeReady, true, func(ctx context.Context) (bool, error) {
   202  		for _, ns := range namespaces {
   203  			gwList := &gatewayv1.GatewayList{}
   204  			err := c.List(ctx, gwList, client.InNamespace(ns))
   205  			if err != nil {
   206  				t.Errorf("Error listing Gateways: %v", err)
   207  			}
   208  			for _, gw := range gwList.Items {
   209  				gw := gw
   210  
   211  				if val, ok := gw.Annotations[GatewayExcludedFromReadinessChecks]; ok && val == "true" {
   212  					t.Logf("Gateway %s/%s is skipped for setup and wont be tested", ns, gw.Name)
   213  					continue
   214  				}
   215  
   216  				if err = ConditionsHaveLatestObservedGeneration(&gw, gw.Status.Conditions); err != nil {
   217  					t.Logf("Gateway %s/%s %v", ns, gw.Name, err)
   218  					return false, nil
   219  				}
   220  
   221  				// Passing an empty string as the Reason means that any Reason will do.
   222  				if !findConditionInList(t, gw.Status.Conditions, string(gatewayv1.GatewayConditionAccepted), "True", "") {
   223  					t.Logf("%s/%s Gateway not Accepted yet", ns, gw.Name)
   224  					return false, nil
   225  				}
   226  
   227  				// Passing an empty string as the Reason means that any Reason will do.
   228  				if !findConditionInList(t, gw.Status.Conditions, string(gatewayv1.GatewayConditionProgrammed), "True", "") {
   229  					t.Logf("%s/%s Gateway not Programmed yet", ns, gw.Name)
   230  					return false, nil
   231  				}
   232  			}
   233  
   234  			podList := &v1.PodList{}
   235  			err = c.List(ctx, podList, client.InNamespace(ns))
   236  			if err != nil {
   237  				t.Errorf("Error listing Pods: %v", err)
   238  			}
   239  			for _, pod := range podList.Items {
   240  				if !findPodConditionInList(t, pod.Status.Conditions, "Ready", "True") &&
   241  					pod.Status.Phase != v1.PodSucceeded &&
   242  					pod.DeletionTimestamp == nil {
   243  					t.Logf("%s/%s Pod not ready yet", ns, pod.Name)
   244  					return false, nil
   245  				}
   246  			}
   247  		}
   248  		t.Logf("Gateways and Pods in %s namespaces ready", strings.Join(namespaces, ", "))
   249  		return true, nil
   250  	})
   251  	require.NoErrorf(t, waitErr, "error waiting for %s namespaces to be ready", strings.Join(namespaces, ", "))
   252  }
   253  
   254  // GatewayMustHaveCondition checks that the supplied Gateway has the supplied Condition,
   255  // halting after the specified timeout is exceeded.
   256  func GatewayMustHaveCondition(
   257  	t *testing.T,
   258  	client client.Client,
   259  	timeoutConfig config.TimeoutConfig,
   260  	gwNN types.NamespacedName,
   261  	expectedCondition metav1.Condition,
   262  ) {
   263  	t.Helper()
   264  
   265  	waitErr := wait.PollUntilContextTimeout(
   266  		context.Background(),
   267  		1*time.Second,
   268  		timeoutConfig.GatewayMustHaveCondition,
   269  		true,
   270  		func(ctx context.Context) (bool, error) {
   271  			gw := &gatewayv1.Gateway{}
   272  			err := client.Get(ctx, gwNN, gw)
   273  			if err != nil {
   274  				return false, fmt.Errorf("error fetching Gateway: %w", err)
   275  			}
   276  
   277  			if err := ConditionsHaveLatestObservedGeneration(gw, gw.Status.Conditions); err != nil {
   278  				return false, err
   279  			}
   280  
   281  			if findConditionInList(t,
   282  				gw.Status.Conditions,
   283  				expectedCondition.Type,
   284  				string(expectedCondition.Status),
   285  				expectedCondition.Reason,
   286  			) {
   287  				return true, nil
   288  			}
   289  
   290  			return false, nil
   291  		},
   292  	)
   293  
   294  	require.NoErrorf(t, waitErr, "error waiting for Gateway status to have a Condition matching expectations")
   295  }
   296  
   297  // MeshNamespacesMustBeReady waits until all Pods are marked Ready. This is
   298  // intended to be used for mesh tests and does not require any Gateways to
   299  // exist. This will cause the test to halt if the specified timeout is exceeded.
   300  func MeshNamespacesMustBeReady(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, namespaces []string) {
   301  	t.Helper()
   302  
   303  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.NamespacesMustBeReady, true, func(ctx context.Context) (bool, error) {
   304  		for _, ns := range namespaces {
   305  			podList := &v1.PodList{}
   306  			err := c.List(ctx, podList, client.InNamespace(ns))
   307  			if err != nil {
   308  				t.Errorf("Error listing Pods: %v", err)
   309  			}
   310  			for _, pod := range podList.Items {
   311  				if !findPodConditionInList(t, pod.Status.Conditions, "Ready", "True") &&
   312  					pod.Status.Phase != v1.PodSucceeded &&
   313  					pod.DeletionTimestamp == nil {
   314  					t.Logf("%s/%s Pod not ready yet", ns, pod.Name)
   315  					return false, nil
   316  				}
   317  			}
   318  		}
   319  		t.Logf("Pods in %s namespaces ready", strings.Join(namespaces, ", "))
   320  		return true, nil
   321  	})
   322  	require.NoErrorf(t, waitErr, "error waiting for %s namespaces to be ready", strings.Join(namespaces, ", "))
   323  }
   324  
   325  // GatewayAndHTTPRoutesMustBeAccepted waits until:
   326  //  1. The specified Gateway has an IP address assigned to it.
   327  //  2. The route has a ParentRef referring to the Gateway.
   328  //  3. All the gateway's listeners have the following conditions set to true:
   329  //     - ListenerConditionResolvedRefs
   330  //     - ListenerConditionAccepted
   331  //     - ListenerConditionProgrammed
   332  //
   333  // The test will fail if these conditions are not met before the timeouts.
   334  func GatewayAndHTTPRoutesMustBeAccepted(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, controllerName string, gw GatewayRef, routeNNs ...types.NamespacedName) string {
   335  	t.Helper()
   336  
   337  	gwAddr, err := WaitForGatewayAddress(t, c, timeoutConfig, gw.NamespacedName)
   338  	require.NoErrorf(t, err, "timed out waiting for Gateway address to be assigned")
   339  
   340  	ns := gatewayv1.Namespace(gw.Namespace)
   341  	kind := gatewayv1.Kind("Gateway")
   342  
   343  	for _, routeNN := range routeNNs {
   344  		namespaceRequired := true
   345  		if routeNN.Namespace == gw.Namespace {
   346  			namespaceRequired = false
   347  		}
   348  
   349  		var parents []gatewayv1.RouteParentStatus
   350  		for _, listener := range gw.listenerNames {
   351  			parents = append(parents, gatewayv1.RouteParentStatus{
   352  				ParentRef: gatewayv1.ParentReference{
   353  					Group:       (*gatewayv1.Group)(&gatewayv1.GroupVersion.Group),
   354  					Kind:        &kind,
   355  					Name:        gatewayv1.ObjectName(gw.Name),
   356  					Namespace:   &ns,
   357  					SectionName: listener,
   358  				},
   359  				ControllerName: gatewayv1.GatewayController(controllerName),
   360  				Conditions: []metav1.Condition{{
   361  					Type:   string(gatewayv1.RouteConditionAccepted),
   362  					Status: metav1.ConditionTrue,
   363  					Reason: string(gatewayv1.RouteReasonAccepted),
   364  				}},
   365  			})
   366  		}
   367  		HTTPRouteMustHaveParents(t, c, timeoutConfig, routeNN, parents, namespaceRequired)
   368  	}
   369  
   370  	requiredListenerConditions := []metav1.Condition{
   371  		{
   372  			Type:   string(gatewayv1.ListenerConditionResolvedRefs),
   373  			Status: metav1.ConditionTrue,
   374  			Reason: "", // any reason
   375  		},
   376  		{
   377  			Type:   string(gatewayv1.ListenerConditionAccepted),
   378  			Status: metav1.ConditionTrue,
   379  			Reason: "", // any reason
   380  		},
   381  		{
   382  			Type:   string(gatewayv1.ListenerConditionProgrammed),
   383  			Status: metav1.ConditionTrue,
   384  			Reason: "", // any reason
   385  		},
   386  	}
   387  	GatewayListenersMustHaveConditions(t, c, timeoutConfig, gw.NamespacedName, requiredListenerConditions)
   388  
   389  	return gwAddr
   390  }
   391  
   392  // WaitForGatewayAddress waits until at least one IP Address has been set in the
   393  // status of the specified Gateway.
   394  func WaitForGatewayAddress(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, gwName types.NamespacedName) (string, error) {
   395  	t.Helper()
   396  
   397  	var ipAddr, port string
   398  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.GatewayMustHaveAddress, true, func(ctx context.Context) (bool, error) {
   399  		gw := &gatewayv1.Gateway{}
   400  		err := client.Get(ctx, gwName, gw)
   401  		if err != nil {
   402  			t.Logf("error fetching Gateway: %v", err)
   403  			return false, fmt.Errorf("error fetching Gateway: %w", err)
   404  		}
   405  
   406  		if err := ConditionsHaveLatestObservedGeneration(gw, gw.Status.Conditions); err != nil {
   407  			t.Log("Gateway", err)
   408  			return false, nil
   409  		}
   410  
   411  		port = strconv.FormatInt(int64(gw.Spec.Listeners[0].Port), 10)
   412  
   413  		// TODO: Support more than IPAddress
   414  		for _, address := range gw.Status.Addresses {
   415  			if address.Type != nil && *address.Type == gatewayv1.IPAddressType {
   416  				ipAddr = address.Value
   417  				return true, nil
   418  			}
   419  		}
   420  
   421  		return false, nil
   422  	})
   423  	require.NoErrorf(t, waitErr, "error waiting for Gateway to have at least one IP address in status")
   424  	return net.JoinHostPort(ipAddr, port), waitErr
   425  }
   426  
   427  // GatewayListenersMustHaveConditions checks if every listener of the specified gateway has all
   428  // the specified conditions.
   429  func GatewayListenersMustHaveConditions(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, gwName types.NamespacedName, conditions []metav1.Condition) {
   430  	t.Helper()
   431  
   432  	var wg sync.WaitGroup
   433  	wg.Add(len(conditions))
   434  
   435  	for _, condition := range conditions {
   436  		go func(condition metav1.Condition) {
   437  			defer wg.Done()
   438  
   439  			waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.GatewayListenersMustHaveCondition, true, func(ctx context.Context) (bool, error) {
   440  				var gw gatewayv1.Gateway
   441  				if err := client.Get(ctx, gwName, &gw); err != nil {
   442  					return false, fmt.Errorf("error fetching Gateway: %w", err)
   443  				}
   444  
   445  				for _, listener := range gw.Status.Listeners {
   446  					if !findConditionInList(t, listener.Conditions, condition.Type, string(condition.Status), condition.Reason) {
   447  						return false, nil
   448  					}
   449  				}
   450  
   451  				return true, nil
   452  			})
   453  
   454  			require.NoErrorf(t, waitErr, "error waiting for Gateway status to have the %s condition set to %s on all listeners",
   455  				condition.Type, condition.Status)
   456  		}(condition)
   457  	}
   458  
   459  	wg.Wait()
   460  }
   461  
   462  // GatewayMustHaveZeroRoutes validates that the gateway has zero routes attached.  The status
   463  // may indicate a single listener with zero attached routes or no listeners.
   464  func GatewayMustHaveZeroRoutes(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, gwName types.NamespacedName) {
   465  	var gotStatus *gatewayv1.GatewayStatus
   466  
   467  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.GatewayStatusMustHaveListeners, true, func(ctx context.Context) (bool, error) {
   468  		gw := &gatewayv1.Gateway{}
   469  
   470  		err := client.Get(ctx, gwName, gw)
   471  		require.NoError(t, err, "error fetching Gateway")
   472  
   473  		if err := ConditionsHaveLatestObservedGeneration(gw, gw.Status.Conditions); err != nil {
   474  			t.Log("Gateway ", err)
   475  			return false, nil
   476  		}
   477  
   478  		// There are two valid ways to represent this:
   479  		// 1. No listeners in status
   480  		// 2. One listener in status with 0 attached routes
   481  		if len(gw.Status.Listeners) == 0 {
   482  			// No listeners in status.
   483  			return true, nil
   484  		}
   485  		if len(gw.Status.Listeners) == 1 && gw.Status.Listeners[0].AttachedRoutes == 0 {
   486  			// One listener with zero attached routes.
   487  			return true, nil
   488  		}
   489  		gotStatus = &gw.Status
   490  		return false, nil
   491  	})
   492  	if waitErr != nil {
   493  		t.Errorf("Error waiting for gateway, got Gateway Status %v, want zero listeners or exactly 1 listener with zero routes", gotStatus)
   494  	}
   495  }
   496  
   497  // HTTPRouteMustHaveNoAcceptedParents waits for the specified HTTPRoute to have either no parents
   498  // or a single parent that is not accepted. This is used to validate HTTPRoute errors.
   499  func HTTPRouteMustHaveNoAcceptedParents(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, routeName types.NamespacedName) {
   500  	t.Helper()
   501  
   502  	var actual []gatewayv1.RouteParentStatus
   503  	emptyChecked := false
   504  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.HTTPRouteMustNotHaveParents, true, func(ctx context.Context) (bool, error) {
   505  		route := &gatewayv1.HTTPRoute{}
   506  		err := client.Get(ctx, routeName, route)
   507  		if err != nil {
   508  			return false, fmt.Errorf("error fetching HTTPRoute: %w", err)
   509  		}
   510  
   511  		actual = route.Status.Parents
   512  
   513  		if len(actual) == 0 {
   514  			// For empty status, we need to distinguish between "correctly did not set" and "hasn't set yet"
   515  			// Ensure we iterate at least two times (taking advantage of the 1s poll delay) to give it some time.
   516  			if !emptyChecked {
   517  				emptyChecked = true
   518  				return false, nil
   519  			}
   520  			return true, nil
   521  		}
   522  		if len(actual) > 1 {
   523  			// Only expect one parent
   524  			return false, nil
   525  		}
   526  
   527  		for _, parent := range actual {
   528  			if err := ConditionsHaveLatestObservedGeneration(route, parent.Conditions); err != nil {
   529  				t.Logf("HTTPRoute(controller=%v,ref=%#v) %v", parent.ControllerName, parent, err)
   530  				return false, nil
   531  			}
   532  		}
   533  
   534  		return conditionsMatch(t, []metav1.Condition{{
   535  			Type:   string(gatewayv1.RouteConditionAccepted),
   536  			Status: "False",
   537  		}}, actual[0].Conditions), nil
   538  	})
   539  	require.NoErrorf(t, waitErr, "error waiting for HTTPRoute to have no accepted parents")
   540  }
   541  
   542  // HTTPRouteMustHaveParents waits for the specified HTTPRoute to have parents
   543  // in status that match the expected parents. This will cause the test to halt
   544  // if the specified timeout is exceeded.
   545  func HTTPRouteMustHaveParents(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, routeName types.NamespacedName, parents []gatewayv1.RouteParentStatus, namespaceRequired bool) {
   546  	t.Helper()
   547  
   548  	var actual []gatewayv1.RouteParentStatus
   549  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.RouteMustHaveParents, true, func(ctx context.Context) (bool, error) {
   550  		route := &gatewayv1.HTTPRoute{}
   551  		err := client.Get(ctx, routeName, route)
   552  		if err != nil {
   553  			return false, fmt.Errorf("error fetching HTTPRoute: %w", err)
   554  		}
   555  
   556  		for _, parent := range actual {
   557  			if err := ConditionsHaveLatestObservedGeneration(route, parent.Conditions); err != nil {
   558  				t.Logf("HTTPRoute(controller=%v,ref=%#v) %v", parent.ControllerName, parent, err)
   559  				return false, nil
   560  			}
   561  		}
   562  
   563  		actual = route.Status.Parents
   564  		return parentsForRouteMatch(t, routeName, parents, actual, namespaceRequired), nil
   565  	})
   566  	require.NoErrorf(t, waitErr, "error waiting for HTTPRoute to have parents matching expectations")
   567  }
   568  
   569  // TLSRouteMustHaveParents waits for the specified TLSRoute to have parents
   570  // in status that match the expected parents, and also returns the TLSRoute.
   571  // This will cause the test to halt if the specified timeout is exceeded.
   572  func TLSRouteMustHaveParents(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, routeName types.NamespacedName, parents []v1alpha2.RouteParentStatus, namespaceRequired bool) v1alpha2.TLSRoute {
   573  	t.Helper()
   574  
   575  	var actual []gatewayv1.RouteParentStatus
   576  	var route v1alpha2.TLSRoute
   577  
   578  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.RouteMustHaveParents, true, func(ctx context.Context) (bool, error) {
   579  		err := client.Get(ctx, routeName, &route)
   580  		if err != nil {
   581  			return false, fmt.Errorf("error fetching TLSRoute: %w", err)
   582  		}
   583  		actual = route.Status.Parents
   584  		match := parentsForRouteMatch(t, routeName, parents, actual, namespaceRequired)
   585  
   586  		return match, nil
   587  	})
   588  	require.NoErrorf(t, waitErr, "error waiting for TLSRoute to have parents matching expectations")
   589  
   590  	return route
   591  }
   592  
   593  func parentsForRouteMatch(t *testing.T, routeName types.NamespacedName, expected, actual []gatewayv1.RouteParentStatus, namespaceRequired bool) bool {
   594  	t.Helper()
   595  
   596  	if len(expected) != len(actual) {
   597  		t.Logf("Route %s/%s expected %d Parents got %d", routeName.Namespace, routeName.Name, len(expected), len(actual))
   598  		return false
   599  	}
   600  
   601  	// TODO(robscott): Allow for arbitrarily ordered parents
   602  	for i, eParent := range expected {
   603  		aParent := actual[i]
   604  		if aParent.ControllerName != eParent.ControllerName {
   605  			t.Logf("Route %s/%s ControllerName doesn't match", routeName.Namespace, routeName.Name)
   606  			return false
   607  		}
   608  		if !reflect.DeepEqual(aParent.ParentRef.Group, eParent.ParentRef.Group) {
   609  			t.Logf("Route %s/%s expected ParentReference.Group to be %v, got %v", routeName.Namespace, routeName.Name, eParent.ParentRef.Group, aParent.ParentRef.Group)
   610  			return false
   611  		}
   612  		if !reflect.DeepEqual(aParent.ParentRef.Kind, eParent.ParentRef.Kind) {
   613  			t.Logf("Route %s/%s expected ParentReference.Kind to be %v, got %v", routeName.Namespace, routeName.Name, eParent.ParentRef.Kind, aParent.ParentRef.Kind)
   614  			return false
   615  		}
   616  		if aParent.ParentRef.Name != eParent.ParentRef.Name {
   617  			t.Logf("Route %s/%s ParentReference.Name doesn't match", routeName.Namespace, routeName.Name)
   618  			return false
   619  		}
   620  		if !reflect.DeepEqual(aParent.ParentRef.Namespace, eParent.ParentRef.Namespace) {
   621  			if namespaceRequired || aParent.ParentRef.Namespace != nil {
   622  				t.Logf("Route %s/%s expected ParentReference.Namespace to be %v, got %v", routeName.Namespace, routeName.Name, eParent.ParentRef.Namespace, aParent.ParentRef.Namespace)
   623  				return false
   624  			}
   625  		}
   626  		if !conditionsMatch(t, eParent.Conditions, aParent.Conditions) {
   627  			return false
   628  		}
   629  	}
   630  
   631  	t.Logf("Route %s/%s Parents matched expectations", routeName.Namespace, routeName.Name)
   632  	return true
   633  }
   634  
   635  // GatewayStatusMustHaveListeners waits for the specified Gateway to have listeners
   636  // in status that match the expected listeners. This will cause the test to halt
   637  // if the specified timeout is exceeded.
   638  func GatewayStatusMustHaveListeners(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, gwNN types.NamespacedName, listeners []gatewayv1.ListenerStatus) {
   639  	t.Helper()
   640  
   641  	var actual []gatewayv1.ListenerStatus
   642  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.GatewayStatusMustHaveListeners, true, func(ctx context.Context) (bool, error) {
   643  		gw := &gatewayv1.Gateway{}
   644  		err := client.Get(ctx, gwNN, gw)
   645  		if err != nil {
   646  			return false, fmt.Errorf("error fetching Gateway: %w", err)
   647  		}
   648  
   649  		if err := ConditionsHaveLatestObservedGeneration(gw, gw.Status.Conditions); err != nil {
   650  			t.Log("Gateway", err)
   651  			return false, nil
   652  		}
   653  
   654  		actual = gw.Status.Listeners
   655  		return listenersMatch(t, listeners, actual), nil
   656  	})
   657  	require.NoErrorf(t, waitErr, "error waiting for Gateway status to have listeners matching expectations")
   658  }
   659  
   660  // HTTPRouteMustHaveCondition checks that the supplied HTTPRoute has the supplied Condition,
   661  // halting after the specified timeout is exceeded.
   662  func HTTPRouteMustHaveCondition(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, routeNN types.NamespacedName, gwNN types.NamespacedName, condition metav1.Condition) {
   663  	t.Helper()
   664  
   665  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.HTTPRouteMustHaveCondition, true, func(ctx context.Context) (bool, error) {
   666  		route := &gatewayv1.HTTPRoute{}
   667  		err := client.Get(ctx, routeNN, route)
   668  		if err != nil {
   669  			return false, fmt.Errorf("error fetching HTTPRoute: %w", err)
   670  		}
   671  
   672  		parents := route.Status.Parents
   673  		var conditionFound bool
   674  		for _, parent := range parents {
   675  			if err := ConditionsHaveLatestObservedGeneration(route, parent.Conditions); err != nil {
   676  				t.Logf("HTTPRoute(parentRef=%v) %v", parentRefToString(parent.ParentRef), err)
   677  				return false, nil
   678  			}
   679  
   680  			if parent.ParentRef.Name == gatewayv1.ObjectName(gwNN.Name) && (parent.ParentRef.Namespace == nil || string(*parent.ParentRef.Namespace) == gwNN.Namespace) {
   681  				if findConditionInList(t, parent.Conditions, condition.Type, string(condition.Status), condition.Reason) {
   682  					conditionFound = true
   683  				}
   684  			}
   685  		}
   686  
   687  		return conditionFound, nil
   688  	})
   689  
   690  	require.NoErrorf(t, waitErr, "error waiting for HTTPRoute status to have a Condition matching expectations")
   691  }
   692  
   693  // HTTPRouteMustHaveResolvedRefsConditionsTrue checks that the supplied HTTPRoute has the resolvedRefsCondition
   694  // set to true.
   695  func HTTPRouteMustHaveResolvedRefsConditionsTrue(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, routeNN types.NamespacedName, gwNN types.NamespacedName) {
   696  	HTTPRouteMustHaveCondition(t, client, timeoutConfig, routeNN, gwNN, metav1.Condition{
   697  		Type:   string(gatewayv1.RouteConditionResolvedRefs),
   698  		Status: metav1.ConditionTrue,
   699  		Reason: string(gatewayv1.RouteReasonResolvedRefs),
   700  	})
   701  }
   702  
   703  func parentRefToString(p gatewayv1.ParentReference) string {
   704  	if p.Namespace != nil && *p.Namespace != "" {
   705  		return fmt.Sprintf("%v/%v", p.Namespace, p.Name)
   706  	}
   707  	return string(p.Name)
   708  }
   709  
   710  // GatewayAndTLSRoutesMustBeAccepted waits until the specified Gateway has an IP
   711  // address assigned to it and the TLSRoute has a ParentRef referring to the
   712  // Gateway. The test will fail if these conditions are not met before the
   713  // timeouts.
   714  func GatewayAndTLSRoutesMustBeAccepted(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, controllerName string, gw GatewayRef, routeNNs ...types.NamespacedName) (string, []gatewayv1.Hostname) {
   715  	t.Helper()
   716  
   717  	var hostnames []gatewayv1.Hostname
   718  
   719  	gwAddr, err := WaitForGatewayAddress(t, c, timeoutConfig, gw.NamespacedName)
   720  	require.NoErrorf(t, err, "timed out waiting for Gateway address to be assigned")
   721  
   722  	ns := gatewayv1.Namespace(gw.Namespace)
   723  	kind := gatewayv1.Kind("Gateway")
   724  
   725  	for _, routeNN := range routeNNs {
   726  		namespaceRequired := true
   727  		if routeNN.Namespace == gw.Namespace {
   728  			namespaceRequired = false
   729  		}
   730  
   731  		var parents []gatewayv1.RouteParentStatus
   732  		for _, listener := range gw.listenerNames {
   733  			parents = append(parents, gatewayv1.RouteParentStatus{
   734  				ParentRef: gatewayv1.ParentReference{
   735  					Group:       (*gatewayv1.Group)(&gatewayv1.GroupVersion.Group),
   736  					Kind:        &kind,
   737  					Name:        gatewayv1.ObjectName(gw.Name),
   738  					Namespace:   &ns,
   739  					SectionName: listener,
   740  				},
   741  				ControllerName: gatewayv1.GatewayController(controllerName),
   742  				Conditions: []metav1.Condition{
   743  					{
   744  						Type:   string(gatewayv1.RouteConditionAccepted),
   745  						Status: metav1.ConditionTrue,
   746  						Reason: string(gatewayv1.RouteReasonAccepted),
   747  					},
   748  				},
   749  			})
   750  		}
   751  		route := TLSRouteMustHaveParents(t, c, timeoutConfig, routeNN, parents, namespaceRequired)
   752  		hostnames = route.Spec.Hostnames
   753  	}
   754  
   755  	return gwAddr, hostnames
   756  }
   757  
   758  // TLSRouteMustHaveCondition checks that the supplied TLSRoute has the supplied Condition,
   759  // halting after the specified timeout is exceeded.
   760  func TLSRouteMustHaveCondition(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, routeNN types.NamespacedName, gwNN types.NamespacedName, condition metav1.Condition) {
   761  	t.Helper()
   762  
   763  	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeoutConfig.TLSRouteMustHaveCondition, true, func(ctx context.Context) (bool, error) {
   764  		route := &v1alpha2.TLSRoute{}
   765  		err := client.Get(ctx, routeNN, route)
   766  		if err != nil {
   767  			return false, fmt.Errorf("error fetching TLSRoute: %w", err)
   768  		}
   769  
   770  		parents := route.Status.Parents
   771  		var conditionFound bool
   772  		for _, parent := range parents {
   773  			if err := ConditionsHaveLatestObservedGeneration(route, parent.Conditions); err != nil {
   774  				t.Logf("TLSRoute(parentRef=%v) %v", parentRefToString(parent.ParentRef), err)
   775  				return false, nil
   776  			}
   777  
   778  			if parent.ParentRef.Name == gatewayv1.ObjectName(gwNN.Name) && (parent.ParentRef.Namespace == nil || string(*parent.ParentRef.Namespace) == gwNN.Namespace) {
   779  				if findConditionInList(t, parent.Conditions, condition.Type, string(condition.Status), condition.Reason) {
   780  					conditionFound = true
   781  				}
   782  			}
   783  		}
   784  
   785  		return conditionFound, nil
   786  	})
   787  
   788  	require.NoErrorf(t, waitErr, "error waiting for TLSRoute status to have a Condition matching expectations")
   789  }
   790  
   791  // TODO(mikemorris): this and parentsMatch could possibly be rewritten as a generic function?
   792  func listenersMatch(t *testing.T, expected, actual []gatewayv1.ListenerStatus) bool {
   793  	t.Helper()
   794  
   795  	if len(expected) != len(actual) {
   796  		t.Logf("Expected %d Gateway status listeners, got %d", len(expected), len(actual))
   797  		return false
   798  	}
   799  
   800  	for _, eListener := range expected {
   801  		var aListener *gatewayv1.ListenerStatus
   802  		for i := range actual {
   803  			if actual[i].Name == eListener.Name {
   804  				aListener = &actual[i]
   805  				break
   806  			}
   807  		}
   808  		if aListener == nil {
   809  			t.Logf("Expected status for listener %s to be present", eListener.Name)
   810  			return false
   811  		}
   812  
   813  		if len(eListener.SupportedKinds) == 0 && len(aListener.SupportedKinds) != 0 {
   814  			t.Logf("Expected list of SupportedKinds was empty, but the actual list for comparison was not:  %v",
   815  				aListener.SupportedKinds)
   816  			return false
   817  		}
   818  		// Ensure that the expected Listener.SupportedKinds items are present in actual Listener.SupportedKinds
   819  		// Find the items instead of performing an exact match of the slice because the implementation
   820  		// might support more Kinds than defined in the test
   821  		for _, eKind := range eListener.SupportedKinds {
   822  			found := false
   823  
   824  			for _, aKind := range aListener.SupportedKinds {
   825  				if eKind.Group == nil {
   826  					eKind.Group = (*gatewayv1.Group)(&gatewayv1.GroupVersion.Group)
   827  				}
   828  
   829  				if aKind.Group == nil {
   830  					aKind.Group = (*gatewayv1.Group)(&gatewayv1.GroupVersion.Group)
   831  				}
   832  
   833  				if *eKind.Group == *aKind.Group && eKind.Kind == aKind.Kind {
   834  					found = true
   835  					break
   836  				}
   837  			}
   838  			if !found {
   839  				t.Logf("Expected Group:%s Kind:%s to be present in SupportedKinds", *eKind.Group, eKind.Kind)
   840  				return false
   841  			}
   842  		}
   843  
   844  		if aListener.AttachedRoutes != eListener.AttachedRoutes {
   845  			t.Logf("Expected AttachedRoutes to be %v, got %v", eListener.AttachedRoutes, aListener.AttachedRoutes)
   846  			return false
   847  		}
   848  		if !conditionsMatch(t, eListener.Conditions, aListener.Conditions) {
   849  			t.Logf("Expected Conditions to be %v, got %v", eListener.Conditions, aListener.Conditions)
   850  			return false
   851  		}
   852  	}
   853  
   854  	t.Logf("Gateway status listeners matched expectations")
   855  	return true
   856  }
   857  
   858  func conditionsMatch(t *testing.T, expected, actual []metav1.Condition) bool {
   859  	t.Helper()
   860  
   861  	if len(actual) < len(expected) {
   862  		t.Logf("Expected more conditions to be present")
   863  		return false
   864  	}
   865  	for _, condition := range expected {
   866  		if !findConditionInList(t, actual, condition.Type, string(condition.Status), condition.Reason) {
   867  			return false
   868  		}
   869  	}
   870  
   871  	t.Logf("Conditions matched expectations")
   872  	return true
   873  }
   874  
   875  // findConditionInList finds a condition in a list of Conditions, checking
   876  // the Name, Value, and Reason. If an empty reason is passed, any Reason will match.
   877  // If an empty status is passed, any Status will match.
   878  func findConditionInList(t *testing.T, conditions []metav1.Condition, condName, expectedStatus, expectedReason string) bool {
   879  	t.Helper()
   880  
   881  	for _, cond := range conditions {
   882  		if cond.Type == condName {
   883  			// an empty Status string means "Match any status".
   884  			if expectedStatus == "" || cond.Status == metav1.ConditionStatus(expectedStatus) {
   885  				// an empty Reason string means "Match any reason".
   886  				if expectedReason == "" || cond.Reason == expectedReason {
   887  					return true
   888  				}
   889  				t.Logf("%s condition Reason set to %s, expected %s", condName, cond.Reason, expectedReason)
   890  			}
   891  
   892  			t.Logf("%s condition set to Status %s with Reason %v, expected Status %s", condName, cond.Status, cond.Reason, expectedStatus)
   893  		}
   894  	}
   895  
   896  	t.Logf("%s was not in conditions list [%v]", condName, conditions)
   897  	return false
   898  }
   899  
   900  func findPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName, condValue string) bool {
   901  	t.Helper()
   902  
   903  	for _, cond := range conditions {
   904  		if cond.Type == v1.PodConditionType(condName) {
   905  			if cond.Status == v1.ConditionStatus(condValue) {
   906  				return true
   907  			}
   908  			t.Logf("%s condition set to %s, expected %s", condName, cond.Status, condValue)
   909  		}
   910  	}
   911  
   912  	t.Logf("%s was not in conditions list", condName)
   913  	return false
   914  }
   915
View as plain text