suite.go

Documentation: sigs.k8s.io/gateway-api/conformance/utils/suite

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package suite
    18  
    19  import (
    20  	"embed"
    21  	"strings"
    22  	"testing"
    23  
    24  	"k8s.io/apimachinery/pkg/util/sets"
    25  	clientset "k8s.io/client-go/kubernetes"
    26  	"k8s.io/client-go/rest"
    27  	"sigs.k8s.io/controller-runtime/pkg/client"
    28  
    29  	"sigs.k8s.io/gateway-api/apis/v1beta1"
    30  	"sigs.k8s.io/gateway-api/conformance"
    31  	"sigs.k8s.io/gateway-api/conformance/utils/config"
    32  	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
    33  	"sigs.k8s.io/gateway-api/conformance/utils/roundtripper"
    34  )
    35  
    36  // ConformanceTestSuite defines the test suite used to run Gateway API
    37  // conformance tests.
    38  type ConformanceTestSuite struct {
    39  	Client                   client.Client
    40  	Clientset                clientset.Interface
    41  	RESTClient               *rest.RESTClient
    42  	RestConfig               *rest.Config
    43  	RoundTripper             roundtripper.RoundTripper
    44  	GatewayClassName         string
    45  	ControllerName           string
    46  	Debug                    bool
    47  	Cleanup                  bool
    48  	BaseManifests            string
    49  	MeshManifests            string
    50  	Applier                  kubernetes.Applier
    51  	SupportedFeatures        sets.Set[SupportedFeature]
    52  	TimeoutConfig            config.TimeoutConfig
    53  	SkipTests                sets.Set[string]
    54  	RunTest                  string
    55  	FS                       embed.FS
    56  	UsableNetworkAddresses   []v1beta1.GatewayAddress
    57  	UnusableNetworkAddresses []v1beta1.GatewayAddress
    58  }
    59  
    60  // Options can be used to initialize a ConformanceTestSuite.
    61  type Options struct {
    62  	Client               client.Client
    63  	Clientset            clientset.Interface
    64  	RestConfig           *rest.Config
    65  	GatewayClassName     string
    66  	Debug                bool
    67  	RoundTripper         roundtripper.RoundTripper
    68  	BaseManifests        string
    69  	MeshManifests        string
    70  	NamespaceLabels      map[string]string
    71  	NamespaceAnnotations map[string]string
    72  
    73  	// CleanupBaseResources indicates whether or not the base test
    74  	// resources such as Gateways should be cleaned up after the run.
    75  	CleanupBaseResources       bool
    76  	SupportedFeatures          sets.Set[SupportedFeature]
    77  	ExemptFeatures             sets.Set[SupportedFeature]
    78  	EnableAllSupportedFeatures bool
    79  	TimeoutConfig              config.TimeoutConfig
    80  	// SkipTests contains all the tests not to be run and can be used to opt out
    81  	// of specific tests
    82  	SkipTests []string
    83  	// RunTest is a single test to run, mostly for development/debugging convenience.
    84  	RunTest string
    85  
    86  	FS *embed.FS
    87  
    88  	// UsableNetworkAddresses is an optional pool of usable addresses for
    89  	// Gateways for tests which need to test manual address assignments.
    90  	UsableNetworkAddresses []v1beta1.GatewayAddress
    91  
    92  	// UnusableNetworkAddresses is an optional pool of unusable addresses for
    93  	// Gateways for tests which need to test failures with manual Gateway
    94  	// address assignment.
    95  	UnusableNetworkAddresses []v1beta1.GatewayAddress
    96  }
    97  
    98  // New returns a new ConformanceTestSuite.
    99  func New(s Options) *ConformanceTestSuite {
   100  	config.SetupTimeoutConfig(&s.TimeoutConfig)
   101  
   102  	roundTripper := s.RoundTripper
   103  	if roundTripper == nil {
   104  		roundTripper = &roundtripper.DefaultRoundTripper{Debug: s.Debug, TimeoutConfig: s.TimeoutConfig}
   105  	}
   106  
   107  	switch {
   108  	case s.EnableAllSupportedFeatures:
   109  		s.SupportedFeatures = AllFeatures
   110  	case s.SupportedFeatures == nil:
   111  		s.SupportedFeatures = GatewayCoreFeatures
   112  	default:
   113  		for feature := range GatewayCoreFeatures {
   114  			s.SupportedFeatures.Insert(feature)
   115  		}
   116  	}
   117  
   118  	for feature := range s.ExemptFeatures {
   119  		s.SupportedFeatures.Delete(feature)
   120  	}
   121  
   122  	if s.FS == nil {
   123  		s.FS = &conformance.Manifests
   124  	}
   125  
   126  	suite := &ConformanceTestSuite{
   127  		Client:           s.Client,
   128  		Clientset:        s.Clientset,
   129  		RestConfig:       s.RestConfig,
   130  		RoundTripper:     roundTripper,
   131  		GatewayClassName: s.GatewayClassName,
   132  		Debug:            s.Debug,
   133  		Cleanup:          s.CleanupBaseResources,
   134  		BaseManifests:    s.BaseManifests,
   135  		MeshManifests:    s.MeshManifests,
   136  		Applier: kubernetes.Applier{
   137  			NamespaceLabels:      s.NamespaceLabels,
   138  			NamespaceAnnotations: s.NamespaceAnnotations,
   139  		},
   140  		SupportedFeatures:        s.SupportedFeatures,
   141  		TimeoutConfig:            s.TimeoutConfig,
   142  		SkipTests:                sets.New(s.SkipTests...),
   143  		RunTest:                  s.RunTest,
   144  		FS:                       *s.FS,
   145  		UsableNetworkAddresses:   s.UsableNetworkAddresses,
   146  		UnusableNetworkAddresses: s.UnusableNetworkAddresses,
   147  	}
   148  
   149  	// apply defaults
   150  	if suite.BaseManifests == "" {
   151  		suite.BaseManifests = "base/manifests.yaml"
   152  	}
   153  	if suite.MeshManifests == "" {
   154  		suite.MeshManifests = "mesh/manifests.yaml"
   155  	}
   156  
   157  	return suite
   158  }
   159  
   160  // Setup ensures the base resources required for conformance tests are installed
   161  // in the cluster. It also ensures that all relevant resources are ready.
   162  func (suite *ConformanceTestSuite) Setup(t *testing.T) {
   163  	suite.Applier.FS = suite.FS
   164  	suite.Applier.UsableNetworkAddresses = suite.UsableNetworkAddresses
   165  	suite.Applier.UnusableNetworkAddresses = suite.UnusableNetworkAddresses
   166  
   167  	if suite.SupportedFeatures.Has(SupportGateway) {
   168  		t.Logf("Test Setup: Ensuring GatewayClass has been accepted")
   169  		suite.ControllerName = kubernetes.GWCMustHaveAcceptedConditionTrue(t, suite.Client, suite.TimeoutConfig, suite.GatewayClassName)
   170  
   171  		suite.Applier.GatewayClass = suite.GatewayClassName
   172  		suite.Applier.ControllerName = suite.ControllerName
   173  
   174  		t.Logf("Test Setup: Applying base manifests")
   175  		suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, suite.BaseManifests, suite.Cleanup)
   176  
   177  		t.Logf("Test Setup: Applying programmatic resources")
   178  		secret := kubernetes.MustCreateSelfSignedCertSecret(t, "gateway-conformance-web-backend", "certificate", []string{"*"})
   179  		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
   180  		secret = kubernetes.MustCreateSelfSignedCertSecret(t, "gateway-conformance-infra", "tls-validity-checks-certificate", []string{"*", "*.org"})
   181  		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
   182  		secret = kubernetes.MustCreateSelfSignedCertSecret(t, "gateway-conformance-infra", "tls-passthrough-checks-certificate", []string{"abc.example.com"})
   183  		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
   184  		secret = kubernetes.MustCreateSelfSignedCertSecret(t, "gateway-conformance-app-backend", "tls-passthrough-checks-certificate", []string{"abc.example.com"})
   185  		suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
   186  
   187  		t.Logf("Test Setup: Ensuring Gateways and Pods from base manifests are ready")
   188  		namespaces := []string{
   189  			"gateway-conformance-infra",
   190  			"gateway-conformance-app-backend",
   191  			"gateway-conformance-web-backend",
   192  		}
   193  		kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, namespaces)
   194  	}
   195  	if suite.SupportedFeatures.Has(SupportMesh) {
   196  		t.Logf("Test Setup: Applying base manifests")
   197  		suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, suite.MeshManifests, suite.Cleanup)
   198  		t.Logf("Test Setup: Ensuring Gateways and Pods from mesh manifests are ready")
   199  		namespaces := []string{
   200  			"gateway-conformance-mesh",
   201  			"gateway-conformance-mesh-consumer",
   202  			"gateway-conformance-app-backend",
   203  			"gateway-conformance-web-backend",
   204  		}
   205  		kubernetes.MeshNamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, namespaces)
   206  	}
   207  }
   208  
   209  // Run runs the provided set of conformance tests.
   210  func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
   211  	for _, test := range tests {
   212  		t.Run(test.ShortName, func(t *testing.T) {
   213  			test.Run(t, suite)
   214  		})
   215  	}
   216  }
   217  
   218  // ConformanceTest is used to define each individual conformance test.
   219  type ConformanceTest struct {
   220  	ShortName   string
   221  	Description string
   222  	Features    []SupportedFeature
   223  	Manifests   []string
   224  	Slow        bool
   225  	Parallel    bool
   226  	Test        func(*testing.T, *ConformanceTestSuite)
   227  }
   228  
   229  // Run runs an individual tests, applying and cleaning up the required manifests
   230  // before calling the Test function.
   231  func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) {
   232  	if test.Parallel {
   233  		t.Parallel()
   234  	}
   235  
   236  	// Check that all features exercised by the test have been opted into by
   237  	// the suite.
   238  	for _, feature := range test.Features {
   239  		if !suite.SupportedFeatures.Has(feature) {
   240  			t.Skipf("Skipping %s: suite does not support %s", test.ShortName, feature)
   241  		}
   242  	}
   243  
   244  	// check that the test should not be skipped
   245  	if suite.SkipTests.Has(test.ShortName) || suite.RunTest != "" && suite.RunTest != test.ShortName {
   246  		t.Skipf("Skipping %s: test explicitly skipped", test.ShortName)
   247  	}
   248  
   249  	for _, manifestLocation := range test.Manifests {
   250  		t.Logf("Applying %s", manifestLocation)
   251  		suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, true)
   252  	}
   253  
   254  	test.Test(t, suite)
   255  }
   256  
   257  // ParseSupportedFeatures parses flag arguments and converts the string to
   258  // sets.Set[suite.SupportedFeature]
   259  func ParseSupportedFeatures(f string) sets.Set[SupportedFeature] {
   260  	if f == "" {
   261  		return nil
   262  	}
   263  	res := sets.Set[SupportedFeature]{}
   264  	for _, value := range strings.Split(f, ",") {
   265  		res.Insert(SupportedFeature(value))
   266  	}
   267  	return res
   268  }
   269  
   270  // ParseKeyValuePairs parses flag arguments and converts the string to
   271  // map[string]string containing label key/value pairs.
   272  func ParseKeyValuePairs(f string) map[string]string {
   273  	if f == "" {
   274  		return nil
   275  	}
   276  	res := map[string]string{}
   277  	for _, kv := range strings.Split(f, ",") {
   278  		parts := strings.Split(kv, "=")
   279  		if len(parts) == 2 {
   280  			res[parts[0]] = parts[1]
   281  		}
   282  	}
   283  	return res
   284  }
   285  
   286  // ParseSkipTests parses flag arguments and converts the string to
   287  // []string containing the tests to be skipped.
   288  func ParseSkipTests(t string) []string {
   289  	if t == "" {
   290  		return nil
   291  	}
   292  	return strings.Split(t, ",")
   293  }
   294
View as plain text