experimental_suite.go

Documentation: sigs.k8s.io/gateway-api/conformance/utils/suite

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package suite
    18  
    19  import (
    20  	"errors"
    21  	"fmt"
    22  	"strings"
    23  	"sync"
    24  	"testing"
    25  	"time"
    26  
    27  	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/apimachinery/pkg/util/sets"
    29  
    30  	"sigs.k8s.io/gateway-api/conformance"
    31  	confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
    32  	"sigs.k8s.io/gateway-api/conformance/utils/config"
    33  	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
    34  	"sigs.k8s.io/gateway-api/conformance/utils/roundtripper"
    35  )
    36  
    37  // -----------------------------------------------------------------------------
    38  // Conformance Test Suite - Public Types
    39  // -----------------------------------------------------------------------------
    40  
    41  // ConformanceTestSuite defines the test suite used to run Gateway API
    42  // conformance tests.
    43  // This is experimental for now and can be used as an alternative to the
    44  // ConformanceTestSuite. Once this won't be experimental any longer,
    45  // the two of them will be merged.
    46  type ExperimentalConformanceTestSuite struct {
    47  	ConformanceTestSuite
    48  
    49  	// implementation contains the details of the implementation, such as
    50  	// organization, project, etc.
    51  	implementation confv1a1.Implementation
    52  
    53  	// conformanceProfiles is a compiled list of profiles to check
    54  	// conformance against.
    55  	conformanceProfiles sets.Set[ConformanceProfileName]
    56  
    57  	// running indicates whether the test suite is currently running
    58  	running bool
    59  
    60  	// results stores the pass or fail results of each test that was run by
    61  	// the test suite, organized by the tests unique name.
    62  	results map[string]testResult
    63  
    64  	// extendedSupportedFeatures is a compiled list of named features that were
    65  	// marked as supported, and is used for reporting the test results.
    66  	extendedSupportedFeatures map[ConformanceProfileName]sets.Set[SupportedFeature]
    67  
    68  	// extendedUnsupportedFeatures is a compiled list of named features that were
    69  	// marked as not supported, and is used for reporting the test results.
    70  	extendedUnsupportedFeatures map[ConformanceProfileName]sets.Set[SupportedFeature]
    71  
    72  	// lock is a mutex to help ensure thread safety of the test suite object.
    73  	lock sync.RWMutex
    74  }
    75  
    76  // Options can be used to initialize a ConformanceTestSuite.
    77  type ExperimentalConformanceOptions struct {
    78  	Options
    79  
    80  	Implementation      confv1a1.Implementation
    81  	ConformanceProfiles sets.Set[ConformanceProfileName]
    82  }
    83  
    84  // NewExperimentalConformanceTestSuite is a helper to use for creating a new ExperimentalConformanceTestSuite.
    85  func NewExperimentalConformanceTestSuite(s ExperimentalConformanceOptions) (*ExperimentalConformanceTestSuite, error) {
    86  	config.SetupTimeoutConfig(&s.TimeoutConfig)
    87  
    88  	roundTripper := s.RoundTripper
    89  	if roundTripper == nil {
    90  		roundTripper = &roundtripper.DefaultRoundTripper{Debug: s.Debug, TimeoutConfig: s.TimeoutConfig}
    91  	}
    92  
    93  	suite := &ExperimentalConformanceTestSuite{
    94  		results:                     make(map[string]testResult),
    95  		extendedUnsupportedFeatures: make(map[ConformanceProfileName]sets.Set[SupportedFeature]),
    96  		extendedSupportedFeatures:   make(map[ConformanceProfileName]sets.Set[SupportedFeature]),
    97  		conformanceProfiles:         s.ConformanceProfiles,
    98  		implementation:              s.Implementation,
    99  	}
   100  
   101  	// test suite callers are required to provide a conformance profile OR at
   102  	// minimum a list of features which they support.
   103  	if s.SupportedFeatures == nil && s.ConformanceProfiles.Len() == 0 && !s.EnableAllSupportedFeatures {
   104  		return nil, fmt.Errorf("no conformance profile was selected for test run, and no supported features were provided so no tests could be selected")
   105  	}
   106  
   107  	// test suite callers can potentially just run all tests by saying they
   108  	// cover all features, if they don't they'll need to have provided a
   109  	// conformance profile or at least some specific features they support.
   110  	if s.EnableAllSupportedFeatures {
   111  		s.SupportedFeatures = AllFeatures
   112  	} else {
   113  		if s.SupportedFeatures == nil {
   114  			s.SupportedFeatures = sets.New[SupportedFeature]()
   115  		}
   116  
   117  		for _, conformanceProfileName := range s.ConformanceProfiles.UnsortedList() {
   118  			conformanceProfile, err := getConformanceProfileForName(conformanceProfileName)
   119  			if err != nil {
   120  				return nil, fmt.Errorf("failed to retrieve conformance profile: %w", err)
   121  			}
   122  			// the use of a conformance profile implicitly enables any features of
   123  			// that profile which are supported at a Core level of support.
   124  			for _, f := range conformanceProfile.CoreFeatures.UnsortedList() {
   125  				if !s.SupportedFeatures.Has(f) {
   126  					s.SupportedFeatures.Insert(f)
   127  				}
   128  			}
   129  			for _, f := range conformanceProfile.ExtendedFeatures.UnsortedList() {
   130  				if s.SupportedFeatures.Has(f) {
   131  					if suite.extendedSupportedFeatures[conformanceProfileName] == nil {
   132  						suite.extendedSupportedFeatures[conformanceProfileName] = sets.New[SupportedFeature]()
   133  					}
   134  					suite.extendedSupportedFeatures[conformanceProfileName].Insert(f)
   135  				} else {
   136  					if suite.extendedUnsupportedFeatures[conformanceProfileName] == nil {
   137  						suite.extendedUnsupportedFeatures[conformanceProfileName] = sets.New[SupportedFeature]()
   138  					}
   139  					suite.extendedUnsupportedFeatures[conformanceProfileName].Insert(f)
   140  				}
   141  				// Add Exempt Features into unsupported features list
   142  				if s.ExemptFeatures.Has(f) {
   143  					suite.extendedUnsupportedFeatures[conformanceProfileName].Insert(f)
   144  				}
   145  			}
   146  		}
   147  	}
   148  
   149  	for feature := range s.ExemptFeatures {
   150  		s.SupportedFeatures.Delete(feature)
   151  	}
   152  
   153  	if s.FS == nil {
   154  		s.FS = &conformance.Manifests
   155  	}
   156  
   157  	suite.ConformanceTestSuite = ConformanceTestSuite{
   158  		Client:           s.Client,
   159  		Clientset:        s.Clientset,
   160  		RestConfig:       s.RestConfig,
   161  		RoundTripper:     roundTripper,
   162  		GatewayClassName: s.GatewayClassName,
   163  		Debug:            s.Debug,
   164  		Cleanup:          s.CleanupBaseResources,
   165  		BaseManifests:    s.BaseManifests,
   166  		MeshManifests:    s.MeshManifests,
   167  		Applier: kubernetes.Applier{
   168  			NamespaceLabels:      s.NamespaceLabels,
   169  			NamespaceAnnotations: s.NamespaceAnnotations,
   170  		},
   171  		SupportedFeatures:        s.SupportedFeatures,
   172  		TimeoutConfig:            s.TimeoutConfig,
   173  		SkipTests:                sets.New(s.SkipTests...),
   174  		FS:                       *s.FS,
   175  		UsableNetworkAddresses:   s.UsableNetworkAddresses,
   176  		UnusableNetworkAddresses: s.UnusableNetworkAddresses,
   177  	}
   178  
   179  	// apply defaults
   180  	if suite.BaseManifests == "" {
   181  		suite.BaseManifests = "base/manifests.yaml"
   182  	}
   183  	if suite.MeshManifests == "" {
   184  		suite.MeshManifests = "mesh/manifests.yaml"
   185  	}
   186  
   187  	return suite, nil
   188  }
   189  
   190  // -----------------------------------------------------------------------------
   191  // Conformance Test Suite - Public Methods
   192  // -----------------------------------------------------------------------------
   193  
   194  // Setup ensures the base resources required for conformance tests are installed
   195  // in the cluster. It also ensures that all relevant resources are ready.
   196  func (suite *ExperimentalConformanceTestSuite) Setup(t *testing.T) {
   197  	suite.ConformanceTestSuite.Setup(t)
   198  }
   199  
   200  // Run runs the provided set of conformance tests.
   201  func (suite *ExperimentalConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) error {
   202  	// verify that the test suite isn't already running, don't start a new run
   203  	// until the previous run finishes
   204  	suite.lock.Lock()
   205  	if suite.running {
   206  		suite.lock.Unlock()
   207  		return fmt.Errorf("can't run the test suite multiple times in parallel: the test suite is already running")
   208  	}
   209  
   210  	// if the test suite is not currently running, reset reporting and start a
   211  	// new test run.
   212  	suite.running = true
   213  	suite.results = nil
   214  	suite.lock.Unlock()
   215  
   216  	// run all tests and collect the test results for conformance reporting
   217  	results := make(map[string]testResult)
   218  	for _, test := range tests {
   219  		succeeded := t.Run(test.ShortName, func(t *testing.T) {
   220  			test.Run(t, &suite.ConformanceTestSuite)
   221  		})
   222  		res := testSucceeded
   223  		if suite.SkipTests.Has(test.ShortName) {
   224  			res = testSkipped
   225  		}
   226  		if !suite.SupportedFeatures.HasAll(test.Features...) {
   227  			res = testNotSupported
   228  		}
   229  
   230  		if !succeeded {
   231  			res = testFailed
   232  		}
   233  
   234  		results[test.ShortName] = testResult{
   235  			test:   test,
   236  			result: res,
   237  		}
   238  	}
   239  
   240  	// now that the tests have completed, mark the test suite as not running
   241  	// and report the test results.
   242  	suite.lock.Lock()
   243  	suite.running = false
   244  	suite.results = results
   245  	suite.lock.Unlock()
   246  
   247  	return nil
   248  }
   249  
   250  // Report emits a ConformanceReport for the previously completed test run.
   251  // If no run completed prior to running the report, and error is emitted.
   252  func (suite *ExperimentalConformanceTestSuite) Report() (*confv1a1.ConformanceReport, error) {
   253  	suite.lock.RLock()
   254  	if suite.running {
   255  		suite.lock.RUnlock()
   256  		return nil, fmt.Errorf("can't generate report: the test suite is currently running")
   257  	}
   258  	defer suite.lock.RUnlock()
   259  
   260  	profileReports := newReports()
   261  	for _, testResult := range suite.results {
   262  		conformanceProfiles := getConformanceProfilesForTest(testResult.test, suite.conformanceProfiles)
   263  		for _, profile := range conformanceProfiles.UnsortedList() {
   264  			profileReports.addTestResults(*profile, testResult)
   265  		}
   266  	}
   267  
   268  	profileReports.compileResults(suite.extendedSupportedFeatures, suite.extendedUnsupportedFeatures)
   269  
   270  	return &confv1a1.ConformanceReport{
   271  		TypeMeta: v1.TypeMeta{
   272  			APIVersion: "gateway.networking.k8s.io/v1alpha1",
   273  			Kind:       "ConformanceReport",
   274  		},
   275  		Date:              time.Now().Format(time.RFC3339),
   276  		Implementation:    suite.implementation,
   277  		GatewayAPIVersion: "TODO",
   278  		ProfileReports:    profileReports.list(),
   279  	}, nil
   280  }
   281  
   282  // ParseImplementation parses implementation-specific flag arguments and
   283  // creates a *confv1a1.Implementation.
   284  func ParseImplementation(org, project, url, version, contact string) (*confv1a1.Implementation, error) {
   285  	if org == "" {
   286  		return nil, errors.New("implementation's organization can not be empty")
   287  	}
   288  	if project == "" {
   289  		return nil, errors.New("implementation's project can not be empty")
   290  	}
   291  	if url == "" {
   292  		return nil, errors.New("implementation's url can not be empty")
   293  	}
   294  	if version == "" {
   295  		return nil, errors.New("implementation's version can not be empty")
   296  	}
   297  	contacts := strings.Split(contact, ",")
   298  	if len(contacts) == 0 {
   299  		return nil, errors.New("implementation's contact can not be empty")
   300  	}
   301  
   302  	// TODO: add data validation https://github.com/kubernetes-sigs/gateway-api/issues/2178
   303  
   304  	return &confv1a1.Implementation{
   305  		Organization: org,
   306  		Project:      project,
   307  		URL:          url,
   308  		Version:      version,
   309  		Contact:      contacts,
   310  	}, nil
   311  }
   312  
   313  // ParseConformanceProfiles parses flag arguments and converts the string to
   314  // sets.Set[ConformanceProfileName].
   315  func ParseConformanceProfiles(p string) sets.Set[ConformanceProfileName] {
   316  	res := sets.Set[ConformanceProfileName]{}
   317  	if p == "" {
   318  		return res
   319  	}
   320  
   321  	for _, value := range strings.Split(p, ",") {
   322  		res.Insert(ConformanceProfileName(value))
   323  	}
   324  	return res
   325  }
   326
View as plain text