experimental_reports.go

Documentation: sigs.k8s.io/gateway-api/conformance/utils/suite

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package suite
    18  
    19  import (
    20  	"k8s.io/apimachinery/pkg/util/sets"
    21  
    22  	confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
    23  )
    24  
    25  // -----------------------------------------------------------------------------
    26  // ConformanceReport - Private Types
    27  // -----------------------------------------------------------------------------
    28  
    29  type testResult struct {
    30  	test   ConformanceTest
    31  	result resultType
    32  }
    33  
    34  type resultType string
    35  
    36  var (
    37  	testSucceeded    resultType = "SUCCEEDED"
    38  	testFailed       resultType = "FAILED"
    39  	testSkipped      resultType = "SKIPPED"
    40  	testNotSupported resultType = "NOT_SUPPORTED"
    41  )
    42  
    43  type profileReportsMap map[ConformanceProfileName]confv1a1.ProfileReport
    44  
    45  func newReports() profileReportsMap {
    46  	return make(profileReportsMap)
    47  }
    48  
    49  func (p profileReportsMap) addTestResults(conformanceProfile ConformanceProfile, result testResult) {
    50  	// initialize the profile report if not already initialized
    51  	if _, ok := p[conformanceProfile.Name]; !ok {
    52  		p[conformanceProfile.Name] = confv1a1.ProfileReport{
    53  			Name: string(conformanceProfile.Name),
    54  		}
    55  	}
    56  
    57  	testIsExtended := isTestExtended(conformanceProfile, result.test)
    58  	report := p[conformanceProfile.Name]
    59  
    60  	switch result.result {
    61  	case testSucceeded:
    62  		if testIsExtended {
    63  			if report.Extended == nil {
    64  				report.Extended = &confv1a1.ExtendedStatus{}
    65  			}
    66  			report.Extended.Statistics.Passed++
    67  		} else {
    68  			report.Core.Statistics.Passed++
    69  		}
    70  	case testFailed:
    71  		if testIsExtended {
    72  			if report.Extended == nil {
    73  				report.Extended = &confv1a1.ExtendedStatus{}
    74  			}
    75  			if report.Extended.FailedTests == nil {
    76  				report.Extended.FailedTests = []string{}
    77  			}
    78  			report.Extended.FailedTests = append(report.Extended.FailedTests, result.test.ShortName)
    79  			report.Extended.Statistics.Failed++
    80  		} else {
    81  			report.Core.Statistics.Failed++
    82  			if report.Core.FailedTests == nil {
    83  				report.Core.FailedTests = []string{}
    84  			}
    85  			report.Core.FailedTests = append(report.Core.FailedTests, result.test.ShortName)
    86  		}
    87  	case testSkipped:
    88  		if testIsExtended {
    89  			if report.Extended == nil {
    90  				report.Extended = &confv1a1.ExtendedStatus{}
    91  			}
    92  			report.Extended.Statistics.Skipped++
    93  			if report.Extended.SkippedTests == nil {
    94  				report.Extended.SkippedTests = []string{}
    95  			}
    96  			report.Extended.SkippedTests = append(report.Extended.SkippedTests, result.test.ShortName)
    97  		} else {
    98  			report.Core.Statistics.Skipped++
    99  			if report.Core.SkippedTests == nil {
   100  				report.Core.SkippedTests = []string{}
   101  			}
   102  			report.Core.SkippedTests = append(report.Core.SkippedTests, result.test.ShortName)
   103  		}
   104  	}
   105  	p[conformanceProfile.Name] = report
   106  }
   107  
   108  func (p profileReportsMap) list() (profileReports []confv1a1.ProfileReport) {
   109  	for _, profileReport := range p {
   110  		profileReports = append(profileReports, profileReport)
   111  	}
   112  	return
   113  }
   114  
   115  func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformanceProfileName]sets.Set[SupportedFeature], unsupportedFeaturesMap map[ConformanceProfileName]sets.Set[SupportedFeature]) {
   116  	for key, report := range p {
   117  		// report the overall result for core features
   118  		switch {
   119  		case report.Core.Failed > 0:
   120  			report.Core.Result = confv1a1.Failure
   121  		case report.Core.Skipped > 0:
   122  			report.Core.Result = confv1a1.Partial
   123  		default:
   124  			report.Core.Result = confv1a1.Success
   125  		}
   126  
   127  		if report.Extended != nil {
   128  			// report the overall result for extended features
   129  			switch {
   130  			case report.Extended.Failed > 0:
   131  				report.Extended.Result = confv1a1.Failure
   132  			case report.Extended.Skipped > 0:
   133  				report.Extended.Result = confv1a1.Partial
   134  			default:
   135  				report.Extended.Result = confv1a1.Success
   136  			}
   137  		}
   138  		p[key] = report
   139  
   140  		supportedFeatures := supportedFeaturesMap[ConformanceProfileName(report.Name)]
   141  		if report.Extended != nil {
   142  			if supportedFeatures != nil {
   143  				if report.Extended.SupportedFeatures == nil {
   144  					report.Extended.SupportedFeatures = make([]string, 0)
   145  				}
   146  				for _, f := range supportedFeatures.UnsortedList() {
   147  					report.Extended.SupportedFeatures = append(report.Extended.SupportedFeatures, string(f))
   148  				}
   149  			}
   150  		}
   151  
   152  		unsupportedFeatures := unsupportedFeaturesMap[ConformanceProfileName(report.Name)]
   153  		if report.Extended != nil {
   154  			if unsupportedFeatures != nil {
   155  				if report.Extended.UnsupportedFeatures == nil {
   156  					report.Extended.UnsupportedFeatures = make([]string, 0)
   157  				}
   158  				for _, f := range unsupportedFeatures.UnsortedList() {
   159  					report.Extended.UnsupportedFeatures = append(report.Extended.UnsupportedFeatures, string(f))
   160  				}
   161  			}
   162  		}
   163  	}
   164  }
   165  
   166  // -----------------------------------------------------------------------------
   167  // ConformanceReport - Private Helper Functions
   168  // -----------------------------------------------------------------------------
   169  
   170  // isTestExtended determines if a provided test is considered to be supported
   171  // at an extended level of support given the provided conformance profile.
   172  //
   173  // TODO: right now the tests themselves don't indicate the conformance
   174  // support level associated with them. The only way we have right now
   175  // in this prototype to know whether a test belongs to any particular
   176  // conformance level is to compare the features needed for the test to
   177  // the conformance profiles known list of core vs extended features.
   178  // Later if we move out of Prototyping/Provisional it would probably
   179  // be best to indicate the conformance support level of each test, but
   180  // for now this hack works.
   181  func isTestExtended(profile ConformanceProfile, test ConformanceTest) bool {
   182  	for _, supportedFeature := range test.Features {
   183  		// if ANY of the features needed for the test are extended features,
   184  		// then we consider the entire test extended level support.
   185  		if profile.ExtendedFeatures.Has(supportedFeature) {
   186  			return true
   187  		}
   188  	}
   189  	return false
   190  }
   191
View as plain text