walk.go

Documentation: k8s.io/kubernetes/test/conformance

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package main
    18  
    19  import (
    20  	"encoding/json"
    21  	"flag"
    22  	"fmt"
    23  	"go/ast"
    24  	"go/parser"
    25  	"go/token"
    26  	"io"
    27  	"log"
    28  	"os"
    29  	"regexp"
    30  	"sort"
    31  	"strings"
    32  	"text/template"
    33  
    34  	"gopkg.in/yaml.v2"
    35  
    36  	"github.com/onsi/ginkgo/v2/types"
    37  )
    38  
    39  // ConformanceData describes the structure of the conformance.yaml file
    40  type ConformanceData struct {
    41  	// A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number.
    42  	URL string `yaml:"-"`
    43  	// Extracted from the "Testname:" comment before the test
    44  	TestName string
    45  	// CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]`
    46  	CodeName string
    47  	// Extracted from the "Description:" comment before the test
    48  	Description string
    49  	// Version when this test is added or modified ex: v1.12, v1.13
    50  	Release string
    51  	// File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes.
    52  	File string
    53  }
    54  
    55  var (
    56  	baseURL = flag.String("url", "https://github.com/kubernetes/kubernetes/tree/master/", "location of the current source")
    57  	k8sPath = flag.String("source", "", "location of the current source on the current machine")
    58  	confDoc = flag.Bool("docs", false, "write a conformance document")
    59  	version = flag.String("version", "v1.9", "version of this conformance document")
    60  
    61  	// If a test name contains any of these tags, it is ineligible for promotion to conformance
    62  	regexIneligibleTags = regexp.MustCompile(`\[(Alpha|Feature:[^\]]+|Flaky)\]`)
    63  
    64  	// Conformance comments should be within this number of lines to the call itself.
    65  	// Allowing for more than one in case a spare comment or two is below it.
    66  	conformanceCommentsLineWindow = 5
    67  
    68  	seenLines map[string]struct{}
    69  )
    70  
    71  type frame struct {
    72  	// File and Line are the file name and line number of the
    73  	// location in this frame. For non-leaf frames, this will be
    74  	// the location of a call. These may be the empty string and
    75  	// zero, respectively, if not known.
    76  	File string
    77  	Line int
    78  }
    79  
    80  func main() {
    81  	flag.Parse()
    82  
    83  	if len(flag.Args()) < 1 {
    84  		log.Fatalln("Requires the name of the test details file as first and only argument.")
    85  	}
    86  	testDetailsFile := flag.Args()[0]
    87  	f, err := os.Open(testDetailsFile)
    88  	if err != nil {
    89  		log.Fatalf("Failed to open file %v: %v", testDetailsFile, err)
    90  	}
    91  	defer f.Close()
    92  
    93  	seenLines = map[string]struct{}{}
    94  	dec := json.NewDecoder(f)
    95  	testInfos := []*ConformanceData{}
    96  	for {
    97  		var spec *types.SpecReport
    98  		if err := dec.Decode(&spec); err == io.EOF {
    99  			break
   100  		} else if err != nil {
   101  			log.Fatal(err)
   102  		}
   103  
   104  		if isConformance(spec) {
   105  			testInfo := getTestInfo(spec)
   106  			if testInfo != nil {
   107  				testInfos = append(testInfos, testInfo)
   108  				if err := validateTestName(testInfo.CodeName); err != nil {
   109  					log.Fatal(err)
   110  				}
   111  			}
   112  		}
   113  	}
   114  
   115  	sort.Slice(testInfos, func(i, j int) bool { return testInfos[i].CodeName < testInfos[j].CodeName })
   116  	saveAllTestInfo(testInfos)
   117  }
   118  
   119  func isConformance(spec *types.SpecReport) bool {
   120  	return strings.Contains(getTestName(spec), "[Conformance]")
   121  }
   122  
   123  func getTestInfo(spec *types.SpecReport) *ConformanceData {
   124  	var c *ConformanceData
   125  	var err error
   126  	// The key to this working is that we don't need to parse every file or walk
   127  	// every types.CodeLocation. The LeafNodeLocation is going to be file:line which
   128  	// attached to the comment that we want.
   129  	leafNodeLocation := spec.LeafNodeLocation
   130  	frame := frame{
   131  		File: leafNodeLocation.FileName,
   132  		Line: leafNodeLocation.LineNumber,
   133  	}
   134  	c, err = getConformanceData(frame)
   135  	if err != nil {
   136  		log.Printf("Error looking for conformance data: %v", err)
   137  	}
   138  	if c == nil {
   139  		log.Printf("Did not find test info for spec: %#v\n", getTestName(spec))
   140  		return nil
   141  	}
   142  	c.CodeName = getTestName(spec)
   143  	return c
   144  }
   145  
   146  func getTestName(spec *types.SpecReport) string {
   147  	return strings.Join(spec.ContainerHierarchyTexts[0:], " ") + " " + spec.LeafNodeText
   148  }
   149  
   150  func saveAllTestInfo(dataSet []*ConformanceData) {
   151  	if *confDoc {
   152  		// Note: this assumes that you're running from the root of the kube src repo
   153  		templ, err := template.ParseFiles("./test/conformance/cf_header.md")
   154  		if err != nil {
   155  			fmt.Printf("Error reading the Header file information: %s\n\n", err)
   156  		}
   157  		data := struct {
   158  			Version string
   159  		}{
   160  			Version: *version,
   161  		}
   162  		templ.Execute(os.Stdout, data)
   163  
   164  		for _, data := range dataSet {
   165  			fmt.Printf("## [%s](%s)\n\n", data.TestName, data.URL)
   166  			fmt.Printf("- Added to conformance in release %s\n", data.Release)
   167  			fmt.Printf("- Defined in code as: %s\n\n", data.CodeName)
   168  			fmt.Printf("%s\n\n", data.Description)
   169  		}
   170  		return
   171  	}
   172  
   173  	// Serialize the list as a whole. Generally meant to end up as conformance.txt which tracks the set of tests.
   174  	b, err := yaml.Marshal(dataSet)
   175  	if err != nil {
   176  		log.Printf("Error marshalling data into YAML: %v", err)
   177  	}
   178  	fmt.Println(string(b))
   179  }
   180  
   181  func getConformanceData(targetFrame frame) (*ConformanceData, error) {
   182  	root := *k8sPath
   183  	if !strings.HasSuffix(root, string(os.PathSeparator)) {
   184  		root += string(os.PathSeparator)
   185  	}
   186  
   187  	trimmedFile := strings.TrimPrefix(targetFrame.File, root)
   188  	targetFrame.File = trimmedFile
   189  
   190  	freader, err := os.Open(targetFrame.File)
   191  	if err != nil {
   192  		return nil, err
   193  	}
   194  	defer freader.Close()
   195  
   196  	cd, err := scanFileForFrame(targetFrame.File, freader, targetFrame)
   197  	if err != nil {
   198  		return nil, err
   199  	}
   200  	if cd != nil {
   201  		return cd, nil
   202  	}
   203  
   204  	return nil, nil
   205  }
   206  
   207  // scanFileForFrame will scan the target and look for a conformance comment attached to the function
   208  // described by the target frame. If the comment can't be found then nil, nil is returned.
   209  func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*ConformanceData, error) {
   210  	fset := token.NewFileSet() // positions are relative to fset
   211  	f, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
   212  	if err != nil {
   213  		return nil, err
   214  	}
   215  
   216  	cmap := ast.NewCommentMap(fset, f, f.Comments)
   217  	for _, cs := range cmap {
   218  		for _, c := range cs {
   219  			if cd := tryCommentGroupAndFrame(fset, c, targetFrame); cd != nil {
   220  				return cd, nil
   221  			}
   222  		}
   223  	}
   224  	return nil, nil
   225  }
   226  
   227  func validateTestName(s string) error {
   228  	matches := regexIneligibleTags.FindAllString(s, -1)
   229  	if matches != nil {
   230  		return fmt.Errorf("'%s' cannot have invalid tags %v", s, strings.Join(matches, ","))
   231  	}
   232  	return nil
   233  }
   234  
   235  func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *ConformanceData {
   236  	if !shouldProcessCommentGroup(fset, cg, f) {
   237  		return nil
   238  	}
   239  
   240  	// Each file/line will either be some helper function (not a conformance comment) or apply to just a single test. Don't revisit.
   241  	if seenLines != nil {
   242  		seenLines[fmt.Sprintf("%v:%v", f.File, f.Line)] = struct{}{}
   243  	}
   244  	cd := commentToConformanceData(cg.Text())
   245  	if cd == nil {
   246  		return nil
   247  	}
   248  
   249  	cd.URL = fmt.Sprintf("%s%s#L%d", *baseURL, f.File, f.Line)
   250  	cd.File = f.File
   251  	return cd
   252  }
   253  
   254  func shouldProcessCommentGroup(fset *token.FileSet, cg *ast.CommentGroup, f frame) bool {
   255  	lineDiff := f.Line - fset.Position(cg.End()).Line
   256  	return lineDiff > 0 && lineDiff <= conformanceCommentsLineWindow
   257  }
   258  
   259  func commentToConformanceData(comment string) *ConformanceData {
   260  	lines := strings.Split(comment, "\n")
   261  	descLines := []string{}
   262  	cd := &ConformanceData{}
   263  	var curLine string
   264  	for _, line := range lines {
   265  		line = strings.TrimSpace(line)
   266  		if len(line) == 0 {
   267  			continue
   268  		}
   269  		if sline := regexp.MustCompile("^Testname\\s*:\\s*").Split(line, -1); len(sline) == 2 {
   270  			curLine = "Testname"
   271  			cd.TestName = sline[1]
   272  			continue
   273  		}
   274  		if sline := regexp.MustCompile("^Release\\s*:\\s*").Split(line, -1); len(sline) == 2 {
   275  			curLine = "Release"
   276  			cd.Release = sline[1]
   277  			continue
   278  		}
   279  		if sline := regexp.MustCompile("^Description\\s*:\\s*").Split(line, -1); len(sline) == 2 {
   280  			curLine = "Description"
   281  			descLines = append(descLines, sline[1])
   282  			continue
   283  		}
   284  
   285  		// Line has no header
   286  		if curLine == "Description" {
   287  			descLines = append(descLines, line)
   288  		}
   289  	}
   290  	if cd.TestName == "" {
   291  		return nil
   292  	}
   293  
   294  	cd.Description = strings.Join(descLines, " ")
   295  	return cd
   296  }
   297
View as plain text