output.go

Documentation: k8s.io/kubernetes/test/e2e/framework/internal/output

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package output
    18  
    19  import (
    20  	"encoding/xml"
    21  	"os"
    22  	"path"
    23  	"regexp"
    24  	"testing"
    25  
    26  	"github.com/google/go-cmp/cmp"
    27  	"github.com/google/go-cmp/cmp/cmpopts"
    28  	"github.com/onsi/ginkgo/v2"
    29  	"github.com/onsi/ginkgo/v2/reporters"
    30  	"github.com/onsi/gomega"
    31  	"github.com/stretchr/testify/require"
    32  
    33  	"k8s.io/kubernetes/test/e2e/framework"
    34  	"k8s.io/kubernetes/test/e2e/framework/internal/junit"
    35  )
    36  
    37  // TestGinkgoOutput runs the current suite and verifies that the generated
    38  // JUnit file matches the expected result.
    39  //
    40  // The Ginkgo output on the console (aka the test suite log) does not get
    41  // checked. It is usually less important for the CI and more relevant when
    42  // using test suite interactively. To see what that Ginkgo output looks like,
    43  // run tests with "go test -v".
    44  func TestGinkgoOutput(t *testing.T, expected TestResult, runSpecsArgs ...interface{}) {
    45  	tmpdir := t.TempDir()
    46  	junitFile := path.Join(tmpdir, "junit.xml")
    47  	gomega.RegisterFailHandler(framework.Fail)
    48  	ginkgo.ReportAfterSuite("write JUnit file", func(report ginkgo.Report) {
    49  		junit.WriteJUnitReport(report, junitFile)
    50  	})
    51  	fakeT := &testing.T{}
    52  	ginkgo.RunSpecs(fakeT, "Logging Suite", runSpecsArgs...)
    53  
    54  	var actual reporters.JUnitTestSuites
    55  	data, err := os.ReadFile(junitFile)
    56  	require.NoError(t, err)
    57  	err = xml.Unmarshal(data, &actual)
    58  	require.NoError(t, err)
    59  
    60  	if len(actual.TestSuites) != 1 {
    61  		t.Fatalf("expected one test suite, got %d, JUnit content:\n%s", len(actual.TestSuites), string(data))
    62  	}
    63  	diff := cmp.Diff(expected.Suite, actual.TestSuites[0],
    64  		// Time varies.
    65  		// Name and Classname are "Logging Suite".
    66  		// Package includes a varying path, not interesting.
    67  		// Properties also too complicated to compare.
    68  		cmpopts.IgnoreFields(reporters.JUnitTestSuite{}, "Time", "Timestamp", "Name", "Package", "Properties"),
    69  		cmpopts.IgnoreFields(reporters.JUnitTestCase{}, "Time", "Classname"),
    70  		cmpopts.SortSlices(func(tc1, tc2 reporters.JUnitTestCase) bool {
    71  			return tc1.Name < tc2.Name
    72  		}),
    73  		cmpopts.AcyclicTransformer("simplify", func(in string) any {
    74  			out := simplify(in, expected)
    75  			// Sometimes cmp.Diff does not print the full string when it is long.
    76  			// Uncommenting this here may help debug differences.
    77  			// if len(out) > 100 {
    78  			// 	t.Logf("%s\n---------------------------------------\n%s\n", in, out)
    79  			// }
    80  
    81  			// Same idea as in
    82  			// https://github.com/google/go-cmp/issues/192#issuecomment-605346277:
    83  			// it forces cmp.Diff to diff strings line-by-line,
    84  			// even when it normally wouldn't.  The downside is
    85  			// that the output is harder to turn back into the
    86  			// expected reference string.
    87  			// if len(out) > 50 {
    88  			// 	return strings.Split(out, "\n")
    89  			// }
    90  
    91  			return out
    92  		}),
    93  	)
    94  	if diff != "" {
    95  		t.Fatalf("Simplified JUnit report not as expected (-want, +got):\n%s\n\nFull XML:\n%s", diff, string(data))
    96  	}
    97  }
    98  
    99  // TestResult is the expected outcome of the suite, with additional parameters that
   100  // determine equality.
   101  type TestResult struct {
   102  	// Called to normalize all output strings before comparison if non-nil.
   103  	NormalizeOutput func(string) string
   104  
   105  	// All test cases and overall suite results.
   106  	Suite reporters.JUnitTestSuite
   107  }
   108  
   109  func simplify(in string, expected TestResult) string {
   110  	out := normalizeLocation(in)
   111  	out = stripTimes(out)
   112  	out = stripAddresses(out)
   113  	out = normalizeInitFunctions(out)
   114  	if expected.NormalizeOutput != nil {
   115  		out = expected.NormalizeOutput(out)
   116  	}
   117  	return out
   118  }
   119  
   120  // timePrefix matches "Jul 17 08:08:25.950: " at the beginning of each line.
   121  var timePrefix = regexp.MustCompile(`(?m)^[[:alpha:]]{3} +[[:digit:]]{1,2} +[[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}.[[:digit:]]{3}: `)
   122  
   123  // elapsedSuffix matches "Elapsed: 16.189µs"
   124  var elapsedSuffix = regexp.MustCompile(`Elapsed: [[:digit:]]+(\.[[:digit:]]+)?(µs|ns|ms|s|m)`)
   125  
   126  // afterSuffix matches "after 5.001s."
   127  var afterSuffix = regexp.MustCompile(`after [[:digit:]]+(\.[[:digit:]]+)?(µs|ns|ms|s|m).`)
   128  
   129  // timeSuffix matches "@ 09/06/22 15:36:43.44 (5.001s)" as printed by Ginkgo v2 for log output, with the duration being optional.
   130  var timeSuffix = regexp.MustCompile(`(?m)@[[:space:]][[:digit:]]{2}/[[:digit:]]{2}/[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}(\.[[:digit:]]{1,3})?( \([[:digit:]]+(\.[[:digit:]]+)?(µs|ns|ms|s|m)\))?$`)
   131  
   132  func stripTimes(in string) string {
   133  	out := timePrefix.ReplaceAllString(in, "")
   134  	out = elapsedSuffix.ReplaceAllString(out, "Elapsed: <elapsed>")
   135  	out = timeSuffix.ReplaceAllString(out, "<time>")
   136  	out = afterSuffix.ReplaceAllString(out, "after <after>.")
   137  	return out
   138  }
   139  
   140  // instanceAddr matches " | 0xc0003dec60>"
   141  var instanceAddr = regexp.MustCompile(` \| 0x[0-9a-fA-F]+>`)
   142  
   143  func stripAddresses(in string) string {
   144  	return instanceAddr.ReplaceAllString(in, ">")
   145  }
   146  
   147  // stackLocation matches "<some path>/<file>.go:75 +0x1f1" after a slash (built
   148  // locally) or one of a few relative paths (built in the Kubernetes CI).
   149  var stackLocation = regexp.MustCompile(`(?:/|vendor/|test/|GOROOT/).*/([[:^space:]]+.go:[[:digit:]]+)( \+0x[0-9a-fA-F]+)?`)
   150  
   151  // functionArgs matches "<function name>(...)" where <function name> may be an anonymous function (e.g. "pod_test.glob..func1.1")
   152  var functionArgs = regexp.MustCompile(`([[:alpha:][:digit:].]+)\(.*\)`)
   153  
   154  // klogPrefix matches "I0822 16:10:39.343790  989127 "
   155  var klogPrefix = regexp.MustCompile(`(?m)^[IEF][[:digit:]]{4} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}\.[[:digit:]]{6}[[:space:]]+[[:digit:]]+ `)
   156  
   157  // testFailureOutput matches TestFailureOutput() and its source followed by additional stack entries:
   158  //
   159  // k8s.io/kubernetes/test/e2e/framework/pod/pod_test.TestFailureOutput(0xc000558800)
   160  //
   161  //	/nvme/gopath/src/k8s.io/kubernetes/test/e2e/framework/pod/wait_test.go:73 +0x1c9
   162  //
   163  // testing.tRunner(0xc000558800, 0x1af2848)
   164  //
   165  //	/nvme/gopath/go/src/testing/testing.go:865 +0xc0
   166  //
   167  // created by testing.(*T).Run
   168  //
   169  //	/nvme/gopath/go/src/testing/testing.go:916 +0x35a
   170  var testFailureOutput = regexp.MustCompile(`(?m)^k8s.io/kubernetes/test/e2e/framework/internal/output\.TestGinkgoOutput\(.*\n\t.*(\n.*\n\t.*)*`)
   171  
   172  // normalizeLocation removes path prefix and function parameters and certain stack entries
   173  // that we don't care about.
   174  func normalizeLocation(in string) string {
   175  	out := in
   176  	out = stackLocation.ReplaceAllString(out, "$1")
   177  	out = functionArgs.ReplaceAllString(out, "$1()")
   178  	out = testFailureOutput.ReplaceAllString(out, "")
   179  	out = klogPrefix.ReplaceAllString(out, "<klog> ")
   180  	return out
   181  }
   182  
   183  var initFunc = regexp.MustCompile(`(init\.+func|glob\.+func)`)
   184  
   185  // normalizeInitFunctions maps both init.func (used by Go >= 1.22) and
   186  // glob..func (used by Go < 1.22) to <init.func>.
   187  func normalizeInitFunctions(in string) string {
   188  	out := initFunc.ReplaceAllString(in, "<init.func>")
   189  	return out
   190  }
   191
View as plain text