junit_report.go

Documentation: github.com/onsi/ginkgo/v2/reporters

     1  /*
     2  
     3  JUnit XML Reporter for Ginkgo
     4  
     5  For usage instructions: http://onsi.github.io/ginkgo/#generating_junit_xml_output
     6  
     7  The schema used for the generated JUnit xml file was adapted from https://llg.cubic.org/docs/junit/
     8  
     9  */
    10  
    11  package reporters
    12  
    13  import (
    14  	"encoding/xml"
    15  	"fmt"
    16  	"os"
    17  	"path"
    18  	"regexp"
    19  	"strings"
    20  
    21  	"github.com/onsi/ginkgo/v2/config"
    22  	"github.com/onsi/ginkgo/v2/types"
    23  )
    24  
    25  type JunitReportConfig struct {
    26  	// Spec States for which no timeline should be emitted for system-err
    27  	// set this to types.SpecStatePassed|types.SpecStateSkipped|types.SpecStatePending to only match failing specs
    28  	OmitTimelinesForSpecState types.SpecState
    29  
    30  	// Enable OmitFailureMessageAttr to prevent failure messages appearing in the "message" attribute of the Failure and Error tags
    31  	OmitFailureMessageAttr bool
    32  
    33  	//Enable OmitCapturedStdOutErr to prevent captured stdout/stderr appearing in system-out
    34  	OmitCapturedStdOutErr bool
    35  
    36  	// Enable OmitSpecLabels to prevent labels from appearing in the spec name
    37  	OmitSpecLabels bool
    38  
    39  	// Enable OmitLeafNodeType to prevent the spec leaf node type from appearing in the spec name
    40  	OmitLeafNodeType bool
    41  
    42  	// Enable OmitSuiteSetupNodes to prevent the creation of testcase entries for setup nodes
    43  	OmitSuiteSetupNodes bool
    44  }
    45  
    46  type JUnitTestSuites struct {
    47  	XMLName xml.Name `xml:"testsuites"`
    48  	// Tests maps onto the total number of specs in all test suites (this includes any suite nodes such as BeforeSuite)
    49  	Tests int `xml:"tests,attr"`
    50  	// Disabled maps onto specs that are pending and/or skipped
    51  	Disabled int `xml:"disabled,attr"`
    52  	// Errors maps onto specs that panicked or were interrupted
    53  	Errors int `xml:"errors,attr"`
    54  	// Failures maps onto specs that failed
    55  	Failures int `xml:"failures,attr"`
    56  	// Time is the time in seconds to execute all test suites
    57  	Time float64 `xml:"time,attr"`
    58  
    59  	//The set of all test suites
    60  	TestSuites []JUnitTestSuite `xml:"testsuite"`
    61  }
    62  
    63  type JUnitTestSuite struct {
    64  	// Name maps onto the description of the test suite - maps onto Report.SuiteDescription
    65  	Name string `xml:"name,attr"`
    66  	// Package maps onto the absolute path to the test suite - maps onto Report.SuitePath
    67  	Package string `xml:"package,attr"`
    68  	// Tests maps onto the total number of specs in the test suite (this includes any suite nodes such as BeforeSuite)
    69  	Tests int `xml:"tests,attr"`
    70  	// Disabled maps onto specs that are pending
    71  	Disabled int `xml:"disabled,attr"`
    72  	// Skiped maps onto specs that are skipped
    73  	Skipped int `xml:"skipped,attr"`
    74  	// Errors maps onto specs that panicked or were interrupted
    75  	Errors int `xml:"errors,attr"`
    76  	// Failures maps onto specs that failed
    77  	Failures int `xml:"failures,attr"`
    78  	// Time is the time in seconds to execute all the test suite - maps onto Report.RunTime
    79  	Time float64 `xml:"time,attr"`
    80  	// Timestamp is the ISO 8601 formatted start-time of the suite - maps onto Report.StartTime
    81  	Timestamp string `xml:"timestamp,attr"`
    82  
    83  	//Properties captures the information stored in the rest of the Report type (including SuiteConfig) as key-value pairs
    84  	Properties JUnitProperties `xml:"properties"`
    85  
    86  	//TestCases capture the individual specs
    87  	TestCases []JUnitTestCase `xml:"testcase"`
    88  }
    89  
    90  type JUnitProperties struct {
    91  	Properties []JUnitProperty `xml:"property"`
    92  }
    93  
    94  func (jup JUnitProperties) WithName(name string) string {
    95  	for _, property := range jup.Properties {
    96  		if property.Name == name {
    97  			return property.Value
    98  		}
    99  	}
   100  	return ""
   101  }
   102  
   103  type JUnitProperty struct {
   104  	Name  string `xml:"name,attr"`
   105  	Value string `xml:"value,attr"`
   106  }
   107  
   108  var ownerRE = regexp.MustCompile(`(?i)^owner:(.*)$`)
   109  
   110  type JUnitTestCase struct {
   111  	// Name maps onto the full text of the spec - equivalent to "[SpecReport.LeafNodeType] SpecReport.FullText()"
   112  	Name string `xml:"name,attr"`
   113  	// Classname maps onto the name of the test suite - equivalent to Report.SuiteDescription
   114  	Classname string `xml:"classname,attr"`
   115  	// Status maps onto the string representation of SpecReport.State
   116  	Status string `xml:"status,attr"`
   117  	// Time is the time in seconds to execute the spec - maps onto SpecReport.RunTime
   118  	Time float64 `xml:"time,attr"`
   119  	// Owner is the owner the spec - is set if a label matching Label("owner:X") is provided.  The last matching label is used as the owner, thereby allowing specs to override owners specified in container nodes.
   120  	Owner string `xml:"owner,attr,omitempty"`
   121  	//Skipped is populated with a message if the test was skipped or pending
   122  	Skipped *JUnitSkipped `xml:"skipped,omitempty"`
   123  	//Error is populated if the test panicked or was interrupted
   124  	Error *JUnitError `xml:"error,omitempty"`
   125  	//Failure is populated if the test failed
   126  	Failure *JUnitFailure `xml:"failure,omitempty"`
   127  	//SystemOut maps onto any captured stdout/stderr output - maps onto SpecReport.CapturedStdOutErr
   128  	SystemOut string `xml:"system-out,omitempty"`
   129  	//SystemOut maps onto any captured GinkgoWriter output - maps onto SpecReport.CapturedGinkgoWriterOutput
   130  	SystemErr string `xml:"system-err,omitempty"`
   131  }
   132  
   133  type JUnitSkipped struct {
   134  	// Message maps onto "pending" if the test was marked pending, "skipped" if the test was marked skipped, and "skipped - REASON" if the user called Skip(REASON)
   135  	Message string `xml:"message,attr"`
   136  }
   137  
   138  type JUnitError struct {
   139  	//Message maps onto the panic/exception thrown - equivalent to SpecReport.Failure.ForwardedPanic - or to "interrupted"
   140  	Message string `xml:"message,attr"`
   141  	//Type is one of "panicked" or "interrupted"
   142  	Type string `xml:"type,attr"`
   143  	//Description maps onto the captured stack trace for a panic, or the failure message for an interrupt which will include the dump of running goroutines
   144  	Description string `xml:",chardata"`
   145  }
   146  
   147  type JUnitFailure struct {
   148  	//Message maps onto the failure message - equivalent to SpecReport.Failure.Message
   149  	Message string `xml:"message,attr"`
   150  	//Type is "failed"
   151  	Type string `xml:"type,attr"`
   152  	//Description maps onto the location and stack trace of the failure
   153  	Description string `xml:",chardata"`
   154  }
   155  
   156  func GenerateJUnitReport(report types.Report, dst string) error {
   157  	return GenerateJUnitReportWithConfig(report, dst, JunitReportConfig{})
   158  }
   159  
   160  func GenerateJUnitReportWithConfig(report types.Report, dst string, config JunitReportConfig) error {
   161  	suite := JUnitTestSuite{
   162  		Name:      report.SuiteDescription,
   163  		Package:   report.SuitePath,
   164  		Time:      report.RunTime.Seconds(),
   165  		Timestamp: report.StartTime.Format("2006-01-02T15:04:05"),
   166  		Properties: JUnitProperties{
   167  			Properties: []JUnitProperty{
   168  				{"SuiteSucceeded", fmt.Sprintf("%t", report.SuiteSucceeded)},
   169  				{"SuiteHasProgrammaticFocus", fmt.Sprintf("%t", report.SuiteHasProgrammaticFocus)},
   170  				{"SpecialSuiteFailureReason", strings.Join(report.SpecialSuiteFailureReasons, ",")},
   171  				{"SuiteLabels", fmt.Sprintf("[%s]", strings.Join(report.SuiteLabels, ","))},
   172  				{"RandomSeed", fmt.Sprintf("%d", report.SuiteConfig.RandomSeed)},
   173  				{"RandomizeAllSpecs", fmt.Sprintf("%t", report.SuiteConfig.RandomizeAllSpecs)},
   174  				{"LabelFilter", report.SuiteConfig.LabelFilter},
   175  				{"FocusStrings", strings.Join(report.SuiteConfig.FocusStrings, ",")},
   176  				{"SkipStrings", strings.Join(report.SuiteConfig.SkipStrings, ",")},
   177  				{"FocusFiles", strings.Join(report.SuiteConfig.FocusFiles, ";")},
   178  				{"SkipFiles", strings.Join(report.SuiteConfig.SkipFiles, ";")},
   179  				{"FailOnPending", fmt.Sprintf("%t", report.SuiteConfig.FailOnPending)},
   180  				{"FailFast", fmt.Sprintf("%t", report.SuiteConfig.FailFast)},
   181  				{"FlakeAttempts", fmt.Sprintf("%d", report.SuiteConfig.FlakeAttempts)},
   182  				{"DryRun", fmt.Sprintf("%t", report.SuiteConfig.DryRun)},
   183  				{"ParallelTotal", fmt.Sprintf("%d", report.SuiteConfig.ParallelTotal)},
   184  				{"OutputInterceptorMode", report.SuiteConfig.OutputInterceptorMode},
   185  			},
   186  		},
   187  	}
   188  	for _, spec := range report.SpecReports {
   189  		if config.OmitSuiteSetupNodes && spec.LeafNodeType != types.NodeTypeIt {
   190  			continue
   191  		}
   192  		name := fmt.Sprintf("[%s]", spec.LeafNodeType)
   193  		if config.OmitLeafNodeType {
   194  			name = ""
   195  		}
   196  		if spec.FullText() != "" {
   197  			name = name + " " + spec.FullText()
   198  		}
   199  		labels := spec.Labels()
   200  		if len(labels) > 0 && !config.OmitSpecLabels {
   201  			name = name + " [" + strings.Join(labels, ", ") + "]"
   202  		}
   203  		owner := ""
   204  		for _, label := range labels {
   205  			if matches := ownerRE.FindStringSubmatch(label); len(matches) == 2 {
   206  				owner = matches[1]
   207  			}
   208  		}
   209  		name = strings.TrimSpace(name)
   210  
   211  		test := JUnitTestCase{
   212  			Name:      name,
   213  			Classname: report.SuiteDescription,
   214  			Status:    spec.State.String(),
   215  			Time:      spec.RunTime.Seconds(),
   216  			Owner:     owner,
   217  		}
   218  		if !spec.State.Is(config.OmitTimelinesForSpecState) {
   219  			test.SystemErr = systemErrForUnstructuredReporters(spec)
   220  		}
   221  		if !config.OmitCapturedStdOutErr {
   222  			test.SystemOut = systemOutForUnstructuredReporters(spec)
   223  		}
   224  		suite.Tests += 1
   225  
   226  		switch spec.State {
   227  		case types.SpecStateSkipped:
   228  			message := "skipped"
   229  			if spec.Failure.Message != "" {
   230  				message += " - " + spec.Failure.Message
   231  			}
   232  			test.Skipped = &JUnitSkipped{Message: message}
   233  			suite.Skipped += 1
   234  		case types.SpecStatePending:
   235  			test.Skipped = &JUnitSkipped{Message: "pending"}
   236  			suite.Disabled += 1
   237  		case types.SpecStateFailed:
   238  			test.Failure = &JUnitFailure{
   239  				Message:     spec.Failure.Message,
   240  				Type:        "failed",
   241  				Description: failureDescriptionForUnstructuredReporters(spec),
   242  			}
   243  			if config.OmitFailureMessageAttr {
   244  				test.Failure.Message = ""
   245  			}
   246  			suite.Failures += 1
   247  		case types.SpecStateTimedout:
   248  			test.Failure = &JUnitFailure{
   249  				Message:     spec.Failure.Message,
   250  				Type:        "timedout",
   251  				Description: failureDescriptionForUnstructuredReporters(spec),
   252  			}
   253  			if config.OmitFailureMessageAttr {
   254  				test.Failure.Message = ""
   255  			}
   256  			suite.Failures += 1
   257  		case types.SpecStateInterrupted:
   258  			test.Error = &JUnitError{
   259  				Message:     spec.Failure.Message,
   260  				Type:        "interrupted",
   261  				Description: failureDescriptionForUnstructuredReporters(spec),
   262  			}
   263  			if config.OmitFailureMessageAttr {
   264  				test.Error.Message = ""
   265  			}
   266  			suite.Errors += 1
   267  		case types.SpecStateAborted:
   268  			test.Failure = &JUnitFailure{
   269  				Message:     spec.Failure.Message,
   270  				Type:        "aborted",
   271  				Description: failureDescriptionForUnstructuredReporters(spec),
   272  			}
   273  			if config.OmitFailureMessageAttr {
   274  				test.Failure.Message = ""
   275  			}
   276  			suite.Errors += 1
   277  		case types.SpecStatePanicked:
   278  			test.Error = &JUnitError{
   279  				Message:     spec.Failure.ForwardedPanic,
   280  				Type:        "panicked",
   281  				Description: failureDescriptionForUnstructuredReporters(spec),
   282  			}
   283  			if config.OmitFailureMessageAttr {
   284  				test.Error.Message = ""
   285  			}
   286  			suite.Errors += 1
   287  		}
   288  
   289  		suite.TestCases = append(suite.TestCases, test)
   290  	}
   291  
   292  	junitReport := JUnitTestSuites{
   293  		Tests:      suite.Tests,
   294  		Disabled:   suite.Disabled + suite.Skipped,
   295  		Errors:     suite.Errors,
   296  		Failures:   suite.Failures,
   297  		Time:       suite.Time,
   298  		TestSuites: []JUnitTestSuite{suite},
   299  	}
   300  
   301  	if err := os.MkdirAll(path.Dir(dst), 0770); err != nil {
   302  		return err
   303  	}
   304  	f, err := os.Create(dst)
   305  	if err != nil {
   306  		return err
   307  	}
   308  	f.WriteString(xml.Header)
   309  	encoder := xml.NewEncoder(f)
   310  	encoder.Indent("  ", "    ")
   311  	encoder.Encode(junitReport)
   312  
   313  	return f.Close()
   314  }
   315  
   316  func MergeAndCleanupJUnitReports(sources []string, dst string) ([]string, error) {
   317  	messages := []string{}
   318  	mergedReport := JUnitTestSuites{}
   319  	for _, source := range sources {
   320  		report := JUnitTestSuites{}
   321  		f, err := os.Open(source)
   322  		if err != nil {
   323  			messages = append(messages, fmt.Sprintf("Could not open %s:\n%s", source, err.Error()))
   324  			continue
   325  		}
   326  		err = xml.NewDecoder(f).Decode(&report)
   327  		_ = f.Close()
   328  		if err != nil {
   329  			messages = append(messages, fmt.Sprintf("Could not decode %s:\n%s", source, err.Error()))
   330  			continue
   331  		}
   332  		os.Remove(source)
   333  
   334  		mergedReport.Tests += report.Tests
   335  		mergedReport.Disabled += report.Disabled
   336  		mergedReport.Errors += report.Errors
   337  		mergedReport.Failures += report.Failures
   338  		mergedReport.Time += report.Time
   339  		mergedReport.TestSuites = append(mergedReport.TestSuites, report.TestSuites...)
   340  	}
   341  
   342  	if err := os.MkdirAll(path.Dir(dst), 0770); err != nil {
   343  		return messages, err
   344  	}
   345  	f, err := os.Create(dst)
   346  	if err != nil {
   347  		return messages, err
   348  	}
   349  	f.WriteString(xml.Header)
   350  	encoder := xml.NewEncoder(f)
   351  	encoder.Indent("  ", "    ")
   352  	encoder.Encode(mergedReport)
   353  
   354  	return messages, f.Close()
   355  }
   356  
   357  func failureDescriptionForUnstructuredReporters(spec types.SpecReport) string {
   358  	out := &strings.Builder{}
   359  	NewDefaultReporter(types.ReporterConfig{NoColor: true, VeryVerbose: true}, out).emitFailure(0, spec.State, spec.Failure, true)
   360  	if len(spec.AdditionalFailures) > 0 {
   361  		out.WriteString("\nThere were additional failures detected after the initial failure. These are visible in the timeline\n")
   362  	}
   363  	return out.String()
   364  }
   365  
   366  func systemErrForUnstructuredReporters(spec types.SpecReport) string {
   367  	return RenderTimeline(spec, true)
   368  }
   369  
   370  func RenderTimeline(spec types.SpecReport, noColor bool) string {
   371  	out := &strings.Builder{}
   372  	NewDefaultReporter(types.ReporterConfig{NoColor: noColor, VeryVerbose: true}, out).emitTimeline(0, spec, spec.Timeline())
   373  	return out.String()
   374  }
   375  
   376  func systemOutForUnstructuredReporters(spec types.SpecReport) string {
   377  	return spec.CapturedStdOutErr
   378  }
   379  
   380  // Deprecated JUnitReporter (so folks can still compile their suites)
   381  type JUnitReporter struct{}
   382  
   383  func NewJUnitReporter(_ string) *JUnitReporter                                                  { return &JUnitReporter{} }
   384  func (reporter *JUnitReporter) SuiteWillBegin(_ config.GinkgoConfigType, _ *types.SuiteSummary) {}
   385  func (reporter *JUnitReporter) BeforeSuiteDidRun(_ *types.SetupSummary)                         {}
   386  func (reporter *JUnitReporter) SpecWillRun(_ *types.SpecSummary)                                {}
   387  func (reporter *JUnitReporter) SpecDidComplete(_ *types.SpecSummary)                            {}
   388  func (reporter *JUnitReporter) AfterSuiteDidRun(_ *types.SetupSummary)                          {}
   389  func (reporter *JUnitReporter) SuiteDidEnd(_ *types.SuiteSummary)                               {}
   390
View as plain text