1 package integration_test
2
3 import (
4 "fmt"
5 "os"
6 "strings"
7
8 . "github.com/onsi/ginkgo/v2"
9 . "github.com/onsi/ginkgo/v2/internal/test_helpers"
10 "github.com/onsi/ginkgo/v2/reporters"
11 "github.com/onsi/ginkgo/v2/types"
12 . "github.com/onsi/gomega"
13 "github.com/onsi/gomega/gbytes"
14 "github.com/onsi/gomega/gexec"
15 )
16
17 var _ = Describe("Reporting", func() {
18 BeforeEach(func() {
19 fm.MountFixture("reporting")
20 })
21
22 Describe("in-suite reporting with ReportBeforeEach, ReportAfterEach, ReportBeforeSuite and ReportAfterSuite", func() {
23 It("preview thes uite via ReportBeforeSuite", func() {
24 session := startGinkgo(fm.PathTo("reporting"), "--no-color", "--seed=17", "--procs=2")
25 Eventually(session).Should(gexec.Exit(1))
26
27 report, err := os.ReadFile(fm.PathTo("reporting", "report-before-suite-1.out"))
28 Ω(err).ShouldNot(HaveOccurred())
29 lines := strings.Split(string(report), "\n")
30 Ω(lines).Should(ConsistOf(
31 "ReportingFixture Suite - 17",
32 "7 of 8",
33 ))
34 })
35
36 It("reports on each test via ReportBeforeEach", func() {
37 session := startGinkgo(fm.PathTo("reporting"), "--no-color")
38 Eventually(session).Should(gexec.Exit(1))
39
40 report, err := os.ReadFile(fm.PathTo("reporting", "report-before-each.out"))
41 Ω(err).ShouldNot(HaveOccurred())
42 lines := strings.Split(string(report), "\n")
43 Ω(lines).Should(ConsistOf(
44 "passes - INVALID SPEC STATE",
45 "is labelled - INVALID SPEC STATE",
46 "fails - INVALID SPEC STATE",
47 "panics - INVALID SPEC STATE",
48 "has a progress report - INVALID SPEC STATE",
49 "is pending - pending",
50 "is skipped - INVALID SPEC STATE",
51 "times out and fails during cleanup - INVALID SPEC STATE",
52 "",
53 ))
54 })
55
56 It("reports on each test via ReportAfterEach", func() {
57 session := startGinkgo(fm.PathTo("reporting"), "--no-color")
58 Eventually(session).Should(gexec.Exit(1))
59
60 report, err := os.ReadFile(fm.PathTo("reporting", "report-after-each.out"))
61 Ω(err).ShouldNot(HaveOccurred())
62 lines := strings.Split(string(report), "\n")
63 Ω(lines).Should(ConsistOf(
64 "passes - passed",
65 "is labelled - passed",
66 "fails - failed",
67 "panics - panicked",
68 "has a progress report - passed",
69 "is pending - pending",
70 "is skipped - skipped",
71 "times out and fails during cleanup - timedout",
72 "",
73 ))
74 })
75
76 It("reports on all the tests via ReportAfterSuite", func() {
77 session := startGinkgo(fm.PathTo("reporting"), "--no-color", "--seed=17")
78 Eventually(session).Should(gexec.Exit(1))
79
80 report, err := os.ReadFile(fm.PathTo("reporting", "report-after-suite.out"))
81 Ω(err).ShouldNot(HaveOccurred())
82 lines := strings.Split(string(report), "\n")
83 Ω(lines).Should(ConsistOf(
84 "ReportingFixture Suite - 17",
85 "1: [ReportBeforeSuite] - passed",
86 "1: [BeforeSuite] - passed",
87 "passes - passed",
88 "is labelled - passed",
89 "fails - failed",
90 "panics - panicked",
91 "has a progress report - passed",
92 "is pending - pending",
93 "is skipped - skipped",
94 "times out and fails during cleanup - timedout",
95 "1: [DeferCleanup (Suite)] - passed",
96 "1: [DeferCleanup (Suite)] - passed",
97 "",
98 ))
99 })
100
101 Context("when running in parallel", func() {
102 It("reports only runs ReportBeforeSuite on proc 1 and reports on all the tests via ReportAfterSuite", func() {
103 session := startGinkgo(fm.PathTo("reporting"), "--no-color", "--seed=17", "--procs=2")
104 Eventually(session).Should(gexec.Exit(1))
105
106 By("validating the ReportBeforeSuite report")
107 report, err := os.ReadFile(fm.PathTo("reporting", "report-before-suite-1.out"))
108 Ω(err).ShouldNot(HaveOccurred())
109 lines := strings.Split(string(report), "\n")
110 Ω(lines).Should(ConsistOf(
111 "ReportingFixture Suite - 17",
112 "7 of 8",
113 ))
114
115 By("ensuring there is only one ReportBeforeSuite report")
116 Ω(fm.PathTo("reporting", "report-before-suite-2.out")).ShouldNot(BeARegularFile())
117
118 By("validating the ReportAfterSuite report")
119 report, err = os.ReadFile(fm.PathTo("reporting", "report-after-suite.out"))
120 Ω(err).ShouldNot(HaveOccurred())
121 lines = strings.Split(string(report), "\n")
122 Ω(lines).Should(ConsistOf(
123 "ReportingFixture Suite - 17",
124 "1: [ReportBeforeSuite] - passed",
125 "1: [BeforeSuite] - passed",
126 "2: [BeforeSuite] - passed",
127 "passes - passed",
128 "is labelled - passed",
129 "fails - failed",
130 "panics - panicked",
131 "has a progress report - passed",
132 "is pending - pending",
133 "is skipped - skipped",
134 "times out and fails during cleanup - timedout",
135 "1: [DeferCleanup (Suite)] - passed",
136 "1: [DeferCleanup (Suite)] - passed",
137 "2: [DeferCleanup (Suite)] - passed",
138 "2: [DeferCleanup (Suite)] - passed",
139 "",
140 ))
141 })
142 })
143
144 Context("when a ReportAfterSuite node fails", func() {
145 It("reports on it", func() {
146 session := startGinkgo(fm.PathTo("reporting"), "--no-color", "--seed=17", "--procs=2")
147 Eventually(session).Should(gexec.Exit(1))
148
149 Ω(string(session.Out.Contents())).Should(ContainSubstring("[ReportAfterSuite] my report"))
150 Ω(string(session.Out.Contents())).Should(ContainSubstring("fail!\n In [ReportAfterSuite] at:"))
151 })
152 })
153 })
154
155 Describe("JSON and JUnit reporting", func() {
156 checkJSONReport := func(report types.Report) {
157 Ω(report.SuitePath).Should(Equal(fm.AbsPathTo("reporting")))
158 Ω(report.SuiteDescription).Should(Equal("ReportingFixture Suite"))
159 Ω(report.SuiteConfig.ParallelTotal).Should(Equal(2))
160 Ω(report.SpecReports).Should(HaveLen(16))
161
162 specReports := Reports(report.SpecReports)
163 Ω(specReports.WithLeafNodeType(types.NodeTypeIt)).Should(HaveLen(8))
164 Ω(specReports.Find("passes")).Should(HavePassed())
165 Ω(specReports.Find("is labelled")).Should(HavePassed())
166 Ω(specReports.Find("is labelled").Labels()).Should(Equal([]string{"dog", "cat"}))
167 Ω(specReports.Find("fails")).Should(HaveFailed("fail!", types.FailureNodeIsLeafNode, CapturedGinkgoWriterOutput("some ginkgo-writer output")))
168 Ω(specReports.Find("panics")).Should(HavePanicked("boom"))
169 Ω(specReports.Find("is pending")).Should(BePending())
170 Ω(specReports.Find("is skipped").State).Should(Equal(types.SpecStateSkipped))
171 Ω(specReports.Find("times out and fails during cleanup")).Should(HaveTimedOut("A node timeout occurred"))
172 Ω(specReports.Find("times out and fails during cleanup").AdditionalFailures[0].Failure.Message).Should(Equal("double-whammy"))
173 Ω(specReports.Find("times out and fails during cleanup").AdditionalFailures[0].Failure.FailureNodeType).Should(Equal(types.NodeTypeCleanupAfterEach))
174 Ω(specReports.FindByLeafNodeType(types.NodeTypeReportBeforeSuite)).Should(HavePassed())
175 Ω(specReports.Find("my report")).Should(HaveFailed("fail!", types.FailureNodeIsLeafNode, types.NodeTypeReportAfterSuite))
176 Ω(specReports.FindByLeafNodeType(types.NodeTypeBeforeSuite)).Should(HavePassed())
177 Ω(specReports.FindByLeafNodeType(types.NodeTypeCleanupAfterSuite)).Should(HavePassed())
178
179
180 Ω(specReports.Find("passes").ProgressReports).Should(BeEmpty())
181 Ω(specReports.Find("has a progress report").ProgressReports).ShouldNot(BeEmpty())
182 var highlightedFunction types.FunctionCall
183 for _, functionCall := range specReports.Find("has a progress report").ProgressReports[0].SpecGoroutine().Stack {
184 if functionCall.Highlight {
185 highlightedFunction = functionCall
186 break
187 }
188 }
189 Ω(highlightedFunction).ShouldNot(BeZero())
190 Ω(highlightedFunction.Source[highlightedFunction.SourceHighlight]).Should(ContainSubstring("time.Sleep("))
191 }
192
193 checkJSONSubpackageReport := func(report types.Report) {
194 Ω(report.SuitePath).Should(Equal(fm.AbsPathTo("reporting", "reporting_sub_package")))
195 Ω(report.SuiteDescription).Should(Equal("Reporting SubPackage Suite"))
196 Ω(report.SuiteConfig.ParallelTotal).Should(Equal(2))
197 Ω(report.SpecReports).Should(HaveLen(3))
198
199 specReports := Reports(report.SpecReports)
200 Ω(specReports.Find("passes here too")).Should(HavePassed())
201 Ω(specReports.Find("fails here too")).Should(HaveFailed("fail!", types.FailureNodeIsLeafNode, CapturedStdOutput("some std output")))
202 Ω(specReports.Find("panics here too")).Should(HavePanicked("bam!"))
203 }
204
205 checkJSONFailedCompilationReport := func(report types.Report) {
206 Ω(report.SuitePath).Should(Equal(fm.AbsPathTo("reporting", "malformed_sub_package")))
207 Ω(report.SuiteDescription).Should(Equal(""))
208 Ω(report.SuiteConfig.RandomSeed).Should(Equal(int64(17)))
209 Ω(report.SpecialSuiteFailureReasons).Should(ContainElement(ContainSubstring("Failed to compile malformed_sub_package:")))
210 Ω(report.SpecReports).Should(HaveLen(0))
211 }
212
213 getTestCase := func(name string, tests []reporters.JUnitTestCase) reporters.JUnitTestCase {
214 for _, test := range tests {
215 if test.Name == name {
216 return test
217 }
218 }
219
220 return reporters.JUnitTestCase{}
221 }
222
223 checkJUnitReport := func(suite reporters.JUnitTestSuite) {
224 Ω(suite.Name).Should(Equal("ReportingFixture Suite"))
225 Ω(suite.Package).Should(Equal(fm.AbsPathTo("reporting")))
226 Ω(suite.Tests).Should(Equal(16))
227 Ω(suite.Disabled).Should(Equal(1))
228 Ω(suite.Skipped).Should(Equal(1))
229 Ω(suite.Errors).Should(Equal(1))
230 Ω(suite.Failures).Should(Equal(3))
231 Ω(suite.Properties.WithName("SuiteSucceeded")).Should(Equal("false"))
232 Ω(suite.Properties.WithName("RandomSeed")).Should(Equal("17"))
233 Ω(suite.Properties.WithName("ParallelTotal")).Should(Equal("2"))
234 Ω(getTestCase("[ReportBeforeSuite]", suite.TestCases).Status).Should(Equal("passed"))
235 Ω(getTestCase("[BeforeSuite]", suite.TestCases).Status).Should(Equal("passed"))
236 Ω(getTestCase("[It] reporting test passes", suite.TestCases).Classname).Should(Equal("ReportingFixture Suite"))
237 Ω(getTestCase("[It] reporting test passes", suite.TestCases).Status).Should(Equal("passed"))
238 Ω(getTestCase("[It] reporting test passes", suite.TestCases).Failure).Should(BeNil())
239 Ω(getTestCase("[It] reporting test passes", suite.TestCases).Error).Should(BeNil())
240 Ω(getTestCase("[It] reporting test passes", suite.TestCases).Skipped).Should(BeNil())
241
242 Ω(getTestCase("[It] reporting test labelled tests is labelled [dog, cat]", suite.TestCases).Status).Should(Equal("passed"))
243
244 Ω(getTestCase("[It] reporting test panics", suite.TestCases).Status).Should(Equal("panicked"))
245 Ω(getTestCase("[It] reporting test panics", suite.TestCases).Error.Message).Should(Equal("boom"))
246 Ω(getTestCase("[It] reporting test panics", suite.TestCases).Error.Type).Should(Equal("panicked"))
247
248 Ω(getTestCase("[It] reporting test fails", suite.TestCases).Failure.Message).Should(Equal("fail!"))
249 Ω(getTestCase("[It] reporting test fails", suite.TestCases).Status).Should(Equal("failed"))
250 Ω(getTestCase("[It] reporting test fails", suite.TestCases).SystemErr).Should(ContainSubstring("some ginkgo-writer output"))
251
252 Ω(getTestCase("[It] reporting test is pending", suite.TestCases).Status).Should(Equal("pending"))
253 Ω(getTestCase("[It] reporting test is pending", suite.TestCases).Skipped.Message).Should(Equal("pending"))
254
255 Ω(getTestCase("[DeferCleanup (Suite)]", suite.TestCases).Status).Should(Equal("passed"))
256 Ω(getTestCase("[ReportAfterSuite] my report", suite.TestCases).Status).Should(Equal("failed"))
257
258 Ω(getTestCase("[It] reporting test is skipped", suite.TestCases).Status).Should(Equal("skipped"))
259 Ω(getTestCase("[It] reporting test is skipped", suite.TestCases).Skipped.Message).Should(Equal("skipped - skip"))
260
261 Ω(getTestCase("[It] reporting test times out and fails during cleanup", suite.TestCases).Status).Should(Equal("timedout"))
262 Ω(getTestCase("[It] reporting test times out and fails during cleanup", suite.TestCases).Failure.Message).Should(Equal("A node timeout occurred"))
263 Ω(getTestCase("[It] reporting test times out and fails during cleanup", suite.TestCases).Failure.Description).Should(ContainSubstring("<-ctx.Done()"))
264 Ω(getTestCase("[It] reporting test times out and fails during cleanup", suite.TestCases).Failure.Description).Should(ContainSubstring("[FAILED] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nfailure-after-timeout"))
265 Ω(getTestCase("[It] reporting test times out and fails during cleanup", suite.TestCases).SystemErr).Should(ContainSubstring("[FAILED] double-whammy"))
266
267 buf := gbytes.NewBuffer()
268 fmt.Fprintf(buf, getTestCase("[It] reporting test has a progress report", suite.TestCases).SystemErr)
269 Ω(buf).Should(gbytes.Say(`some ginkgo-writer preamble`))
270 Ω(buf).Should(gbytes.Say(`reporting test has a progress report \(Spec Runtime:`))
271 Ω(buf).Should(gbytes.Say(`goroutine \d+ \[sleep\]`))
272 Ω(buf).Should(gbytes.Say(`>\s*time\.Sleep\(`))
273 Ω(buf).Should(gbytes.Say(`some ginkgo-writer postamble`))
274 }
275
276 checkJUnitSubpackageReport := func(suite reporters.JUnitTestSuite) {
277 Ω(suite.Name).Should(Equal("Reporting SubPackage Suite"))
278 Ω(suite.Package).Should(Equal(fm.AbsPathTo("reporting", "reporting_sub_package")))
279 Ω(suite.Tests).Should(Equal(3))
280 Ω(suite.Errors).Should(Equal(1))
281 Ω(suite.Failures).Should(Equal(1))
282 }
283
284 checkJUnitFailedCompilationReport := func(suite reporters.JUnitTestSuite) {
285 Ω(suite.Name).Should(Equal(""))
286 Ω(suite.Package).Should(Equal(fm.AbsPathTo("reporting", "malformed_sub_package")))
287 Ω(suite.Properties.WithName("SpecialSuiteFailureReason")).Should(ContainSubstring("Failed to compile malformed_sub_package:"))
288 }
289
290 checkUnifiedJUnitReport := func(report reporters.JUnitTestSuites) {
291 Ω(report.TestSuites).Should(HaveLen(3))
292 Ω(report.Tests).Should(Equal(19))
293 Ω(report.Disabled).Should(Equal(2))
294 Ω(report.Errors).Should(Equal(2))
295 Ω(report.Failures).Should(Equal(4))
296
297 checkJUnitReport(report.TestSuites[0])
298 checkJUnitFailedCompilationReport(report.TestSuites[1])
299 checkJUnitSubpackageReport(report.TestSuites[2])
300 }
301
302 checkTeamcityReport := func(data string) {
303 lines := strings.Split(data, "\n")
304 Ω(lines).Should(ContainElement("##teamcity[testSuiteStarted name='ReportingFixture Suite']"))
305
306 Ω(lines).Should(ContainElement("##teamcity[testStarted name='|[BeforeSuite|]']"))
307 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testFinished name='|[BeforeSuite|]'")))
308
309 Ω(lines).Should(ContainElement("##teamcity[testStarted name='|[It|] reporting test passes']"))
310 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testFinished name='|[It|] reporting test passes'")))
311
312 Ω(lines).Should(ContainElement("##teamcity[testStarted name='|[It|] reporting test panics']"))
313 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testFailed name='|[It|] reporting test panics' message='panicked - boom'")))
314 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testFinished name='|[It|] reporting test panics'")))
315
316 Ω(lines).Should(ContainElement("##teamcity[testStarted name='|[It|] reporting test fails']"))
317 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testFailed name='|[It|] reporting test fails' message='failed - fail!'")))
318 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testStdErr name='|[It|] reporting test fails' out='> Enter")))
319 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testFinished name='|[It|] reporting test fails'")))
320
321 Ω(lines).Should(ContainElement("##teamcity[testStarted name='|[It|] reporting test is pending']"))
322 Ω(lines).Should(ContainElement("##teamcity[testIgnored name='|[It|] reporting test is pending' message='pending']"))
323 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testFinished name='|[It|] reporting test is pending'")))
324
325 Ω(lines).Should(ContainElement("##teamcity[testStarted name='|[It|] reporting test is skipped']"))
326 Ω(lines).Should(ContainElement("##teamcity[testIgnored name='|[It|] reporting test is skipped' message='skipped - skip']"))
327 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testFailed name='|[It|] reporting test times out and fails during cleanup' message='timedout")))
328
329 Ω(lines).Should(ContainElement(HavePrefix("##teamcity[testFinished name='|[It|] reporting test is skipped'")))
330
331 Ω(lines).Should(ContainElement("##teamcity[testSuiteFinished name='ReportingFixture Suite']"))
332 }
333
334 checkTeamcitySubpackageReport := func(data string) {
335 lines := strings.Split(data, "\n")
336 Ω(lines).Should(ContainElement("##teamcity[testSuiteStarted name='Reporting SubPackage Suite']"))
337 Ω(lines).Should(ContainElement("##teamcity[testSuiteFinished name='Reporting SubPackage Suite']"))
338 }
339
340 checkTeamcityFailedCompilationReport := func(data string) {
341 lines := strings.Split(data, "\n")
342 Ω(lines).Should(ContainElement("##teamcity[testSuiteStarted name='']"))
343 Ω(lines).Should(ContainElement("##teamcity[testSuiteFinished name='']"))
344 }
345
346 Context("the default behavior", func() {
347 BeforeEach(func() {
348 session := startGinkgo(fm.PathTo("reporting"), "--no-color", "-r", "--keep-going", "--procs=2", "--json-report=out.json", "--junit-report=out.xml", "--teamcity-report=out.tc", "-seed=17")
349 Eventually(session).Should(gexec.Exit(1))
350 Ω(session).ShouldNot(gbytes.Say("Could not open"))
351 })
352
353 It("generates single unified json and junit reports", func() {
354 reports := fm.LoadJSONReports("reporting", "out.json")
355 Ω(reports).Should(HaveLen(3))
356 checkJSONReport(reports[0])
357 checkJSONFailedCompilationReport(reports[1])
358 checkJSONSubpackageReport(reports[2])
359
360 junitReport := fm.LoadJUnitReport("reporting", "out.xml")
361 checkUnifiedJUnitReport(junitReport)
362
363 checkTeamcityReport(fm.ContentOf("reporting", "out.tc"))
364 checkTeamcitySubpackageReport(fm.ContentOf("reporting", "out.tc"))
365 checkTeamcityFailedCompilationReport(fm.ContentOf("reporting", "out.tc"))
366 })
367 })
368
369 Context("with -output-dir", func() {
370 BeforeEach(func() {
371 session := startGinkgo(fm.PathTo("reporting"), "--no-color", "-r", "--keep-going", "--procs=2", "--json-report=out.json", "--junit-report=out.xml", "--teamcity-report=out.tc", "--output-dir=./reports", "-seed=17")
372 Eventually(session).Should(gexec.Exit(1))
373 Ω(session).ShouldNot(gbytes.Say("Could not open"))
374 })
375
376 It("places the single unified json and junit reports in output-dir", func() {
377 reports := fm.LoadJSONReports("reporting", "reports/out.json")
378 Ω(reports).Should(HaveLen(3))
379 checkJSONReport(reports[0])
380 checkJSONFailedCompilationReport(reports[1])
381 checkJSONSubpackageReport(reports[2])
382
383 junitReport := fm.LoadJUnitReport("reporting", "reports/out.xml")
384 checkUnifiedJUnitReport(junitReport)
385
386 checkTeamcityReport(fm.ContentOf("reporting", "reports/out.tc"))
387 checkTeamcitySubpackageReport(fm.ContentOf("reporting", "reports/out.tc"))
388 checkTeamcityFailedCompilationReport(fm.ContentOf("reporting", "reports/out.tc"))
389 })
390 })
391
392 Context("with -keep-separate-reports", func() {
393 BeforeEach(func() {
394 session := startGinkgo(fm.PathTo("reporting"), "--no-color", "-r", "--keep-going", "--procs=2", "--json-report=out.json", "--junit-report=out.xml", "--teamcity-report=out.tc", "--keep-separate-reports", "-seed=17")
395 Eventually(session).Should(gexec.Exit(1))
396 Ω(session).ShouldNot(gbytes.Say("Could not open"))
397 })
398
399 It("keeps the separate reports in their respective packages", func() {
400 reports := fm.LoadJSONReports("reporting", "out.json")
401 Ω(reports).Should(HaveLen(1))
402 checkJSONReport(reports[0])
403 checkJUnitReport(fm.LoadJUnitReport("reporting", "out.xml").TestSuites[0])
404 checkTeamcityReport(fm.ContentOf("reporting", "out.tc"))
405
406 reports = fm.LoadJSONReports("reporting", "reporting_sub_package/out.json")
407 Ω(reports).Should(HaveLen(1))
408 checkJSONSubpackageReport(reports[0])
409 checkJUnitSubpackageReport(fm.LoadJUnitReport("reporting", "reporting_sub_package/out.xml").TestSuites[0])
410 checkTeamcitySubpackageReport(fm.ContentOf("reporting", "reporting_sub_package/out.tc"))
411
412 reports = fm.LoadJSONReports("reporting", "malformed_sub_package/out.json")
413 Ω(reports).Should(HaveLen(1))
414 checkJSONFailedCompilationReport(reports[0])
415 checkJUnitFailedCompilationReport(fm.LoadJUnitReport("reporting", "malformed_sub_package/out.xml").TestSuites[0])
416 checkTeamcityFailedCompilationReport(fm.ContentOf("reporting", "malformed_sub_package/out.tc"))
417
418 Ω(fm.PathTo("reporting", "nonginkgo_sub_package/out.json")).ShouldNot(BeAnExistingFile())
419 Ω(fm.PathTo("reporting", "nonginkgo_sub_package/out.xml")).ShouldNot(BeAnExistingFile())
420 Ω(fm.PathTo("reporting", "nonginkgo_sub_package/out.tc")).ShouldNot(BeAnExistingFile())
421 })
422 })
423
424 Context("with -keep-separate-reports and -output-dir", func() {
425 BeforeEach(func() {
426 session := startGinkgo(fm.PathTo("reporting"), "--no-color", "-r", "--keep-going", "--procs=2", "--json-report=out.json", "--junit-report=out.xml", "--teamcity-report=out.tc", "--keep-separate-reports", "--output-dir=./reports", "-seed=17")
427 Eventually(session).Should(gexec.Exit(1))
428 Ω(session).ShouldNot(gbytes.Say("Could not open"))
429 })
430
431 It("places the separate reports in the -output-dir", func() {
432 reports := fm.LoadJSONReports("reporting", "reports/reporting_out.json")
433 Ω(reports).Should(HaveLen(1))
434 checkJSONReport(reports[0])
435 checkJUnitReport(fm.LoadJUnitReport("reporting", "reports/reporting_out.xml").TestSuites[0])
436 checkTeamcityReport(fm.ContentOf("reporting", "reports/reporting_out.tc"))
437
438 reports = fm.LoadJSONReports("reporting", "reports/reporting_sub_package_out.json")
439 Ω(reports).Should(HaveLen(1))
440 checkJSONSubpackageReport(reports[0])
441 checkJUnitSubpackageReport(fm.LoadJUnitReport("reporting", "reports/reporting_sub_package_out.xml").TestSuites[0])
442 checkTeamcitySubpackageReport(fm.ContentOf("reporting", "reports/reporting_sub_package_out.tc"))
443
444 reports = fm.LoadJSONReports("reporting", "reports/malformed_sub_package_out.json")
445 Ω(reports).Should(HaveLen(1))
446 checkJSONFailedCompilationReport(reports[0])
447 checkJUnitFailedCompilationReport(fm.LoadJUnitReport("reporting", "reports/malformed_sub_package_out.xml").TestSuites[0])
448 checkTeamcityFailedCompilationReport(fm.ContentOf("reporting", "reports/malformed_sub_package_out.tc"))
449
450 Ω(fm.PathTo("reporting", "reports/nonginkgo_sub_package_out.json")).ShouldNot(BeAnExistingFile())
451 })
452 })
453
454 Context("when keep-going is not set and a suite fails", func() {
455 BeforeEach(func() {
456 session := startGinkgo(fm.PathTo("reporting"), "--no-color", "-r", "--procs=2", "--json-report=out.json", "--junit-report=out.xml", "--teamcity-report=out.tc", "-coverprofile=cover.out", "-cpuprofile=cpu.out", "-seed=17", "--output-dir=./reports")
457 Eventually(session).Should(gexec.Exit(1))
458 Ω(session).ShouldNot(gbytes.Say("Could not open"))
459 })
460
461 It("reports about the suites that did not run", func() {
462 reports := fm.LoadJSONReports("reporting", "reports/out.json")
463 Ω(reports).Should(HaveLen(3))
464 checkJSONReport(reports[0])
465 Ω(reports[1].SpecialSuiteFailureReasons).Should(ContainElement(ContainSubstring("Failed to compile malformed_sub_package")))
466 Ω(reports[2].SpecialSuiteFailureReasons).Should(ContainElement("Suite did not run because prior suites failed and --keep-going is not set"))
467 })
468 })
469 })
470 })
471
View as plain text