1 package integration_test
2
3 import (
4 "fmt"
5 "os"
6 "regexp"
7 "runtime"
8 "sort"
9 "strings"
10
11 . "github.com/onsi/ginkgo/v2"
12 . "github.com/onsi/ginkgo/v2/internal/test_helpers"
13 "github.com/onsi/ginkgo/v2/types"
14 . "github.com/onsi/gomega"
15 "github.com/onsi/gomega/gbytes"
16 "github.com/onsi/gomega/gexec"
17 )
18
19 var _ = Describe("Running Specs", func() {
20 denoter := "•"
21 if runtime.GOOS == "windows" {
22 denoter = "+"
23 }
24
25 Context("when pointed at the current directory", func() {
26 BeforeEach(func() {
27 fm.MountFixture("passing_ginkgo_tests")
28 })
29
30 It("should run the tests in the working directory", func() {
31 session := startGinkgo(fm.PathTo("passing_ginkgo_tests"), "--no-color")
32 Eventually(session).Should(gexec.Exit(0))
33 output := string(session.Out.Contents())
34
35 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite"))
36 Ω(output).Should(ContainSubstring(strings.Repeat(denoter, 4)))
37 Ω(output).Should(ContainSubstring("SUCCESS! -- 5 Passed"))
38 Ω(output).Should(ContainSubstring("Test Suite Passed"))
39 })
40 })
41
42 Context("when passed an explicit package to run", func() {
43 BeforeEach(func() {
44 fm.MountFixture("passing_ginkgo_tests")
45 })
46
47 It("should run the ginkgo style tests", func() {
48 session := startGinkgo(fm.TmpDir, "--no-color", "passing_ginkgo_tests")
49 Eventually(session).Should(gexec.Exit(0))
50 output := string(session.Out.Contents())
51
52 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite"))
53 Ω(output).Should(ContainSubstring(strings.Repeat(denoter, 4)))
54 Ω(output).Should(ContainSubstring("SUCCESS! -- 5 Passed"))
55 Ω(output).Should(ContainSubstring("Test Suite Passed"))
56 })
57 })
58
59 Context("when passed a number of packages to run", func() {
60 BeforeEach(func() {
61 fm.MountFixture("passing_ginkgo_tests")
62 fm.MountFixture("more_ginkgo_tests")
63 })
64
65 It("should run the ginkgo style tests", func() {
66 session := startGinkgo(fm.TmpDir, "--no-color", "--succinct=false", "passing_ginkgo_tests", "more_ginkgo_tests")
67 Eventually(session).Should(gexec.Exit(0))
68 output := string(session.Out.Contents())
69
70 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite"))
71 Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite"))
72 Ω(output).Should(ContainSubstring("Test Suite Passed"))
73 })
74 })
75
76 Context("when passed a number of packages to run, some of which have focused tests", func() {
77 BeforeEach(func() {
78 fm.MountFixture("passing_ginkgo_tests")
79 fm.MountFixture("more_ginkgo_tests")
80 fm.MountFixture("focused")
81 })
82
83 It("should exit with a status code of 2 and explain why", func() {
84 session := startGinkgo(fm.TmpDir, "--no-color", "--succinct=false", "-r")
85 Eventually(session).Should(gexec.Exit(types.GINKGO_FOCUS_EXIT_CODE))
86 output := string(session.Out.Contents())
87
88 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite"))
89 Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite"))
90 Ω(output).Should(ContainSubstring("Test Suite Passed"))
91 Ω(output).Should(ContainSubstring("Detected Programmatic Focus - setting exit status to %d", types.GINKGO_FOCUS_EXIT_CODE))
92 })
93
94 Context("when the GINKGO_EDITOR_INTEGRATION environment variable is set", func() {
95 BeforeEach(func() {
96 os.Setenv("GINKGO_EDITOR_INTEGRATION", "true")
97 })
98 AfterEach(func() {
99 os.Setenv("GINKGO_EDITOR_INTEGRATION", "")
100 })
101 It("should exit with a status code of 0 to allow a coverage file to be generated", func() {
102 session := startGinkgo(fm.TmpDir, "--no-color", "--succinct=false", "-r")
103 Eventually(session).Should(gexec.Exit(0))
104 output := string(session.Out.Contents())
105
106 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite"))
107 Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite"))
108 Ω(output).Should(ContainSubstring("Test Suite Passed"))
109 })
110 })
111 })
112
113 Context("when told to skipPackages", func() {
114 BeforeEach(func() {
115 fm.MountFixture("passing_ginkgo_tests")
116 fm.MountFixture("more_ginkgo_tests")
117 fm.MountFixture("focused")
118 })
119
120 It("should skip packages that match the list", func() {
121 session := startGinkgo(fm.TmpDir, "--no-color", "--skip-package=more_ginkgo_tests,focused", "-r")
122 Eventually(session).Should(gexec.Exit(0))
123 output := string(session.Out.Contents())
124
125 Ω(output).Should(ContainSubstring("Passing_ginkgo_tests Suite"))
126 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite"))
127 Ω(output).ShouldNot(ContainSubstring("Focused_fixture Suite"))
128 Ω(output).Should(ContainSubstring("Test Suite Passed"))
129 })
130
131 Context("when all packages are skipped", func() {
132 It("should not run anything, but still exit 0", func() {
133 session := startGinkgo(fm.TmpDir, "--no-color", "--skip-package=passing_ginkgo_tests,more_ginkgo_tests,focused", "-r")
134 Eventually(session).Should(gexec.Exit(0))
135 output := string(session.Out.Contents())
136 errOutput := string(session.Err.Contents())
137
138 Ω(errOutput).Should(ContainSubstring("All tests skipped!"))
139 Ω(output).ShouldNot(ContainSubstring("Passing_ginkgo_tests Suite"))
140 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite"))
141 Ω(output).ShouldNot(ContainSubstring("Focused_fixture Suite"))
142 Ω(output).ShouldNot(ContainSubstring("Test Suite Passed"))
143 })
144 })
145 })
146
147 Context("when there are no tests to run", func() {
148 It("should exit 1", func() {
149 session := startGinkgo(fm.TmpDir, "--no-color", "-r")
150 Eventually(session).Should(gexec.Exit(1))
151 output := string(session.Err.Contents())
152
153 Ω(output).Should(ContainSubstring("Found no test suites"))
154 })
155 })
156
157 Context("when there are test files but `go test` reports there are no tests to run", func() {
158 BeforeEach(func() {
159 fm.MountFixture("no_test_fn")
160 })
161
162 It("suggests running ginkgo bootstrap", func() {
163 session := startGinkgo(fm.TmpDir, "--no-color", "-r")
164 Eventually(session).Should(gexec.Exit(0))
165 output := string(session.Err.Contents())
166
167 Ω(output).Should(ContainSubstring(`Found no test suites, did you forget to run "ginkgo bootstrap"?`))
168 })
169
170 It("fails if told to requireSuite", func() {
171 session := startGinkgo(fm.TmpDir, "--no-color", "-r", "-require-suite")
172 Eventually(session).Should(gexec.Exit(1))
173 output := string(session.Err.Contents())
174
175 Ω(output).Should(ContainSubstring(`Found no test suites, did you forget to run "ginkgo bootstrap"?`))
176 })
177 })
178
179 Context("when told to randomize-suites", func() {
180 BeforeEach(func() {
181 fm.MountFixture("passing_ginkgo_tests")
182 fm.MountFixture("more_ginkgo_tests")
183 })
184
185 It("should mix up the order of the test suites", Label("slow"), func() {
186 session := startGinkgo(fm.TmpDir, "--no-color", "--randomize-suites", "-r", "--seed=1")
187 Eventually(session).Should(gexec.Exit(0))
188
189 Ω(session).Should(gbytes.Say("More_ginkgo_tests Suite"))
190 Ω(session).Should(gbytes.Say("Passing_ginkgo_tests Suite"))
191
192 session = startGinkgo(fm.TmpDir, "--no-color", "--randomize-suites", "-r", "--seed=4")
193 Eventually(session).Should(gexec.Exit(0))
194
195 Ω(session).Should(gbytes.Say("Passing_ginkgo_tests Suite"))
196 Ω(session).Should(gbytes.Say("More_ginkgo_tests Suite"))
197 })
198 })
199
200 Context("when pointed at a package with xunit style tests", func() {
201 BeforeEach(func() {
202 fm.MountFixture("xunit")
203 })
204
205 It("should run the xunit style tests, always setting -test.v and passing in supported go test flags", func() {
206 session := startGinkgo(fm.PathTo("xunit"), "-blockprofile=block-profile.out")
207 Eventually(session).Should(gexec.Exit(0))
208 output := string(session.Out.Contents())
209
210 Ω(output).Should(ContainSubstring("--- PASS: TestAlwaysTrue"))
211 Ω(output).Should(ContainSubstring("Test Suite Passed"))
212
213 Ω(fm.PathTo("xunit", "block-profile.out")).Should(BeAnExistingFile())
214 })
215 })
216
217 Context("when pointed at a package with no tests", func() {
218 BeforeEach(func() {
219 fm.MountFixture("no_tests")
220 })
221
222 It("should fail", func() {
223 session := startGinkgo(fm.PathTo("no_tests"), "--no-color")
224 Eventually(session).Should(gexec.Exit(1))
225
226 Ω(session.Err.Contents()).Should(ContainSubstring("Found no test suites"))
227 })
228 })
229
230 Context("when pointed at a package with tests, but the tests have been excluded via go tags", func() {
231 BeforeEach(func() {
232 fm.MountFixture("no_tagged_tests")
233 })
234
235 It("should exit 0, not run anything, and generate a json report if asked", func() {
236 session := startGinkgo(fm.PathTo("no_tagged_tests"), "--no-color", "--json-report=report.json")
237 Eventually(session).Should(gexec.Exit(0))
238 Ω(session).Should(gbytes.Say("no test files"))
239
240 report := fm.LoadJSONReports("no_tagged_tests", "report.json")[0]
241 Ω(report.SuiteSucceeded).Should(BeTrue())
242 Ω(report.SpecialSuiteFailureReasons).Should(ConsistOf("Suite did not run go test reported that no test files were found"))
243 })
244 })
245
246 Context("when pointed at a package that fails to compile", func() {
247 BeforeEach(func() {
248 fm.MountFixture("does_not_compile")
249 })
250
251 It("should fail", func() {
252 session := startGinkgo(fm.PathTo("does_not_compile"), "--no-color")
253 Eventually(session).Should(gexec.Exit(1))
254 output := string(session.Out.Contents())
255
256 Ω(output).Should(ContainSubstring("Failed to compile"))
257 })
258 })
259
260 Context("when running in parallel", func() {
261 BeforeEach(func() {
262 fm.MountFixture("passing_ginkgo_tests")
263 })
264
265 Context("with a specific number of -procs", func() {
266 It("should use the specified number of processes", func() {
267 session := startGinkgo(fm.PathTo("passing_ginkgo_tests"), "--no-color", "-succinct", "--procs=2")
268 Eventually(session).Should(gexec.Exit(0))
269 output := string(session.Out.Contents())
270
271 Ω(output).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs - 2 procs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s`, regexp.QuoteMeta(denoter)))
272 Ω(output).Should(ContainSubstring("Test Suite Passed"))
273 })
274 })
275
276 Context("with -p", func() {
277 It("it should autocompute the number of nodes", func() {
278 session := startGinkgo(fm.PathTo("passing_ginkgo_tests"), "--no-color", "-succinct", "-p")
279 Eventually(session).Should(gexec.Exit(0))
280 output := string(session.Out.Contents())
281
282 nodes := runtime.NumCPU()
283 if nodes == 1 {
284 Skip("Can't test parallel testings with 1 CPU")
285 }
286 if nodes > 4 {
287 nodes = nodes - 1
288 }
289 Ω(output).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs - %d procs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]?s`, nodes, regexp.QuoteMeta(denoter)))
290 Ω(output).Should(ContainSubstring("Test Suite Passed"))
291 })
292 })
293 })
294
295 Context("when running in parallel and there are specs marked Serial", Label("slow"), func() {
296 BeforeEach(func() {
297 fm.MountFixture("serial")
298 })
299
300 It("runs the serial specs after all the parallel specs have finished", func() {
301 By("running a carefully crafted test without the serial decorator")
302 session := startGinkgo(fm.PathTo("serial"), "--no-color", "--procs=2", "--randomize-all", "--fail-fast", "--", "--no-serial")
303 Eventually(session).Should(gexec.Exit(1))
304
305 By("running a carefully crafted test with the serial decorator")
306 session = startGinkgo(fm.PathTo("serial"), "--no-color", "--procs=2", "--randomize-all", "--fail-fast")
307 Eventually(session).Should(gexec.Exit(0))
308 })
309 })
310
311 Context("when running with ordered specs", func() {
312 BeforeEach(func() {
313 fm.MountFixture("ordered")
314 })
315
316 It("always preserve spec order within ordered contexts", func() {
317 By("running a carefully crafted test without the ordered decorator")
318 session := startGinkgo(fm.PathTo("ordered"), "--no-color", "--procs=2", "-v", "--randomize-all", "--fail-fast", "--", "--no-ordered")
319 Eventually(session).Should(gexec.Exit(1))
320
321 By("running a carefully crafted test with the ordered decorator")
322 session = startGinkgo(fm.PathTo("ordered"), "--no-color", "--procs=2", "-v", "--randomize-all", "--fail-fast")
323 Eventually(session).Should(gexec.Exit(0))
324 })
325 })
326
327 Context("when running multiple tests", func() {
328 BeforeEach(func() {
329 fm.MountFixture("passing_ginkgo_tests")
330 fm.MountFixture("more_ginkgo_tests")
331 fm.MountFixture("no_tagged_tests")
332 })
333
334 Context("when all the tests pass", func() {
335 Context("with the -r flag", func() {
336 It("should run all the tests (in succinct mode) and succeed", func() {
337 session := startGinkgo(fm.TmpDir, "--no-color", "-r", ".")
338 Eventually(session).Should(gexec.Exit(0))
339 output := string(session.Out.Contents())
340
341 outputLines := strings.Split(output, "\n")
342 Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 3/3 specs [%s]{3} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter)))
343 Ω(outputLines[1]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)"))
344 Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter)))
345 Ω(output).Should(ContainSubstring("Test Suite Passed"))
346 })
347 })
348 Context("with a trailing /...", func() {
349 It("should run all the tests (in succinct mode) and succeed", func() {
350 session := startGinkgo(fm.TmpDir, "--no-color", "./...")
351 Eventually(session).Should(gexec.Exit(0))
352 output := string(session.Out.Contents())
353
354 outputLines := strings.Split(output, "\n")
355 Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 3/3 specs [%s]{3} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter)))
356 Ω(outputLines[1]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)"))
357 Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter)))
358 Ω(output).Should(ContainSubstring("Test Suite Passed"))
359 })
360 })
361 })
362
363 Context("when one of the packages has a failing tests", func() {
364 BeforeEach(func() {
365 fm.MountFixture("failing_ginkgo_tests")
366 })
367
368 It("should fail and stop running tests", func() {
369 session := startGinkgo(fm.TmpDir, "--no-color", "no_tagged_tests", "passing_ginkgo_tests", "failing_ginkgo_tests", "more_ginkgo_tests")
370 Eventually(session).Should(gexec.Exit(1))
371 output := string(session.Out.Contents())
372
373 outputLines := strings.Split(output, "\n")
374 Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)"))
375 Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter)))
376 Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`))
377 Ω(output).Should(ContainSubstring(fmt.Sprintf("%s [FAILED]", denoter)))
378 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite"))
379 Ω(output).Should(ContainSubstring("Test Suite Failed"))
380 })
381 })
382
383 Context("when one of the packages fails to compile", func() {
384 BeforeEach(func() {
385 fm.MountFixture("does_not_compile")
386 })
387
388 It("should fail and stop running tests", func() {
389 session := startGinkgo(fm.TmpDir, "--no-color", "no_tagged_tests", "passing_ginkgo_tests", "does_not_compile", "more_ginkgo_tests")
390 Eventually(session).Should(gexec.Exit(1))
391 output := string(session.Out.Contents())
392
393 outputLines := strings.Split(output, "\n")
394 Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)"))
395 Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter)))
396 Ω(outputLines[2]).Should(ContainSubstring("Failed to compile does_not_compile:"))
397 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite"))
398 Ω(output).Should(ContainSubstring("Test Suite Failed"))
399 })
400 })
401
402 Context("when either is the case, but the keep-going flag is set", func() {
403 BeforeEach(func() {
404 fm.MountFixture("does_not_compile")
405 fm.MountFixture("failing_ginkgo_tests")
406 })
407
408 It("should soldier on", func() {
409 session := startGinkgo(fm.TmpDir, "--no-color", "-keep-going", "no_tagged_tests", "passing_ginkgo_tests", "does_not_compile", "failing_ginkgo_tests", "more_ginkgo_tests")
410 Eventually(session).Should(gexec.Exit(1))
411 output := string(session.Out.Contents())
412
413 outputLines := strings.Split(output, "\n")
414 Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)"))
415 Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter)))
416 Ω(outputLines[2]).Should(ContainSubstring("Failed to compile does_not_compile:"))
417 Ω(output).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`))
418 Ω(output).Should(ContainSubstring(fmt.Sprintf("%s [FAILED]", denoter)))
419 Ω(output).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 3/3 specs [%s]{3} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter)))
420 Ω(output).Should(ContainSubstring("Test Suite Failed"))
421 })
422 })
423 })
424
425 Context("when running large suites in parallel", Label("slow"), func() {
426 BeforeEach(func() {
427 fm.MountFixture("large")
428 })
429
430 It("doesn't miss any tests (a sanity test)", func() {
431 session := startGinkgo(fm.PathTo("large"), "--no-color", "--procs=3", "--json-report=report.json")
432 Eventually(session).Should(gexec.Exit(0))
433 report := Reports(fm.LoadJSONReports("large", "report.json")[0].SpecReports)
434
435 expectedNames := []string{}
436 for i := 0; i < 2048; i++ {
437 expectedNames = append(expectedNames, fmt.Sprintf("%d", i))
438 }
439
440 names := report.Names()
441 sort.Strings(names)
442 sort.Strings(expectedNames)
443 Ω(names).Should(Equal(expectedNames))
444 })
445 })
446
447 Context("when running a suite in parallel that may have specs that are not deterministically ordered", func() {
448 BeforeEach(func() {
449 fm.MountFixture("nondeterministic")
450 })
451 It("successfully generates a stable sort across all parallel processes and ensures exactly the correct specs are run", func() {
452 By("Note: the assertions live in the ReportAfterSuite of the fixture. So we simply assert that we succeeded")
453 session := startGinkgo(fm.PathTo("nondeterministic"), "--no-color", "--procs=3")
454 Eventually(session).Should(gexec.Exit(0))
455
456 session = startGinkgo(fm.PathTo("nondeterministic"), "--no-color", "--procs=3", "--randomize-all")
457 Eventually(session).Should(gexec.Exit(0))
458 })
459 })
460
461 Context("when there is a version mismatch between the cli and the test package", func() {
462 It("emits a useful error and tries running", func() {
463 fm.MountFixture(("version_mismatch"))
464 session := startGinkgo(fm.PathTo("version_mismatch"), "--no-color")
465 Eventually(session).Should(gbytes.Say("Ginkgo detected a version mismatch between the Ginkgo CLI and the version of Ginkgo imported by your packages"))
466 Eventually(session).Should(gbytes.Say("Mismatched package versions found"))
467 Eventually(session).Should(gbytes.Say("2.2.0"))
468 Eventually(session).Should(gbytes.Say("used by version_mismatch"))
469
470 session.Kill()
471 Eventually(session).Should(gexec.Exit())
472 })
473 })
474 })
475
View as plain text