package integration_test import ( "fmt" "os" "regexp" "runtime" "sort" "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2/internal/test_helpers" "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" ) var _ = Describe("Running Specs", func() { denoter := "•" if runtime.GOOS == "windows" { denoter = "+" } Context("when pointed at the current directory", func() { BeforeEach(func() { fm.MountFixture("passing_ginkgo_tests") }) It("should run the tests in the working directory", func() { session := startGinkgo(fm.PathTo("passing_ginkgo_tests"), "--no-color") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring(strings.Repeat(denoter, 4))) Ω(output).Should(ContainSubstring("SUCCESS! -- 5 Passed")) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) Context("when passed an explicit package to run", func() { BeforeEach(func() { fm.MountFixture("passing_ginkgo_tests") }) It("should run the ginkgo style tests", func() { session := startGinkgo(fm.TmpDir, "--no-color", "passing_ginkgo_tests") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring(strings.Repeat(denoter, 4))) Ω(output).Should(ContainSubstring("SUCCESS! -- 5 Passed")) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) Context("when passed a number of packages to run", func() { BeforeEach(func() { fm.MountFixture("passing_ginkgo_tests") fm.MountFixture("more_ginkgo_tests") }) It("should run the ginkgo style tests", func() { session := startGinkgo(fm.TmpDir, "--no-color", "--succinct=false", "passing_ginkgo_tests", "more_ginkgo_tests") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) Context("when passed a number of packages to run, some of which have focused tests", func() { BeforeEach(func() { fm.MountFixture("passing_ginkgo_tests") fm.MountFixture("more_ginkgo_tests") fm.MountFixture("focused") }) It("should exit with a status code of 2 and explain why", func() { session := startGinkgo(fm.TmpDir, "--no-color", "--succinct=false", "-r") Eventually(session).Should(gexec.Exit(types.GINKGO_FOCUS_EXIT_CODE)) output := string(session.Out.Contents()) Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Test Suite Passed")) Ω(output).Should(ContainSubstring("Detected Programmatic Focus - setting exit status to %d", types.GINKGO_FOCUS_EXIT_CODE)) }) Context("when the GINKGO_EDITOR_INTEGRATION environment variable is set", func() { BeforeEach(func() { os.Setenv("GINKGO_EDITOR_INTEGRATION", "true") }) AfterEach(func() { os.Setenv("GINKGO_EDITOR_INTEGRATION", "") }) It("should exit with a status code of 0 to allow a coverage file to be generated", func() { session := startGinkgo(fm.TmpDir, "--no-color", "--succinct=false", "-r") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) }) Context("when told to skipPackages", func() { BeforeEach(func() { fm.MountFixture("passing_ginkgo_tests") fm.MountFixture("more_ginkgo_tests") fm.MountFixture("focused") }) It("should skip packages that match the list", func() { session := startGinkgo(fm.TmpDir, "--no-color", "--skip-package=more_ginkgo_tests,focused", "-r") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) Ω(output).Should(ContainSubstring("Passing_ginkgo_tests Suite")) Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) Ω(output).ShouldNot(ContainSubstring("Focused_fixture Suite")) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) Context("when all packages are skipped", func() { It("should not run anything, but still exit 0", func() { session := startGinkgo(fm.TmpDir, "--no-color", "--skip-package=passing_ginkgo_tests,more_ginkgo_tests,focused", "-r") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) errOutput := string(session.Err.Contents()) Ω(errOutput).Should(ContainSubstring("All tests skipped!")) Ω(output).ShouldNot(ContainSubstring("Passing_ginkgo_tests Suite")) Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) Ω(output).ShouldNot(ContainSubstring("Focused_fixture Suite")) Ω(output).ShouldNot(ContainSubstring("Test Suite Passed")) }) }) }) Context("when there are no tests to run", func() { It("should exit 1", func() { session := startGinkgo(fm.TmpDir, "--no-color", "-r") Eventually(session).Should(gexec.Exit(1)) output := string(session.Err.Contents()) Ω(output).Should(ContainSubstring("Found no test suites")) }) }) Context("when there are test files but `go test` reports there are no tests to run", func() { BeforeEach(func() { fm.MountFixture("no_test_fn") }) It("suggests running ginkgo bootstrap", func() { session := startGinkgo(fm.TmpDir, "--no-color", "-r") Eventually(session).Should(gexec.Exit(0)) output := string(session.Err.Contents()) Ω(output).Should(ContainSubstring(`Found no test suites, did you forget to run "ginkgo bootstrap"?`)) }) It("fails if told to requireSuite", func() { session := startGinkgo(fm.TmpDir, "--no-color", "-r", "-require-suite") Eventually(session).Should(gexec.Exit(1)) output := string(session.Err.Contents()) Ω(output).Should(ContainSubstring(`Found no test suites, did you forget to run "ginkgo bootstrap"?`)) }) }) Context("when told to randomize-suites", func() { BeforeEach(func() { fm.MountFixture("passing_ginkgo_tests") fm.MountFixture("more_ginkgo_tests") }) It("should mix up the order of the test suites", Label("slow"), func() { session := startGinkgo(fm.TmpDir, "--no-color", "--randomize-suites", "-r", "--seed=1") Eventually(session).Should(gexec.Exit(0)) Ω(session).Should(gbytes.Say("More_ginkgo_tests Suite")) Ω(session).Should(gbytes.Say("Passing_ginkgo_tests Suite")) session = startGinkgo(fm.TmpDir, "--no-color", "--randomize-suites", "-r", "--seed=4") Eventually(session).Should(gexec.Exit(0)) Ω(session).Should(gbytes.Say("Passing_ginkgo_tests Suite")) Ω(session).Should(gbytes.Say("More_ginkgo_tests Suite")) }) }) Context("when pointed at a package with xunit style tests", func() { BeforeEach(func() { fm.MountFixture("xunit") }) It("should run the xunit style tests, always setting -test.v and passing in supported go test flags", func() { session := startGinkgo(fm.PathTo("xunit"), "-blockprofile=block-profile.out") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) Ω(output).Should(ContainSubstring("--- PASS: TestAlwaysTrue")) Ω(output).Should(ContainSubstring("Test Suite Passed")) Ω(fm.PathTo("xunit", "block-profile.out")).Should(BeAnExistingFile()) }) }) Context("when pointed at a package with no tests", func() { BeforeEach(func() { fm.MountFixture("no_tests") }) It("should fail", func() { session := startGinkgo(fm.PathTo("no_tests"), "--no-color") Eventually(session).Should(gexec.Exit(1)) Ω(session.Err.Contents()).Should(ContainSubstring("Found no test suites")) }) }) Context("when pointed at a package with tests, but the tests have been excluded via go tags", func() { BeforeEach(func() { fm.MountFixture("no_tagged_tests") }) It("should exit 0, not run anything, and generate a json report if asked", func() { session := startGinkgo(fm.PathTo("no_tagged_tests"), "--no-color", "--json-report=report.json") Eventually(session).Should(gexec.Exit(0)) Ω(session).Should(gbytes.Say("no test files")) report := fm.LoadJSONReports("no_tagged_tests", "report.json")[0] Ω(report.SuiteSucceeded).Should(BeTrue()) Ω(report.SpecialSuiteFailureReasons).Should(ConsistOf("Suite did not run go test reported that no test files were found")) }) }) Context("when pointed at a package that fails to compile", func() { BeforeEach(func() { fm.MountFixture("does_not_compile") }) It("should fail", func() { session := startGinkgo(fm.PathTo("does_not_compile"), "--no-color") Eventually(session).Should(gexec.Exit(1)) output := string(session.Out.Contents()) Ω(output).Should(ContainSubstring("Failed to compile")) }) }) Context("when running in parallel", func() { BeforeEach(func() { fm.MountFixture("passing_ginkgo_tests") }) Context("with a specific number of -procs", func() { It("should use the specified number of processes", func() { session := startGinkgo(fm.PathTo("passing_ginkgo_tests"), "--no-color", "-succinct", "--procs=2") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) Ω(output).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs - 2 procs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s`, regexp.QuoteMeta(denoter))) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) Context("with -p", func() { It("it should autocompute the number of nodes", func() { session := startGinkgo(fm.PathTo("passing_ginkgo_tests"), "--no-color", "-succinct", "-p") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) nodes := runtime.NumCPU() if nodes == 1 { Skip("Can't test parallel testings with 1 CPU") } if nodes > 4 { nodes = nodes - 1 } Ω(output).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs - %d procs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]?s`, nodes, regexp.QuoteMeta(denoter))) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) }) Context("when running in parallel and there are specs marked Serial", Label("slow"), func() { BeforeEach(func() { fm.MountFixture("serial") }) It("runs the serial specs after all the parallel specs have finished", func() { By("running a carefully crafted test without the serial decorator") session := startGinkgo(fm.PathTo("serial"), "--no-color", "--procs=2", "--randomize-all", "--fail-fast", "--", "--no-serial") Eventually(session).Should(gexec.Exit(1)) By("running a carefully crafted test with the serial decorator") session = startGinkgo(fm.PathTo("serial"), "--no-color", "--procs=2", "--randomize-all", "--fail-fast") Eventually(session).Should(gexec.Exit(0)) }) }) Context("when running with ordered specs", func() { BeforeEach(func() { fm.MountFixture("ordered") }) It("always preserve spec order within ordered contexts", func() { By("running a carefully crafted test without the ordered decorator") session := startGinkgo(fm.PathTo("ordered"), "--no-color", "--procs=2", "-v", "--randomize-all", "--fail-fast", "--", "--no-ordered") Eventually(session).Should(gexec.Exit(1)) By("running a carefully crafted test with the ordered decorator") session = startGinkgo(fm.PathTo("ordered"), "--no-color", "--procs=2", "-v", "--randomize-all", "--fail-fast") Eventually(session).Should(gexec.Exit(0)) }) }) Context("when running multiple tests", func() { BeforeEach(func() { fm.MountFixture("passing_ginkgo_tests") fm.MountFixture("more_ginkgo_tests") fm.MountFixture("no_tagged_tests") }) Context("when all the tests pass", func() { Context("with the -r flag", func() { It("should run all the tests (in succinct mode) and succeed", func() { session := startGinkgo(fm.TmpDir, "--no-color", "-r", ".") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) outputLines := strings.Split(output, "\n") Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 3/3 specs [%s]{3} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(outputLines[1]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) Context("with a trailing /...", func() { It("should run all the tests (in succinct mode) and succeed", func() { session := startGinkgo(fm.TmpDir, "--no-color", "./...") Eventually(session).Should(gexec.Exit(0)) output := string(session.Out.Contents()) outputLines := strings.Split(output, "\n") Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 3/3 specs [%s]{3} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(outputLines[1]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) }) Context("when one of the packages has a failing tests", func() { BeforeEach(func() { fm.MountFixture("failing_ginkgo_tests") }) It("should fail and stop running tests", func() { session := startGinkgo(fm.TmpDir, "--no-color", "no_tagged_tests", "passing_ginkgo_tests", "failing_ginkgo_tests", "more_ginkgo_tests") Eventually(session).Should(gexec.Exit(1)) output := string(session.Out.Contents()) outputLines := strings.Split(output, "\n") Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`)) Ω(output).Should(ContainSubstring(fmt.Sprintf("%s [FAILED]", denoter))) Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Test Suite Failed")) }) }) Context("when one of the packages fails to compile", func() { BeforeEach(func() { fm.MountFixture("does_not_compile") }) It("should fail and stop running tests", func() { session := startGinkgo(fm.TmpDir, "--no-color", "no_tagged_tests", "passing_ginkgo_tests", "does_not_compile", "more_ginkgo_tests") Eventually(session).Should(gexec.Exit(1)) output := string(session.Out.Contents()) outputLines := strings.Split(output, "\n") Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(outputLines[2]).Should(ContainSubstring("Failed to compile does_not_compile:")) Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Test Suite Failed")) }) }) Context("when either is the case, but the keep-going flag is set", func() { BeforeEach(func() { fm.MountFixture("does_not_compile") fm.MountFixture("failing_ginkgo_tests") }) It("should soldier on", func() { session := startGinkgo(fm.TmpDir, "--no-color", "-keep-going", "no_tagged_tests", "passing_ginkgo_tests", "does_not_compile", "failing_ginkgo_tests", "more_ginkgo_tests") Eventually(session).Should(gexec.Exit(1)) output := string(session.Out.Contents()) outputLines := strings.Split(output, "\n") Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 5/5 specs [%s]{5} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(outputLines[2]).Should(ContainSubstring("Failed to compile does_not_compile:")) Ω(output).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`)) Ω(output).Should(ContainSubstring(fmt.Sprintf("%s [FAILED]", denoter))) Ω(output).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 3/3 specs [%s]{3} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(output).Should(ContainSubstring("Test Suite Failed")) }) }) }) Context("when running large suites in parallel", Label("slow"), func() { BeforeEach(func() { fm.MountFixture("large") }) It("doesn't miss any tests (a sanity test)", func() { session := startGinkgo(fm.PathTo("large"), "--no-color", "--procs=3", "--json-report=report.json") Eventually(session).Should(gexec.Exit(0)) report := Reports(fm.LoadJSONReports("large", "report.json")[0].SpecReports) expectedNames := []string{} for i := 0; i < 2048; i++ { expectedNames = append(expectedNames, fmt.Sprintf("%d", i)) } names := report.Names() sort.Strings(names) sort.Strings(expectedNames) Ω(names).Should(Equal(expectedNames)) }) }) Context("when running a suite in parallel that may have specs that are not deterministically ordered", func() { BeforeEach(func() { fm.MountFixture("nondeterministic") }) It("successfully generates a stable sort across all parallel processes and ensures exactly the correct specs are run", func() { By("Note: the assertions live in the ReportAfterSuite of the fixture. So we simply assert that we succeeded") session := startGinkgo(fm.PathTo("nondeterministic"), "--no-color", "--procs=3") Eventually(session).Should(gexec.Exit(0)) session = startGinkgo(fm.PathTo("nondeterministic"), "--no-color", "--procs=3", "--randomize-all") Eventually(session).Should(gexec.Exit(0)) }) }) Context("when there is a version mismatch between the cli and the test package", func() { It("emits a useful error and tries running", func() { fm.MountFixture(("version_mismatch")) session := startGinkgo(fm.PathTo("version_mismatch"), "--no-color") Eventually(session).Should(gbytes.Say("Ginkgo detected a version mismatch between the Ginkgo CLI and the version of Ginkgo imported by your packages")) Eventually(session).Should(gbytes.Say("Mismatched package versions found")) Eventually(session).Should(gbytes.Say("2.2.0")) Eventually(session).Should(gbytes.Say("used by version_mismatch")) session.Kill() Eventually(session).Should(gexec.Exit()) }) }) })