package f2 import ( "context" "errors" "fmt" "math/rand" "os" "runtime/debug" "strings" "sync" "testing" "slices" "edge-infra.dev/pkg/lib/fog" ) type Framework interface { Setup(...FrameworkFn) Framework Teardown(...FrameworkFn) Framework BeforeEachFeature(...FeatureFn) Framework AfterEachFeature(...FeatureFn) Framework BeforeEachTest(...FrameworkTestFn) Framework AfterEachTest(...FrameworkTestFn) Framework // Test executes features from a standard Go test function. Test(*testing.T, ...Feature) // TestInParallel executes features in parallel. TestInParallel(*testing.T, ...Feature) // Run executes the collection of tests from TestMain. Run(*testing.M) int // WithLabel adds framework-level labels for skipping or categorizing entire // test suites. Use [FeatureBuilder.WithLabel] to label individual features // within a suite. WithLabel(k string, v ...string) Framework Component(c ...string) Framework Priviledged(c ...string) Framework WithID(c ...string) Framework Slow() Framework Disruptive() Framework Serial() Framework Flaky() Framework } type framework struct { ctx Context actions []action labels map[string]string exts []Extension } func New(ctx context.Context, opts ...Option) Framework { options := makeOptions(opts...) // inject logger into context ctx = fog.IntoContext(ctx, fog.New().WithName("f2")) f := &framework{ actions: []action{ { fns: []FrameworkFn{ func(ctx Context) (Context, error) { // #nosec G404 - random string is cosmetic ctx.RunID = fmt.Sprintf("%08x", rand.Int31()) return ctx, nil }, }, phase: phaseSetup, }, }, ctx: Context{Context: ctx}, labels: make(map[string]string), exts: options.extensions, } // check if any extensions want to configure flags before the framework parses // them for _, e := range f.exts { if b, ok := e.(FlagBinder); ok { b.BindFlags(Flags) } } if err := handleFlags(); err != nil { // TODO: panic is bad panic(fmt.Errorf("failed to parse flags: %w", err)) } // Call each extension to setup lifecycle hooks, add itself to the test context, // and add labels now that flags have been parsed and config is computed. for _, e := range f.exts { e.RegisterFns(f) f.ctx = e.IntoContext(f.ctx) // if the extension also implements Labeler, let it label the framework if l, ok := e.(Labeler); ok { for k, v := range l.Labels() { f.labels[k] = v } } } return f } func (f *framework) Run(m *testing.M) int { return f.run(m) } func (f *framework) run(m TestingMain) (exitCode int) { // fail fast on setup, upon err exit var err error log := fog.FromContext(f.ctx) // recover from any panics, print the stack and set a failing exit code defer func() { if rErr := recover(); rErr != nil { exitCode = 1 log.V(0).Info("Recovering from panic and running finish actions", "err", rErr, "stacktrace", string(debug.Stack())) } }() // attempt to teardown defer func() { // attempt to gracefully clean up. // Upon error, log and continue. // TODO: allow disabling graceful teardown // if f.cfg.DisableGracefulTeardown() { // return // } for _, fin := range f.getActionsInPhase(phaseTeardown) { if f.ctx, err = fin.run(f.ctx); err != nil { exitCode = 1 log.V(0).Error(err, "Cleanup failed", "action", fin.phase) } } }() for _, setup := range f.getActionsInPhase(phaseSetup) { f.ctx, err = setup.run(f.ctx) switch { case errors.Is(err, ErrSkip): log.V(1).Info("Skip detected") os.Exit(0) case err != nil: log.V(0).Info("Failure during phase", "phase", setup.phase, "err", err) panic(err) } } // Execute the test suite exitCode = m.Run() return } func (f *framework) Setup(fns ...FrameworkFn) Framework { f.actions = append(f.actions, action{phase: phaseSetup, fns: fns}) return f } func (f *framework) Teardown(fns ...FrameworkFn) Framework { f.actions = append(f.actions, action{phase: phaseTeardown, fns: fns}) return f } func (f *framework) BeforeEachFeature(fns ...FeatureFn) Framework { f.actions = append(f.actions, action{phase: phaseBeforeFeature, featureFns: fns}) return f } func (f *framework) AfterEachFeature(fns ...FeatureFn) Framework { f.actions = append(f.actions, action{phase: phaseAfterFeature, featureFns: fns}) return f } func (f *framework) BeforeEachTest(fns ...FrameworkTestFn) Framework { f.actions = append(f.actions, action{phase: phaseBeforeTest, testFns: fns}) return f } func (f *framework) AfterEachTest(fns ...FrameworkTestFn) Framework { f.actions = append(f.actions, action{phase: phaseAfterTest, testFns: fns}) return f } func (f *framework) TestInParallel(t *testing.T, features ...Feature) { // TODO: all tests should really run in parallel by default, but need some // mechanism for signaling Serial where necessary // TODO: make sure that framework submodules like K8s can handle parallel execution of lifecycle funcs f.processTests(t, true, features...) } func (f *framework) Test(t *testing.T, features ...Feature) { f.processTests(t, false, features...) } func (f *framework) processTests(t *testing.T, parallel bool, features ...Feature) { if len(features) == 0 { t.Log("No test features provided, skipping") return } f.processTestActions(t, f.getActionsInPhase(phaseBeforeTest)) var wg sync.WaitGroup for _, feature := range features { fcopy := feature // capture copy of feature in closure switch parallel { case true: wg.Add(1) go func(w *sync.WaitGroup, feat Feature) { defer w.Done() f.processTestFeature(t, feat) }(&wg, fcopy) case false: f.processTestFeature(t, fcopy) } } if parallel { wg.Wait() } f.processTestActions(t, f.getActionsInPhase(phaseAfterTest)) } func (f *framework) processTestActions(t *testing.T, actions []action) { var err error for _, action := range actions { if f.ctx, err = action.runWithT(f.ctx, t); err != nil { t.Fatalf("%s failure: %s", action.phase, err) } } } func (f *framework) processFeatureActions(t *testing.T, feat Feature, actions []action) { var err error for _, action := range actions { if f.ctx, err = action.runWithFeature(f.ctx, t, deepCopyFeature(feat)); err != nil { t.Fatalf("%s failure: %s", action.phase, err) } } } func (f *framework) processTestFeature(t *testing.T, feat Feature) { f.processFeatureActions(t, feat, f.getActionsInPhase(phaseBeforeFeature)) f.ctx = f.execFeature(f.ctx, t, feat.Name(), feat) f.processFeatureActions(t, feat, f.getActionsInPhase(phaseAfterFeature)) } func (f *framework) execFeature(ctx Context, t *testing.T, name string, feat Feature) Context { t.Run(name, func(t *testing.T) { // check if feature needs to be skipped mergedLabels, skippedLabels := mergeLabels(f.labels, feat.Labels()) if len(skippedLabels) > 0 { t.Logf("ignoring the following framework labels because they're duplicated on the feature: %s", strings.Join(skippedLabels[:], ", ")) } skip, msg := SkipBasedOnLabels(mergedLabels, Labels, SkipLabels) if skip { t.Skipf("skipping: %s", msg) } setupSteps := getStepsInPhase(feat, phaseBeforeFeature) ctx = f.execSteps(ctx, t, setupSteps) tests := getStepsInPhase(feat, phaseTest) for _, test := range tests { t.Run(test.Name, func(t *testing.T) { ctx = f.execSteps(ctx, t, []Step{test}) }) // TODO: fail fast via config } teardownSteps := getStepsInPhase(feat, phaseAfterFeature) ctx = f.execSteps(ctx, t, teardownSteps) }) return ctx } func (f *framework) execSteps(ctx Context, t *testing.T, steps []Step) Context { for _, s := range steps { ctx = s.Fn(ctx, t) } return ctx } func (f *framework) getActionsInPhase(p Phase) []action { if f.actions == nil { return nil } result := make([]action, 0) for _, a := range f.actions { if a.phase == p { result = append(result, a) } } return result } // you can find a list of labels here: // https://docs.edge-infra.dev/dev/contributing/testing/conventions/ // WithLabel adds a label with one or more values to the framework to describe the block of tests // the framework instance will be used for. Returns a pointer to itself so it // can be chained together to add multiple labels: // // f = f2.New().WithLabel("foo", "bar", "baz", "boo").Flaky(). func (f *framework) WithLabel(key string, values ...string) Framework { if f.labels == nil { f.labels = map[string]string{} } f.labels[key] = commaSepList(values...) return f } // WithID is a label used to associate Tests with specific user-facing features. // Should be identified by unique identifier (GitHub issue number or JIRA ticket) // Can have multiple values, separated by comma. func (f *framework) WithID(feat ...string) Framework { return f.WithLabel("id", feat...) } // Priviledged is a label that defines tests that require escalted privileges in various contexts. // Examples: FolderAdmin, BillingAdmin, ProjectAdmin describe GCP privileges that tests may need. func (f *framework) Priviledged(p ...string) Framework { return f.WithLabel("priviledged", p...) } // Component is syntactic sugar for adding a component label to the test block. // Returns a pointer to itself for function chaining. func (f *framework) Component(c ...string) Framework { return f.WithLabel("component", c...) } // Slow is a label that should be applied to all test suites which take longer // than 5 minutes to execute. func (f *framework) Slow() Framework { return f.WithLabel("slow", "true") } // Disruptive is a label that should be applied to all test suites that are // likely to disrupt other test cases being ran at the same time. func (f *framework) Disruptive() Framework { return f.WithLabel("disruptive", "true") } // Serial is a label that should be applied to all test suites that can't be // run in parallel. func (f *framework) Serial() Framework { return f.WithLabel("serial", "true") } // Flaky is a label that should be applied to tests with inconsistent results. func (f *framework) Flaky() Framework { return f.WithLabel("flaky", "true") } // SkipBasedOnLabels determines if a test should be skipped based on label maps // which should be executed and labels that should be skipped. func SkipBasedOnLabels(test, labels, skip map[string]string) (bool, string) { // handle test block with no labels, dont run it if -labels is provided if len(test) == 0 && len(labels) != 0 { return true, "test without labels skipped due to -labels flag being provided" } // evalute skip labels first bc we should exit sooner on average if skip != nil && len(test) != 0 { for k := range test { if v, ok := skip[k]; ok && checkLabelList(test[k], v) { return true, fmt.Sprintf("test with label %s=%s skipped due to -skip-labels flag", k, test[k]) } } } if len(labels) != 0 { matches := map[string]bool{} // match all labels for k := range labels { matches[k] = false if v, ok := test[k]; ok && checkLabelList(v, labels[k]) { matches[k] = true } } skip := false for _, match := range matches { if !match { skip = true } } if skip { return true, fmt.Sprintf("test without labels skipped due to -labels flag: %v", labels) } } return false, "" } // if the framework and feature both have the same label only use the feature label func mergeLabels(framework, feat map[string]string) (map[string]string, []string) { // if theres no features just return the framework labels if len(feat) == 0 { return framework, nil } skipped := []string{} for k, v := range framework { // if the key already exists in the feature map add to skipped and continue if _, ok := feat[k]; ok { skipped = append(skipped, fmt.Sprintf("%s = %s", k, v)) continue } feat[k] = v } return feat, skipped } // checkLabelList checks if the test labels has at least one of the input labels func checkLabelList(testLabels, inputLabels string) bool { test := strings.Split(testLabels, ",") for _, val := range strings.Split(inputLabels, ",") { if slices.Contains(test, val) { return true } } return false }