...

Source file src/k8s.io/apimachinery/pkg/util/wait/loop_test.go

Documentation: k8s.io/apimachinery/pkg/util/wait

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package wait
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"reflect"
    24  	"testing"
    25  	"time"
    26  
    27  	"github.com/google/go-cmp/cmp"
    28  	"k8s.io/utils/clock"
    29  	testingclock "k8s.io/utils/clock/testing"
    30  )
    31  
    32  func timerWithClock(t Timer, c clock.WithTicker) Timer {
    33  	switch t := t.(type) {
    34  	case *fixedTimer:
    35  		t.new = c.NewTicker
    36  	case *variableTimer:
    37  		t.new = c.NewTimer
    38  	default:
    39  		panic("unrecognized timer type, cannot inject clock")
    40  	}
    41  	return t
    42  }
    43  
    44  func Test_loopConditionWithContextImmediateDelay(t *testing.T) {
    45  	fakeClock := testingclock.NewFakeClock(time.Time{})
    46  	backoff := Backoff{Duration: time.Second}
    47  
    48  	ctx, cancel := context.WithCancel(context.Background())
    49  	defer cancel()
    50  
    51  	expectedError := errors.New("Expected error")
    52  	var attempt int
    53  	f := ConditionFunc(func() (bool, error) {
    54  		attempt++
    55  		return false, expectedError
    56  	})
    57  
    58  	doneCh := make(chan struct{})
    59  	go func() {
    60  		defer close(doneCh)
    61  		if err := loopConditionUntilContext(ctx, timerWithClock(backoff.Timer(), fakeClock), false, true, f.WithContext()); err == nil || err != expectedError {
    62  			t.Errorf("unexpected error: %v", err)
    63  		}
    64  	}()
    65  
    66  	for !fakeClock.HasWaiters() {
    67  		time.Sleep(time.Microsecond)
    68  	}
    69  
    70  	fakeClock.Step(time.Second - time.Millisecond)
    71  	if attempt != 0 {
    72  		t.Fatalf("should still be waiting for condition")
    73  	}
    74  	fakeClock.Step(2 * time.Millisecond)
    75  
    76  	select {
    77  	case <-doneCh:
    78  	case <-time.After(time.Second):
    79  		t.Fatalf("should have exited after a single loop")
    80  	}
    81  	if attempt != 1 {
    82  		t.Fatalf("expected attempt")
    83  	}
    84  }
    85  
    86  func Test_loopConditionUntilContext_semantic(t *testing.T) {
    87  	defaultCallback := func(_ int) (bool, error) {
    88  		return false, nil
    89  	}
    90  
    91  	conditionErr := errors.New("condition failed")
    92  
    93  	tests := []struct {
    94  		name               string
    95  		immediate          bool
    96  		sliding            bool
    97  		context            func() (context.Context, context.CancelFunc)
    98  		callback           func(calls int) (bool, error)
    99  		cancelContextAfter int
   100  		attemptsExpected   int
   101  		errExpected        error
   102  		timer              Timer
   103  	}{
   104  		{
   105  			name: "condition successful is only one attempt",
   106  			callback: func(attempts int) (bool, error) {
   107  				return true, nil
   108  			},
   109  			attemptsExpected: 1,
   110  		},
   111  		{
   112  			name: "delayed condition successful causes return and attempts",
   113  			callback: func(attempts int) (bool, error) {
   114  				return attempts > 1, nil
   115  			},
   116  			attemptsExpected: 2,
   117  		},
   118  		{
   119  			name: "delayed condition successful causes return and attempts many times",
   120  			callback: func(attempts int) (bool, error) {
   121  				return attempts >= 100, nil
   122  			},
   123  			attemptsExpected: 100,
   124  		},
   125  		{
   126  			name: "condition returns error even if ok is true",
   127  			callback: func(_ int) (bool, error) {
   128  				return true, conditionErr
   129  			},
   130  			attemptsExpected: 1,
   131  			errExpected:      conditionErr,
   132  		},
   133  		{
   134  			name: "condition exits after an error",
   135  			callback: func(_ int) (bool, error) {
   136  				return false, conditionErr
   137  			},
   138  			attemptsExpected: 1,
   139  			errExpected:      conditionErr,
   140  		},
   141  		{
   142  			name:             "context already canceled no attempts expected",
   143  			context:          cancelledContext,
   144  			callback:         defaultCallback,
   145  			attemptsExpected: 0,
   146  			errExpected:      context.Canceled,
   147  		},
   148  		{
   149  			name:    "context already canceled condition success and immediate 1 attempt expected",
   150  			context: cancelledContext,
   151  			callback: func(_ int) (bool, error) {
   152  				return true, nil
   153  			},
   154  			immediate:        true,
   155  			attemptsExpected: 1,
   156  		},
   157  		{
   158  			name:    "context already canceled condition fail and immediate 1 attempt expected",
   159  			context: cancelledContext,
   160  			callback: func(_ int) (bool, error) {
   161  				return false, conditionErr
   162  			},
   163  			immediate:        true,
   164  			attemptsExpected: 1,
   165  			errExpected:      conditionErr,
   166  		},
   167  		{
   168  			name:             "context already canceled and immediate 1 attempt expected",
   169  			context:          cancelledContext,
   170  			callback:         defaultCallback,
   171  			immediate:        true,
   172  			attemptsExpected: 1,
   173  			errExpected:      context.Canceled,
   174  		},
   175  		{
   176  			name:               "context cancelled after 5 attempts",
   177  			context:            defaultContext,
   178  			callback:           defaultCallback,
   179  			cancelContextAfter: 5,
   180  			attemptsExpected:   5,
   181  			errExpected:        context.Canceled,
   182  		},
   183  		{
   184  			name:               "context cancelled and immediate after 5 attempts",
   185  			context:            defaultContext,
   186  			callback:           defaultCallback,
   187  			immediate:          true,
   188  			cancelContextAfter: 5,
   189  			attemptsExpected:   5,
   190  			errExpected:        context.Canceled,
   191  		},
   192  		{
   193  			name:             "context at deadline and immediate 1 attempt expected",
   194  			context:          deadlinedContext,
   195  			callback:         defaultCallback,
   196  			immediate:        true,
   197  			attemptsExpected: 1,
   198  			errExpected:      context.DeadlineExceeded,
   199  		},
   200  		{
   201  			name:             "context at deadline no attempts expected",
   202  			context:          deadlinedContext,
   203  			callback:         defaultCallback,
   204  			attemptsExpected: 0,
   205  			errExpected:      context.DeadlineExceeded,
   206  		},
   207  		{
   208  			name:      "context canceled before the second execution and immediate",
   209  			immediate: true,
   210  			context: func() (context.Context, context.CancelFunc) {
   211  				return context.WithTimeout(context.Background(), time.Second)
   212  			},
   213  			callback: func(attempts int) (bool, error) {
   214  				return false, nil
   215  			},
   216  			attemptsExpected: 1,
   217  			errExpected:      context.DeadlineExceeded,
   218  			timer:            Backoff{Duration: 2 * time.Second}.Timer(),
   219  		},
   220  		{
   221  			name:      "immediate and long duration of condition and sliding false",
   222  			immediate: true,
   223  			sliding:   false,
   224  			context: func() (context.Context, context.CancelFunc) {
   225  				return context.WithTimeout(context.Background(), time.Second)
   226  			},
   227  			callback: func(attempts int) (bool, error) {
   228  				if attempts >= 4 {
   229  					return true, nil
   230  				}
   231  				time.Sleep(time.Second / 5)
   232  				return false, nil
   233  			},
   234  			attemptsExpected: 4,
   235  			timer:            Backoff{Duration: time.Second / 5, Jitter: 0.001}.Timer(),
   236  		},
   237  		{
   238  			name:      "immediate and long duration of condition and sliding true",
   239  			immediate: true,
   240  			sliding:   true,
   241  			context: func() (context.Context, context.CancelFunc) {
   242  				return context.WithTimeout(context.Background(), time.Second)
   243  			},
   244  			callback: func(attempts int) (bool, error) {
   245  				if attempts >= 4 {
   246  					return true, nil
   247  				}
   248  				time.Sleep(time.Second / 5)
   249  				return false, nil
   250  			},
   251  			errExpected:      context.DeadlineExceeded,
   252  			attemptsExpected: 3,
   253  			timer:            Backoff{Duration: time.Second / 5, Jitter: 0.001}.Timer(),
   254  		},
   255  	}
   256  
   257  	for _, test := range tests {
   258  		t.Run(test.name, func(t *testing.T) {
   259  			contextFn := test.context
   260  			if contextFn == nil {
   261  				contextFn = defaultContext
   262  			}
   263  			ctx, cancel := contextFn()
   264  			defer cancel()
   265  
   266  			timer := test.timer
   267  			if timer == nil {
   268  				timer = Backoff{Duration: time.Microsecond}.Timer()
   269  			}
   270  			attempts := 0
   271  			err := loopConditionUntilContext(ctx, timer, test.immediate, test.sliding, func(_ context.Context) (bool, error) {
   272  				attempts++
   273  				defer func() {
   274  					if test.cancelContextAfter > 0 && test.cancelContextAfter == attempts {
   275  						cancel()
   276  					}
   277  				}()
   278  				return test.callback(attempts)
   279  			})
   280  
   281  			if test.errExpected != err {
   282  				t.Errorf("expected error: %v but got: %v", test.errExpected, err)
   283  			}
   284  
   285  			if test.attemptsExpected != attempts {
   286  				t.Errorf("expected attempts count: %d but got: %d", test.attemptsExpected, attempts)
   287  			}
   288  		})
   289  	}
   290  }
   291  
   292  type timerWrapper struct {
   293  	timer   clock.Timer
   294  	resets  []time.Duration
   295  	onReset func(d time.Duration)
   296  }
   297  
   298  func (w *timerWrapper) C() <-chan time.Time { return w.timer.C() }
   299  func (w *timerWrapper) Stop() bool          { return w.timer.Stop() }
   300  func (w *timerWrapper) Reset(d time.Duration) bool {
   301  	w.resets = append(w.resets, d)
   302  	b := w.timer.Reset(d)
   303  	if w.onReset != nil {
   304  		w.onReset(d)
   305  	}
   306  	return b
   307  }
   308  
   309  func Test_loopConditionUntilContext_timings(t *testing.T) {
   310  	// Verify that timings returned by the delay func are passed to the timer, and that
   311  	// the timer advancing is enough to drive the state machine. Not a deep verification
   312  	// of the behavior of the loop, but tests that we drive the scenario to completion.
   313  	tests := []struct {
   314  		name               string
   315  		delayFn            DelayFunc
   316  		immediate          bool
   317  		sliding            bool
   318  		context            func() (context.Context, context.CancelFunc)
   319  		callback           func(calls int, lastInterval time.Duration) (bool, error)
   320  		cancelContextAfter int
   321  		attemptsExpected   int
   322  		errExpected        error
   323  		expectedIntervals  func(t *testing.T, delays []time.Duration, delaysRequested []time.Duration)
   324  	}{
   325  		{
   326  			name:    "condition success",
   327  			delayFn: Backoff{Duration: time.Second, Steps: 2, Factor: 2.0, Jitter: 0}.DelayFunc(),
   328  			callback: func(attempts int, _ time.Duration) (bool, error) {
   329  				return true, nil
   330  			},
   331  			attemptsExpected: 1,
   332  			expectedIntervals: func(t *testing.T, delays []time.Duration, delaysRequested []time.Duration) {
   333  				if reflect.DeepEqual(delays, []time.Duration{time.Second, 2 * time.Second}) {
   334  					return
   335  				}
   336  				if reflect.DeepEqual(delaysRequested, []time.Duration{time.Second}) {
   337  					return
   338  				}
   339  			},
   340  		},
   341  		{
   342  			name:      "condition success and immediate",
   343  			immediate: true,
   344  			delayFn:   Backoff{Duration: time.Second, Steps: 2, Factor: 2.0, Jitter: 0}.DelayFunc(),
   345  			callback: func(attempts int, _ time.Duration) (bool, error) {
   346  				return true, nil
   347  			},
   348  			attemptsExpected: 1,
   349  			expectedIntervals: func(t *testing.T, delays []time.Duration, delaysRequested []time.Duration) {
   350  				if reflect.DeepEqual(delays, []time.Duration{time.Second}) {
   351  					return
   352  				}
   353  				if reflect.DeepEqual(delaysRequested, []time.Duration{}) {
   354  					return
   355  				}
   356  			},
   357  		},
   358  		{
   359  			name:    "condition success and sliding",
   360  			sliding: true,
   361  			delayFn: Backoff{Duration: time.Second, Steps: 2, Factor: 2.0, Jitter: 0}.DelayFunc(),
   362  			callback: func(attempts int, _ time.Duration) (bool, error) {
   363  				return true, nil
   364  			},
   365  			attemptsExpected: 1,
   366  			expectedIntervals: func(t *testing.T, delays []time.Duration, delaysRequested []time.Duration) {
   367  				if reflect.DeepEqual(delays, []time.Duration{time.Second}) {
   368  					return
   369  				}
   370  				if !reflect.DeepEqual(delays, delaysRequested) {
   371  					t.Fatalf("sliding non-immediate should have equal delays: %v", cmp.Diff(delays, delaysRequested))
   372  				}
   373  			},
   374  		},
   375  	}
   376  
   377  	for _, test := range tests {
   378  		t.Run(fmt.Sprintf("%s/sliding=%t/immediate=%t", test.name, test.sliding, test.immediate), func(t *testing.T) {
   379  			contextFn := test.context
   380  			if contextFn == nil {
   381  				contextFn = defaultContext
   382  			}
   383  			ctx, cancel := contextFn()
   384  			defer cancel()
   385  
   386  			fakeClock := &testingclock.FakeClock{}
   387  			var fakeTimers []*timerWrapper
   388  			timerFn := func(d time.Duration) clock.Timer {
   389  				t := fakeClock.NewTimer(d)
   390  				fakeClock.Step(d + 1)
   391  				w := &timerWrapper{timer: t, resets: []time.Duration{d}, onReset: func(d time.Duration) {
   392  					fakeClock.Step(d + 1)
   393  				}}
   394  				fakeTimers = append(fakeTimers, w)
   395  				return w
   396  			}
   397  
   398  			delayFn := test.delayFn
   399  			if delayFn == nil {
   400  				delayFn = Backoff{Duration: time.Microsecond}.DelayFunc()
   401  			}
   402  			var delays []time.Duration
   403  			wrappedDelayFn := func() time.Duration {
   404  				d := delayFn()
   405  				delays = append(delays, d)
   406  				return d
   407  			}
   408  			timer := &variableTimer{fn: wrappedDelayFn, new: timerFn}
   409  
   410  			attempts := 0
   411  			err := loopConditionUntilContext(ctx, timer, test.immediate, test.sliding, func(_ context.Context) (bool, error) {
   412  				attempts++
   413  				defer func() {
   414  					if test.cancelContextAfter > 0 && test.cancelContextAfter == attempts {
   415  						cancel()
   416  					}
   417  				}()
   418  				lastInterval := time.Duration(-1)
   419  				if len(delays) > 0 {
   420  					lastInterval = delays[len(delays)-1]
   421  				}
   422  				return test.callback(attempts, lastInterval)
   423  			})
   424  
   425  			if test.errExpected != err {
   426  				t.Errorf("expected error: %v but got: %v", test.errExpected, err)
   427  			}
   428  
   429  			if test.attemptsExpected != attempts {
   430  				t.Errorf("expected attempts count: %d but got: %d", test.attemptsExpected, attempts)
   431  			}
   432  			switch len(fakeTimers) {
   433  			case 0:
   434  				test.expectedIntervals(t, delays, nil)
   435  			case 1:
   436  				test.expectedIntervals(t, delays, fakeTimers[0].resets)
   437  			default:
   438  				t.Fatalf("expected zero or one timers: %#v", fakeTimers)
   439  			}
   440  		})
   441  	}
   442  }
   443  
   444  // Test_loopConditionUntilContext_timings runs actual timing loops and calculates the delta. This
   445  // test depends on high precision wakeups which depends on low CPU contention so it is not a
   446  // candidate to run during normal unit test execution (nor is it a benchmark or example). Instead,
   447  // it can be run manually if there is a scenario where we suspect the timings are off and other
   448  // tests haven't caught it. A final sanity test that would have to be run serially in isolation.
   449  func Test_loopConditionUntilContext_Elapsed(t *testing.T) {
   450  	const maxAttempts = 10
   451  	// TODO: this may be too aggressive, but the overhead should be minor
   452  	const estimatedLoopOverhead = time.Millisecond
   453  	// estimate how long this delay can be
   454  	intervalMax := func(backoff Backoff) time.Duration {
   455  		d := backoff.Duration
   456  		if backoff.Jitter > 0 {
   457  			d += time.Duration(backoff.Jitter * float64(d))
   458  		}
   459  		return d
   460  	}
   461  	// estimate how short this delay can be
   462  	intervalMin := func(backoff Backoff) time.Duration {
   463  		d := backoff.Duration
   464  		return d
   465  	}
   466  
   467  	// Because timing is dependent other factors in test environments, such as
   468  	// whether the OS or go runtime scheduler wake the timers, excess duration
   469  	// is logged by default and can be converted to a fatal error for testing.
   470  	// fail := t.Fatalf
   471  	fail := t.Logf
   472  
   473  	for _, test := range []struct {
   474  		name    string
   475  		backoff Backoff
   476  		t       reflect.Type
   477  	}{
   478  		{name: "variable timer with jitter", backoff: Backoff{Duration: time.Millisecond, Jitter: 1.0}, t: reflect.TypeOf(&variableTimer{})},
   479  		{name: "fixed timer", backoff: Backoff{Duration: time.Millisecond}, t: reflect.TypeOf(&fixedTimer{})},
   480  		{name: "no-op timer", backoff: Backoff{}, t: reflect.TypeOf(noopTimer{})},
   481  	} {
   482  		t.Run(test.name, func(t *testing.T) {
   483  			var attempts int
   484  			start := time.Now()
   485  			timer := test.backoff.Timer()
   486  			if test.t != reflect.ValueOf(timer).Type() {
   487  				t.Fatalf("unexpected timer type %T: expected %v", timer, test.t)
   488  			}
   489  			if err := loopConditionUntilContext(context.Background(), timer, false, false, func(_ context.Context) (bool, error) {
   490  				attempts++
   491  				if attempts > maxAttempts {
   492  					t.Fatalf("should not reach %d attempts", maxAttempts+1)
   493  				}
   494  				return attempts >= maxAttempts, nil
   495  			}); err != nil {
   496  				t.Fatal(err)
   497  			}
   498  			duration := time.Since(start)
   499  			if min := maxAttempts * intervalMin(test.backoff); duration < min {
   500  				fail("elapsed duration %v < expected min duration %v", duration, min)
   501  			}
   502  			if max := maxAttempts * (intervalMax(test.backoff) + estimatedLoopOverhead); duration > max {
   503  				fail("elapsed duration %v > expected max duration %v", duration, max)
   504  			}
   505  		})
   506  	}
   507  }
   508  
   509  func Benchmark_loopConditionUntilContext_ZeroDuration(b *testing.B) {
   510  	ctx := context.Background()
   511  	b.ResetTimer()
   512  	for i := 0; i < b.N; i++ {
   513  		attempts := 0
   514  		if err := loopConditionUntilContext(ctx, Backoff{Duration: 0}.Timer(), true, false, func(_ context.Context) (bool, error) {
   515  			attempts++
   516  			return attempts >= 100, nil
   517  		}); err != nil {
   518  			b.Fatalf("unexpected err: %v", err)
   519  		}
   520  	}
   521  }
   522  
   523  func Benchmark_loopConditionUntilContext_ShortDuration(b *testing.B) {
   524  	ctx := context.Background()
   525  	b.ResetTimer()
   526  	for i := 0; i < b.N; i++ {
   527  		attempts := 0
   528  		if err := loopConditionUntilContext(ctx, Backoff{Duration: time.Microsecond}.Timer(), true, false, func(_ context.Context) (bool, error) {
   529  			attempts++
   530  			return attempts >= 100, nil
   531  		}); err != nil {
   532  			b.Fatalf("unexpected err: %v", err)
   533  		}
   534  	}
   535  }
   536  

View as plain text